mirror of
https://github.com/xroche/httrack.git
synced 2026-07-23 01:01:41 +03:00
Compare commits
1 Commits
master
...
prose-typo
| Author | SHA1 | Date | |
|---|---|---|---|
|
0f7b4ed66c |
3
.github/workflows/windows-build.yml
vendored
3
.github/workflows/windows-build.yml
vendored
@@ -202,8 +202,7 @@ jobs:
|
||||
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
# footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581.
|
||||
expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test"
|
||||
expected_skips=" 48_local-crange-memresume.test 71_local-crange-repaircache.test" # pending #581
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
157
html/cmddoc.html
Normal file
157
html/cmddoc.html
Normal file
@@ -0,0 +1,157 @@
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
|
||||
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows 95, windows 98, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
<!--
|
||||
|
||||
body {
|
||||
margin: 0; padding: 0; margin-bottom: 15px; margin-top: 8px;
|
||||
background: #77b;
|
||||
}
|
||||
body, td {
|
||||
font: 14px "Trebuchet MS", Verdana, Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
#subTitle {
|
||||
background: #000; color: #fff; padding: 4px; font-weight: bold;
|
||||
}
|
||||
|
||||
#siteNavigation a, #siteNavigation .current {
|
||||
font-weight: bold; color: #448;
|
||||
}
|
||||
#siteNavigation a:link { text-decoration: none; }
|
||||
#siteNavigation a:visited { text-decoration: none; }
|
||||
|
||||
#siteNavigation .current { background-color: #ccd; }
|
||||
|
||||
#siteNavigation a:hover { text-decoration: none; background-color: #fff; color: #000; }
|
||||
#siteNavigation a:active { text-decoration: none; background-color: #ccc; }
|
||||
|
||||
|
||||
a:link { text-decoration: underline; color: #00f; }
|
||||
a:visited { text-decoration: underline; color: #000; }
|
||||
a:hover { text-decoration: underline; color: #c00; }
|
||||
a:active { text-decoration: underline; }
|
||||
|
||||
#pageContent {
|
||||
clear: both;
|
||||
border-bottom: 6px solid #000;
|
||||
padding: 10px; padding-top: 20px;
|
||||
line-height: 1.65em;
|
||||
background-image: url(images/bg_rings.gif);
|
||||
background-repeat: no-repeat;
|
||||
background-position: top right;
|
||||
}
|
||||
|
||||
#pageContent, #siteNavigation {
|
||||
background-color: #ccd;
|
||||
}
|
||||
|
||||
|
||||
.imgLeft { float: left; margin-right: 10px; margin-bottom: 10px; }
|
||||
.imgRight { float: right; margin-left: 10px; margin-bottom: 10px; }
|
||||
|
||||
hr { height: 1px; color: #000; background-color: #000; margin-bottom: 15px; }
|
||||
|
||||
h1 { margin: 0; font-weight: bold; font-size: 2em; }
|
||||
h2 { margin: 0; font-weight: bold; font-size: 1.6em; }
|
||||
h3 { margin: 0; font-weight: bold; font-size: 1.3em; }
|
||||
h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
.blak { background-color: #000; }
|
||||
.hide { display: none; }
|
||||
.tableWidth { min-width: 400px; }
|
||||
|
||||
.tblRegular { border-collapse: collapse; }
|
||||
.tblRegular td { padding: 6px; background-image: url(fade.gif); border: 2px solid #99c; }
|
||||
.tblHeaderColor, .tblHeaderColor td { background: #99c; }
|
||||
.tblNoBorder td { border: 0; }
|
||||
|
||||
|
||||
// -->
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr>
|
||||
<td><img src="images/header_title_4.gif" width="400" height="34" alt="HTTrack Website Copier" title="" border="0" id="title" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="3" class="tableWidth">
|
||||
<tr>
|
||||
<td id="subTitle">Open Source offline browser</td>
|
||||
</tr>
|
||||
</table>
|
||||
<table width="76%" border="0" align="center" cellspacing="0" cellpadding="0" class="tableWidth">
|
||||
<tr class="blak">
|
||||
<td>
|
||||
<table width="100%" border="0" align="center" cellspacing="1" cellpadding="0">
|
||||
<tr>
|
||||
<td colspan="6">
|
||||
<table width="100%" border="0" align="center" cellspacing="0" cellpadding="10">
|
||||
<tr>
|
||||
<td id="pageContent">
|
||||
<!-- ==================== End prologue ==================== -->
|
||||
|
||||
<h2 align="center"><em>Command-Line Documentation</em></h2>
|
||||
|
||||
<br>
|
||||
|
||||
The command-line version
|
||||
|
||||
<ul>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
<br>A task-oriented guide: how to do the common things, and the defaults that surprise newcomers<br><br>
|
||||
<li><a href="options.html">Command line Options</a></li>
|
||||
<br>List of all powerful command line options<br><br>
|
||||
<li>How to use httrack command-line version:</li>
|
||||
<ul>
|
||||
<li>Open a shell window</li>
|
||||
<br>
|
||||
<br>
|
||||
<li>Type in <tt>httrack</tt> (or the complete path to the httrack executable)</li>
|
||||
<br><small><tt>httrack</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Add the URLs, separated by a blank space</li>
|
||||
<br><small><tt>httrack www.example.com/foo/</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>If you need, add some options (see the <a href="options.html">option list</a>)</li>
|
||||
<br><small><tt>httrack www.example.com/foo/ -O "/webs" -N4 -P proxy.myhost.com:3128</tt></small>
|
||||
<br>
|
||||
<br>
|
||||
<li>Launch the command line, and wait until the mirror is finishing</li>
|
||||
<br><small>You can (especially on the Unix release) press ^C to stop the mirror or put httrack in background</small>
|
||||
<br>
|
||||
<br>
|
||||
</ul>
|
||||
</ul>
|
||||
|
||||
<!-- ==================== Start epilogue ==================== -->
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table width="76%" border="0" align="center" valign="bottom" cellspacing="0" cellpadding="0">
|
||||
<tr>
|
||||
<td id="footer"><small>© 1998-2026 Xavier Roche & other contributors - Web Design: Leto Kauler.</small></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
@@ -323,18 +323,10 @@ options control what HTTrack says about itself.</p>
|
||||
<tr><td><tt>--user-agent (-F)</tt></td><td>The <tt>User-Agent</tt>. Set a browser string to get past crawler blocks; <tt>--user-agent ""</tt> sends none.</td></tr>
|
||||
<tr><td><tt>--referer (-%R), --from (-%E), --language (-%l), --accept (-%a)</tt></td><td>Referer, From, Accept-Language and Accept headers.</td></tr>
|
||||
<tr><td><tt>--headers (-%X)</tt></td><td>Add raw header lines to every request.</td></tr>
|
||||
<tr><td><tt>--footer (-%F)</tt></td><td>A footer written into saved pages (on disk, not a network header). See <b>Footer fields</b> below.</td></tr>
|
||||
<tr><td><tt>--footer (-%F)</tt></td><td>A footer written into saved pages (on disk, not a network header).</td></tr>
|
||||
<tr><td><tt>--cookies (-b), --cookies-file (-%K)</tt></td><td>Accept cookies, and preload a Netscape <tt>cookies.txt</tt>.</td></tr>
|
||||
</table>
|
||||
|
||||
<p><b>Footer fields.</b> A footer with no <tt>%s</tt> may reference named fields:
|
||||
<tt>{addr}</tt>, <tt>{path}</tt>, <tt>{url}</tt>, <tt>{date}</tt> (mirror time),
|
||||
<tt>{lastmodified}</tt> (the page's Last-Modified), <tt>{version}</tt>,
|
||||
<tt>{mime}</tt>, <tt>{charset}</tt>, <tt>{status}</tt> and <tt>{size}</tt>; write
|
||||
<tt>{{</tt> or <tt>}}</tt> for a literal brace. A footer that contains <tt>%s</tt>
|
||||
keeps the older positional form (host, path, date in that order). Example:
|
||||
<tt>-%F "<!-- Mirrored from {url} on {date} -->"</tt>.</p>
|
||||
|
||||
<p><b>Login.</b> For HTTP Basic auth, put the credentials in the URL:
|
||||
<tt>http://user:pass@host/</tt>. An <tt>@</tt> inside the username must be written
|
||||
<tt>%40</tt>. Only Basic is supported, not Digest.</p>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<meta name="description" content="HTTrack is an easy-to-use website mirror utility. It allows you to download a World Wide website from the Internet to a local directory,building recursively all structures, getting html, images, and other files from the server to your computer. Links are rebuiltrelatively so that you can freely browse to the local site (works with any browser). You can mirror several sites together so that you can jump from one toanother. You can, also, update an existing mirror site, or resume an interrupted download. The robot is fully configurable, with an integrated help" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, linux, windows, macos, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<meta name="keywords" content="httrack, HTTRACK, HTTrack, winhttrack, WINHTTRACK, WinHTTrack, offline browser, web mirror utility, aspirateur web, surf offline, web capture, www mirror utility, browse offline, local site builder, website mirroring, aspirateur www, internet grabber, capture de site web, internet tool, hors connexion, unix, dos, windows vista, windows seven, windows 8, solaris, ibm580, AIX 4.0, HTS, HTGet, web aspirator, web aspirateur, libre, GPL, GNU, free software" />
|
||||
<title>HTTrack Website Copier - Offline Browser</title>
|
||||
|
||||
<style type="text/css">
|
||||
@@ -135,7 +135,9 @@ clear language!
|
||||
|
||||
<li><a href="#QG2">Where can I find French/other languages documentation?</a><br></li>
|
||||
|
||||
<li><a href="#QG3">Which systems does HTTrack run on?</a><br></li>
|
||||
<li><a href="#QG3b">Is HTTrack working on Windows Vista/Windows Seven/Windows 8 ?</a><br></li>
|
||||
|
||||
<li><a href="#QG3">Is HTTrack working on Windows 95/98 ?</a><br></li>
|
||||
|
||||
<li><a href="#QG4">What's the difference between HTTrack, WinHTTrack and WebHTTrack?</a><br></li>
|
||||
|
||||
@@ -318,10 +320,10 @@ This can easily done by using filters: go to the Option panel, select the 'Scan
|
||||
<tt>+www.example.com/gallery/trees/*<br>
|
||||
+www.example.com/photos/*</tt><br>
|
||||
<br>
|
||||
This means "accept all links beginning with <tt>www.example.com/gallery/trees/</tt> and <tt>www.example.com/photos/</tt>"
|
||||
This means "accept all links begining with <tt>www.example.com/gallery/trees/</tt> and <tt>www.example.com/photos/</tt>"
|
||||
- the <tt>+</tt> means "accept" and the final <tt>*</tt> means "any character will match after the previous ones".
|
||||
Remember the <tt>*.doc</tt> or <tt>*.zip</tt> encountered when you want to select all files from a certain type on your computer:
|
||||
it is almost the same here, except the beginning "+"<br>
|
||||
it is almost the same here, except the begining "+"<br>
|
||||
<br>
|
||||
Now, we might want to exclude all links in <tt>www.example.com/gallery/trees/hugetrees/</tt>, because with the previous filter,
|
||||
we accepted too many files. Here again, you can add a filter rule to refuse these links. Modify the previous filters to:<br>
|
||||
@@ -329,8 +331,8 @@ we accepted too many files. Here again, you can add a filter rule to refuse thes
|
||||
+www.example.com/photos/*<br>
|
||||
-www.example.com/gallery/trees/hugetrees/*</tt><br>
|
||||
<br>
|
||||
You have noticed the <tt>-</tt> in the beginning of the third rule: this means "refuse links matching the rule"
|
||||
; and the rule is "any files beginning with <tt>www.example.com/gallery/trees/hugetrees/</tt><br>
|
||||
You have noticed the <tt>-</tt> in the begining of the third rule: this means "refuse links matching the rule"
|
||||
; and the rule is "any files begining with <tt>www.example.com/gallery/trees/hugetrees/</tt><br>
|
||||
|
||||
Voila! With these three rules, you have precisely defined what you wanted to capture.<br>
|
||||
<br>
|
||||
@@ -359,12 +361,12 @@ You can freely download it, without paying any fees, copy it to your friends, an
|
||||
There are NO official/authorized resellers, because HTTrack is <b>NOT</b> a commercial product.
|
||||
But you can be charged for duplication fees, or any other services (example: software CDroms or shareware collections, or fees for maintenance),
|
||||
but you should have been informed that the software was free software/GPL, and you <b><u>MUST</u></b> have received a copy of the GNU General Public License.
|
||||
Otherwise this is dishonest and unfair (ie. selling httrack on ebay without telling that it was a free software is a scam).
|
||||
Otherwise this is dishonnest and unfair (ie. selling httrack on ebay without telling that it was a free software is a scam).
|
||||
</em>
|
||||
|
||||
<br><br><a NAME="QG0b">Q: <strong>Are there any risks of viruses with this software?</strong></a><br>
|
||||
A: <em>For the software itself:
|
||||
All official releases (at httrack.com) are checked against all known viruses, and the packaging process is also checked. Archives are stored on Un*x servers, not really concerned by viruses. It has been reported, however, that some rogue freeware sites are embedding free software and freeware inside badware installers. Always download httrack from the main site (www.httrack.com), and never from an untrusted source!<br>
|
||||
All official releases (at httrack.com) are checked against all known viruses, and the packaging process is also checked. Archives are stored on Un*x servers, not really concerned by viruses. It has been reported, however, that some rogue freeware sites are embedding free softwares and freewares inside badware installers. Always download httrack from the main site (www.httrack.com), and never from an untrusted source!<br>
|
||||
For files you are downloading on the WWW using HTTrack: You may encounter websites which were corrupted by viruses, and downloading data on these websites might be dangerous if you execute downloaded executables, or if embedded pages contain infected material (as dangerous as if using a regular Browser). Always ensure that websites you are crawling are safe.
|
||||
(Note: remember that using an antivirus software is a good idea once you are connected to the Internet)</em>
|
||||
|
||||
@@ -374,8 +376,11 @@ A: <em>That's right. You can, however, install WinHTTrack on your own machine, a
|
||||
<br><br><a NAME="QG2">Q: <strong>Where can I find French/other languages documentation?</strong></a><br>
|
||||
A: <em>Windows interface is available on several languages, but not yet the documentation!</em>
|
||||
|
||||
<br><br><a NAME="QG3">Q: <strong>Which systems does HTTrack run on?</strong></a><br>
|
||||
A: <em>HTTrack runs on current Windows, Linux and other Unix-like systems, and macOS. Very old platforms such as Windows 95/98 are no longer supported; you may try an older release (such as 3.33) on those.</em>
|
||||
<br><br><a NAME="QG3b">Q: <strong>Is HTTrack working on Windows Vista/Windows Seven/Windows 8 ?</strong></a><br>
|
||||
A: <em>Yes, it does</em>
|
||||
|
||||
<br><br><a NAME="QG3">Q: <strong>Is HTTrack working on Windows 95/98 ?</strong></a><br>
|
||||
A: <em>No, not anymore. You may try to pick an older release (such as 3.33)</em>
|
||||
|
||||
<br><br><a NAME="QG4">Q: <strong>What's the difference between HTTrack, WinHTTrack and WebHTTrack?</strong></a><br>
|
||||
A: <em>WinHTTrack is the Windows GUI release of HTTrack (with a native graphic shell) and WebHTTrack is the Linux/Posix release of HTTrack (with an html graphic shell)</em>
|
||||
@@ -389,7 +394,7 @@ A: <em>It should. The <tt>configure.ac</tt> may be modified in some cases, howev
|
||||
<br><br><a NAME="QG7">Q: <strong>I use HTTrack for professional purpose. What about restrictions/license fee?</strong></a><br>
|
||||
A: <em>HTTrack is covered by the GNU General Public License (GPL). There is no restrictions using HTTrack for professional purpose,
|
||||
except if you develop a software which uses HTTrack components (parts of the source, or any other component).
|
||||
See the <tt>license.txt</tt> file for more information</em>. See also the next question regarding copyright issues when redistributing downloaded material.
|
||||
See the <tt>license.txt</tt> file for more information</em>. See also the next question regarding copyright issues when reditributing downloaded material.
|
||||
|
||||
<br><br><a NAME="QG7b">Q: <strong>Is there any license royalties for distributing a mirror made with HTTrack?</strong></a><br>
|
||||
A: <em>On the HTTrack side, no. However, sharing, publishing or reusing copyrighted material downloaded from a site requires the authorization of the copyright holders, and possibly paying royalty fees. Always ask the authorization before creating a mirror of a site, even if the site appears to be royalty-free and/or without copyright notice.</em>
|
||||
@@ -410,7 +415,7 @@ There are several reasons (and solutions) for a mirror to fail. Reading the log
|
||||
|
||||
<ul>
|
||||
<li>Links within the site refers to external links, or links located in another (or upper) directories, not captured by default - the use of filters is generally THE solution, as this is one of the powerful option in HTTrack. <u>See the above questions/answers</u>.</li>
|
||||
<li>Website <a href="#Q1b1">'robots.txt' rules</a> forbid access to several website parts - you can disable them, but only with great care!</li>
|
||||
<li>Website <a href="#Q1b1">'robots.txt' rules</a> forbide access to several website parts - you can disable them, but only with great care!</li>
|
||||
<li>HTTrack is filtered (by its default User-agent IDentity) - you can change the Browser User-Agent identity to an anonymous one (MSIE, Netscape..) - here again, use this option with care, as this measure might have been put to avoid some bandwidth abuse (see also the <a href="abuse.html">abuse faq</a>!)</li>
|
||||
</ul>
|
||||
|
||||
@@ -452,14 +457,14 @@ A: <em>Yes, HTTrack does support (since 3.20 release) ipv6 sites, using A/AAAA e
|
||||
A: <em>Check the build options (you may have selected user-defined structure with wrong parameters!)</em>
|
||||
|
||||
<br><br><a NAME="QT5">Q: <strong>When capturing real audio/video links (.ram), I only get a shortcut!</a></strong></a></br>
|
||||
A: <em>The .ra/.rm associated file can be captured together with the shortcut, if proper filters are set. Streaming protocols such as rtsp:// are out of scope: HTTrack is an HTTP/FTP mirror and does not capture rtsp streams.</em>
|
||||
A: <em>Yes, but .ra/.rm associated file should be captured together - except if rtsp:// protocol is used (not supported by HTTrack yet), or if proper filters are needed</em>
|
||||
|
||||
<br><br><a NAME="QT6">Q: <strong>Using user:password@address is not working!</a></strong></a></br>
|
||||
A: <em>Again, first check the <tt>hts-log.txt</tt> and <tt>hts-err.txt</tt> error log files - this can give you precious information<br>
|
||||
The site may have a different authentication scheme - form based authentication, for example.
|
||||
In this case, use the URL capture features of HTTrack, it might work.
|
||||
<br>Note: If your username and/or password contains a '<tt>@</tt>' character, you may have to replace all '<tt>@</tt>'
|
||||
occurrences by '<tt>%40</tt>' so that it can work, such as in <tt>user%40domain.com:foobar@www.foo.com/auth/.
|
||||
occurences by '<tt>%40</tt>' so that it can work, such as in <tt>user%40domain.com:foobar@www.foo.com/auth/.
|
||||
You may have to do the same for all "special" characters like spaces (%20), quotes (%22)..</tt>
|
||||
</em>
|
||||
<br><br>
|
||||
@@ -515,7 +520,7 @@ These rules, stored in a file called robots.txt, are given by the website, to sp
|
||||
- for example, /cgi-bin or large images files.
|
||||
They are followed by default by HTTrack, as it is advised. Therefore, you may miss some files that would have been downloaded without
|
||||
these rules - check in your logs if it is the case:<br>
|
||||
<tt>Info: Note: due to www.foobar.com remote robots.txt rules, links beginning with these path will be forbidden: /cgi-bin/,/images/ (see in the options to disable this)
|
||||
<tt>Info: Note: due to www.foobar.com remote robots.txt rules, links begining with these path will be forbidden: /cgi-bin/,/images/ (see in the options to disable this)
|
||||
</tt>
|
||||
<br>
|
||||
If you want to disable them, just change the corresponding option in the option list! (but only disable this option with great care,
|
||||
@@ -535,7 +540,7 @@ HTTrack must find one. Therefore, two index.html will be produced, one with the
|
||||
<br>
|
||||
It might be a good idea to consider that http://www.foobar.com/ and http://www.foobar.com/index.html are the same links, to avoid
|
||||
duplicate files, isn't it?
|
||||
NO, because the top index (/) can refer to ANY filename, and if index.html is generally the default name, index.htm can be chosen,
|
||||
NO, because the top index (/) can refer to ANY filename, and if index.html is generally the default name, index.htm can be choosen,
|
||||
or index.php3, mydog.jpg, or anything you may imagine. (some webmasters are really crazy)
|
||||
<br>
|
||||
<br>
|
||||
@@ -590,7 +595,8 @@ A: <em>Simply use the <tt>--assume dat=application/x-zip</tt> option
|
||||
A: <em>You may need cookies! Cookies are specific data (for example, your username or password) that are sent to your browser once
|
||||
you have logged in certain sites so that you only have to log-in once. For example, after having entered your username in a website, you can
|
||||
view pages and articles, and the next time you will go to this site, you will not have to re-enter your username/password.<br>
|
||||
To supply your own cookies to an HTTrack project, put a Netscape-format <tt>cookies.txt</tt> file in your project folder, or point HTTrack at one with the <tt>--cookies-file</tt> option. You can export your browser's session cookies to that format with a browser extension.
|
||||
To "merge" your personnal cookies to an HTTrack project, just copy the cookies.txt file from your Netscape folder (or the cookies located into the Temporary Internet Files folder for IE)
|
||||
into your project folder (or even the HTTrack folder)
|
||||
</em>
|
||||
<br>
|
||||
<br>
|
||||
@@ -655,7 +661,7 @@ Shell version, skip some slow files, too.</em><br>
|
||||
|
||||
</a><a NAME="Q3b">Q: <strong>I want to update a site, but it's taking too much time! What's happening?</strong><br>
|
||||
A: <em>First, HTTrack always tries to minimize the download flow by interrogating the server about the
|
||||
file changes. But, because HTTrack has to rescan all files from the beginning to rebuild the local site structure,
|
||||
file changes. But, because HTTrack has to rescan all files from the begining to rebuild the local site structure,
|
||||
it can take some time.
|
||||
Besides, some servers are not very smart and always consider that they get newer files, forcing HTTrack to reload them,
|
||||
even if no changes have been made!
|
||||
@@ -734,7 +740,7 @@ retransferred.</em><br>
|
||||
</a><a NAME="Q7">Q: <strong>I just want to retrieve all ZIP files or other files in a web
|
||||
site/in a page. How do I do it?</strong><br>
|
||||
A: <em>You can use different methods. You can use the 'get files near a link' option if
|
||||
files are in a foreign domain. You can use, too, a filter address: adding <tt>+*.zip</tt>
|
||||
files are in a foreign domain. You can use, too, a filter adress: adding <tt>+*.zip</tt>
|
||||
in the URL list (or in the filter list) will accept all ZIP files, even if these files are
|
||||
outside the address. <br>
|
||||
Example : <tt>httrack www.example.com/someaddress.html +*.zip</tt> will allow
|
||||
|
||||
@@ -90,41 +90,38 @@ offline browser : copy websites to a local directory</p>
|
||||
] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
|
||||
--delayed-type-check</b> ] [ <b>-%D,
|
||||
--cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
|
||||
] [ <b>-%r, --warc</b> ] [ <b>-LN, --long-names[=N]</b> ] [
|
||||
<b>-KN, --keep-links[=N]</b> ] [ <b>-x,
|
||||
--replace-external</b> ] [ <b>-%x, --disable-passwords</b> ]
|
||||
[ <b>-%q, --include-query-string</b> ] [ <b>-%g,
|
||||
--strip-query</b> ] [ <b>-o, --generate-errors</b> ] [
|
||||
<b>-X, --purge-old[=N]</b> ] [ <b>-%p, --preserve</b> ] [
|
||||
<b>-%T, --utf8-conversion</b> ] [ <b>-bN, --cookies[=N]</b>
|
||||
] [ <b>-%K, --cookies-file</b> ] [ <b>-%Y, --why</b> ] [
|
||||
<b>-u, --check-type[=N]</b> ] [ <b>-j, --parse-java[=N]</b>
|
||||
] [ <b>-sN, --robots[=N]</b> ] [ <b>-%h, --http-10</b> ] [
|
||||
<b>-%k, --keep-alive</b> ] [ <b>-%z,
|
||||
--disable-compression</b> ] [ <b>-%B, --tolerant</b> ] [
|
||||
<b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
|
||||
<b>-%A, --assume</b> ] [ <b>-@iN, --protocol[=N]</b> ] [
|
||||
<b>-%w, --disable-module</b> ] [ <b>-F, --user-agent</b> ] [
|
||||
<b>-%R, --referer</b> ] [ <b>-%E, --from</b> ] [ <b>-%F,
|
||||
--footer</b> ] [ <b>-%l, --language</b> ] [ <b>-%a,
|
||||
--accept</b> ] [ <b>-%X, --headers</b> ] [ <b>-C,
|
||||
--cache[=N]</b> ] [ <b>-k, --store-all-in-cache</b> ] [
|
||||
<b>-%n, --do-not-recatch</b> ] [ <b>-%v, --display</b> ] [
|
||||
<b>-Q, --do-not-log</b> ] [ <b>-q, --quiet</b> ] [ <b>-z,
|
||||
--extra-log</b> ] [ <b>-Z, --debug-log</b> ] [ <b>-v,
|
||||
--verbose</b> ] [ <b>-f, --file-log</b> ] [ <b>-f2,
|
||||
--single-log</b> ] [ <b>-I, --index</b> ] [ <b>-%i,
|
||||
--build-top-index</b> ] [ <b>-%I, --search-index</b> ] [
|
||||
<b>-pN, --priority[=N]</b> ] [ <b>-S, --stay-on-same-dir</b>
|
||||
] [ <b>-D, --can-go-down</b> ] [ <b>-U, --can-go-up</b> ] [
|
||||
<b>-B, --can-go-up-and-down</b> ] [ <b>-a,
|
||||
--stay-on-same-address</b> ] [ <b>-d,
|
||||
] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
|
||||
--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
|
||||
<b>-%x, --disable-passwords</b> ] [ <b>-%q,
|
||||
--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
|
||||
<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
|
||||
] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
|
||||
] [ <b>-bN, --cookies[=N]</b> ] [ <b>-%K, --cookies-file</b>
|
||||
] [ <b>-%Y, --why</b> ] [ <b>-u, --check-type[=N]</b> ] [
|
||||
<b>-j, --parse-java[=N]</b> ] [ <b>-sN, --robots[=N]</b> ] [
|
||||
<b>-%h, --http-10</b> ] [ <b>-%k, --keep-alive</b> ] [
|
||||
<b>-%B, --tolerant</b> ] [ <b>-%s, --updatehack</b> ] [
|
||||
<b>-%u, --urlhack</b> ] [ <b>-%A, --assume</b> ] [ <b>-@iN,
|
||||
--protocol[=N]</b> ] [ <b>-%w, --disable-module</b> ] [
|
||||
<b>-F, --user-agent</b> ] [ <b>-%R, --referer</b> ] [
|
||||
<b>-%E, --from</b> ] [ <b>-%F, --footer</b> ] [ <b>-%l,
|
||||
--language</b> ] [ <b>-%a, --accept</b> ] [ <b>-%X,
|
||||
--headers</b> ] [ <b>-C, --cache[=N]</b> ] [ <b>-k,
|
||||
--store-all-in-cache</b> ] [ <b>-%n, --do-not-recatch</b> ]
|
||||
[ <b>-%v, --display</b> ] [ <b>-Q, --do-not-log</b> ] [
|
||||
<b>-q, --quiet</b> ] [ <b>-z, --extra-log</b> ] [ <b>-Z,
|
||||
--debug-log</b> ] [ <b>-v, --verbose</b> ] [ <b>-f,
|
||||
--file-log</b> ] [ <b>-f2, --single-log</b> ] [ <b>-I,
|
||||
--index</b> ] [ <b>-%i, --build-top-index</b> ] [ <b>-%I,
|
||||
--search-index</b> ] [ <b>-pN, --priority[=N]</b> ] [ <b>-S,
|
||||
--stay-on-same-dir</b> ] [ <b>-D, --can-go-down</b> ] [
|
||||
<b>-U, --can-go-up</b> ] [ <b>-B, --can-go-up-and-down</b> ]
|
||||
[ <b>-a, --stay-on-same-address</b> ] [ <b>-d,
|
||||
--stay-on-same-domain</b> ] [ <b>-l, --stay-on-same-tld</b>
|
||||
] [ <b>-e, --go-everywhere</b> ] [ <b>-%H,
|
||||
--debug-headers</b> ] [ <b>-%!,
|
||||
--disable-security-limits</b> ] [ <b>-V, --userdef-cmd</b> ]
|
||||
[ <b>-%W, --callback</b> ] [ <b>-y,
|
||||
--background-on-suspend</b> ] [ <b>-K, --keep-links[=N]</b>
|
||||
[ <b>-%W, --callback</b> ] [ <b>-K, --keep-links[=N]</b>
|
||||
]</p>
|
||||
|
||||
<h2>DESCRIPTION
|
||||
@@ -649,31 +646,6 @@ don’t wait) (--cached-delayed-type-check)</p></td></tr>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%r</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
|
||||
sets the output name, --warc-max-size N rotates segments
|
||||
past N bytes (--warc)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%t</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>keep the original file extension, don’t rewrite it
|
||||
from the MIME type (%t0 rewrite)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-LN</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
@@ -906,18 +878,6 @@ for small files and test requests (%k0 don’t use)
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%z</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>do not request compressed content (%z0 request)
|
||||
(--disable-compression)</p> </td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-%B</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
@@ -1063,10 +1023,9 @@ headers (-F "user-agent name") (--user-agent
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>footer string in Html code (-%F "Mirrored from
|
||||
{url} on {date}"; fields {addr} {path} {url} {date}
|
||||
{lastmodified} {version} {mime} {charset} {status} {size},
|
||||
or legacy %s) (--footer <param>)</p></td></tr>
|
||||
<p>footer string in Html code (-%F "Mirrored [from
|
||||
host %s [file %s [at %s]]]" (--footer
|
||||
<param>)</p> </td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
@@ -1424,13 +1383,25 @@ p7 get html files before, then treat other files</p>
|
||||
<td width="8%">
|
||||
|
||||
|
||||
<p style="margin-top: 1em">-#test</p></td>
|
||||
<p style="margin-top: 1em">-#X</p></td>
|
||||
<td width="1%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p style="margin-top: 1em">list engine self-tests (run one
|
||||
with -#test=NAME [args])</p></td></tr>
|
||||
<p style="margin-top: 1em">*use optimized engine (limited
|
||||
memory boundary checks) (--fast-engine)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="8%">
|
||||
|
||||
|
||||
<p>-#test</p></td>
|
||||
<td width="1%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>list engine self-tests (run one with -#test=NAME
|
||||
[args])</p> </td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="8%">
|
||||
@@ -1561,6 +1532,17 @@ with -#test=NAME [args])</p></td></tr>
|
||||
<td width="8%">
|
||||
|
||||
|
||||
<p>-#R</p></td>
|
||||
<td width="1%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>old FTP routines (debug) (--repair-cache)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="8%">
|
||||
|
||||
|
||||
<p>-#T</p></td>
|
||||
<td width="1%"></td>
|
||||
<td width="82%">
|
||||
@@ -1651,18 +1633,6 @@ each files ($0 is the filename: -V "rm \$0")
|
||||
|
||||
<p>use an external library function as a wrapper (-%W
|
||||
myfoo.so[,myparameters]) (--callback <param>)</p></td></tr>
|
||||
<tr valign="top" align="left">
|
||||
<td width="9%"></td>
|
||||
<td width="4%">
|
||||
|
||||
|
||||
<p>-y</p></td>
|
||||
<td width="5%"></td>
|
||||
<td width="82%">
|
||||
|
||||
|
||||
<p>go to background when suspended (y0 don’t)
|
||||
(--background-on-suspend)</p> </td></tr>
|
||||
</table>
|
||||
|
||||
<h3>Details: Option N
|
||||
|
||||
@@ -109,12 +109,12 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li>How to Use</li>
|
||||
<ul>
|
||||
<li><a href="shelldoc.html">WinHTTrack/WebHTTrack (GUI version for Windows or Linux/Unix)</a></li>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix release (WebHTTrack)<br>
|
||||
<br>HTTrack GUI documentation, with step-by-step example, for the Windows release (WinHTTrack) and the Linux/Unix relese (WebHTTrack)<br>
|
||||
<br>
|
||||
<li><a href="android.html">HTTrack on Android</a></li>
|
||||
<br>Step-by-step guide for the Android app<br>
|
||||
<br>
|
||||
<li><a href="httrack.man.html">Command-line version</a></li>
|
||||
<li><a href="cmddoc.html">Command-line version</a></li>
|
||||
<br>How to run HTTrack from the command line, and where to find every option<br>
|
||||
<br>
|
||||
<li><a href="cmdguide.html">Command-line Guide</a></li>
|
||||
@@ -123,7 +123,7 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
<li><a href="fcguide.html">HTTrack Users Guide By Fred Cohen</a></li>
|
||||
<br>A tutorial that describes all command-line options, for Linux and Windows users<br>
|
||||
<br>
|
||||
<li><a href="dev.html">Developer/Programming</a></li>
|
||||
<li><a href="dev.html">Developper/Programming</a></li>
|
||||
<br>How to use HTTrack in batch files, and how to use the library<br>
|
||||
</ul>
|
||||
<br>
|
||||
|
||||
@@ -101,10 +101,6 @@ h4 { margin: 0; font-weight: bold; font-size: 1.18em; }
|
||||
|
||||
<h2 align="center"><em>Options</em></h2>
|
||||
|
||||
<p><strong>Note:</strong> this is a legacy option list and may be out of date.
|
||||
For the authoritative, always-current reference see <a href="httrack.man.html">the httrack manual page</a>,
|
||||
and for a task-oriented walkthrough see the <a href="cmdguide.html">Command-line Guide</a>.</p>
|
||||
|
||||
<ul>
|
||||
<li>Filters: <a href="filters.html">how to use them</a></li>
|
||||
<br><small>Here you can find informations on filters: how to accept all gif files in a mirror, for example</small>
|
||||
|
||||
@@ -103,17 +103,6 @@ ${do:end-if}
|
||||
> ${LANG_I61}
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="warc" ${checked:warc}
|
||||
title='${html:LANG_WARCTIP}' onMouseOver="info('${html:LANG_WARCTIP}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_WARC}
|
||||
<br><br>
|
||||
|
||||
${LANG_WARCFILE}
|
||||
<input name="warcfile" value="${warcfile}" size="40"
|
||||
title='${html:LANG_WARCFILETIP}' onMouseOver="info('${html:LANG_WARCFILETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
<br><br>
|
||||
|
||||
<input type="checkbox" name="norecatch" ${checked:norecatch}
|
||||
title='${html:LANG_I5b}' onMouseOver="info('${html:LANG_I5b}'); return true" onMouseOut="info(' '); return true"
|
||||
> ${LANG_I34b}
|
||||
|
||||
@@ -141,8 +141,6 @@ ${do:copy:KeepSlashes:keepslashes}
|
||||
${do:copy:KeepQueryOrder:keepqueryorder}
|
||||
${do:copy:StripQuery:stripquery}
|
||||
${do:copy:StoreAllInCache:cache2}
|
||||
${do:copy:Warc:warc}
|
||||
${do:copy:WarcFile:warcfile}
|
||||
${do:copy:LogType:logtype}
|
||||
${do:copy:UseHTTPProxyForFTP:ftpprox}
|
||||
${do:copy:ProxyType:proxytype}
|
||||
|
||||
@@ -95,7 +95,7 @@ ${do:end-if}
|
||||
|
||||
<table border="0" width="100%">
|
||||
<tr><td width="90%">
|
||||
<h2 align="center"><em>${LANG_G44}</em></h2>
|
||||
<h2 align="center"><em>Select URLs</em></h2>
|
||||
</td>
|
||||
${/* show help only if available */}
|
||||
${do:if-file-exists:html/index.html}
|
||||
|
||||
@@ -77,7 +77,7 @@ ${do:end-if}
|
||||
|
||||
<table border="0" width="100%">
|
||||
<tr><td width="90%">
|
||||
<h2 align="center"><em>${LANG_J9}</em></h2>
|
||||
<h2 align="center"><em>Start</em></h2>
|
||||
</td>
|
||||
${/* show help only if available */}
|
||||
${do:if-file-exists:html/index.html}
|
||||
@@ -187,8 +187,6 @@ ${do:end-if}
|
||||
${test:toler:--tolerant}
|
||||
${test:http10:--http-10}
|
||||
${test:cache2:--store-all-in-cache}
|
||||
${test:warc:--warc}
|
||||
${test:warcfile:--warc-file "}${html:warcfile}${test:warcfile:"}
|
||||
${test:norecatch:--do-not-recatch}
|
||||
${test:logf:--single-log}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
@@ -239,8 +237,6 @@ KeepSlashes=${ztest:keepslashes:0:1}
|
||||
KeepQueryOrder=${ztest:keepqueryorder:0:1}
|
||||
StripQuery=${stripquery}
|
||||
StoreAllInCache=${ztest:cache2:0:1}
|
||||
Warc=${ztest:warc:0:1}
|
||||
WarcFile=${warcfile}
|
||||
LogType=${logtype}
|
||||
UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
|
||||
ProxyType=${proxytype}
|
||||
|
||||
8
lang.def
8
lang.def
@@ -1034,11 +1034,3 @@ LANG_STRIPQUERY
|
||||
Strip query keys:
|
||||
LANG_STRIPQUERYTIP
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
LANG_WARC
|
||||
Write a WARC archive of the crawl
|
||||
LANG_WARCTIP
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
LANG_WARCFILE
|
||||
WARC archive name:
|
||||
LANG_WARCFILETIP
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Ïðåìàõâàíå íà êëþ÷îâå îò çàÿâêàòà:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Êëþ÷îâå îò çàÿâêàòà, ðàçäåëåíè ñúñ çàïåòàÿ, êîèòî äà ñå ïðåìàõíàò îò èìåòî íà çàïèñàíèÿ ôàéë (íàïðèìåð sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Çàïèñâàíå íà WARC àðõèâ íà îáõîæäàíåòî
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Çàïèñâàíå íà âñåêè èçòåãëåí îòãîâîð è â WARC/1.1 àðõèâ ïî ISO-28500, äî îãëåäàëîòî.
|
||||
WARC archive name:
|
||||
Èìå íà WARC àðõèâà:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Íåçàäúëæèòåëíî áàçîâî èìå çà WARC àðõèâà; îñòàâåòå ïðàçíî çà àâòîìàòè÷íî èìåíóâàíå â èçõîäíàòà äèðåêòîðèÿ.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Eliminar claves de query string:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Claves de query string, separadas por comas, que se eliminarán del nombre de los archivos guardados (p. ej. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Escribir un archivo WARC del rastreo
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Guardar también cada respuesta descargada en un archivo WARC/1.1 ISO-28500, junto a la réplica.
|
||||
WARC archive name:
|
||||
Nombre del archivo WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nombre base opcional para el archivo WARC; déjelo en blanco para nombrarlo automáticamente en el directorio de salida.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Odebrat klíèe dotazu:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Klíèe dotazu oddìlené èárkami, které se vynechají z pojmenování ukládaných souborù (napø. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Zapsat archiv WARC z procházení
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Uložit také každou staženou odpovìï do archivu WARC/1.1 podle ISO-28500 vedle zrcadla.
|
||||
WARC archive name:
|
||||
Název archivu WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Volitelný základní název archivu WARC; ponechte prázdné pro automatické pojmenování ve výstupním adresáøi.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
移除查詢鍵:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
以逗號分隔的查詢鍵，將其從儲存檔案的命名中移除 (例如 sid,utm_source)。
|
||||
Write a WARC archive of the crawl
|
||||
寫入此次抓取的 WARC 封存檔
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
同時將每個已擷取的回應儲存為 ISO-28500 WARC/1.1 封存檔，置於鏡像網站旁。
|
||||
WARC archive name:
|
||||
WARC 封存檔名稱：
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC 封存檔的選用基本名稱；留空則於輸出目錄中自動命名。
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
剥离查询键:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
用逗号分隔的查询键，将其从保存文件的命名中删除 (例如 sid,utm_source)。
|
||||
Write a WARC archive of the crawl
|
||||
写入本次抓取的 WARC 归档
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
同时将每个已获取的响应保存为 ISO-28500 WARC/1.1 归档，置于镜像站点旁边。
|
||||
WARC archive name:
|
||||
WARC 归档名称：
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC 归档的可选基本名称；留空则在输出目录中自动命名。
|
||||
|
||||
@@ -958,11 +958,3 @@ Strip query keys:
|
||||
Ukloniti kljuèeve upita:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Zarezom odvojeni kljuèevi upita koji se izostavljaju iz naziva spremljenih datoteka (npr. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Zapi¹i WARC arhivu obilaska
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Spremi i svaki preuzeti odgovor u ISO-28500 WARC/1.1 arhivu, uz zrcalo.
|
||||
WARC archive name:
|
||||
Naziv WARC arhive:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Neobavezni osnovni naziv WARC arhive; ostavite prazno za automatsko imenovanje u izlaznom direktoriju.
|
||||
|
||||
@@ -1004,11 +1004,3 @@ Strip query keys:
|
||||
Fjern forespørgselsnøgler:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Kommaseparerede forespørgselsnøgler, der udelades i navngivningen af gemte filer (f.eks. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Skriv et WARC-arkiv af gennemsøgningen
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Gem også hvert hentet svar i et ISO-28500 WARC/1.1-arkiv ved siden af spejlet.
|
||||
WARC archive name:
|
||||
Navn på WARC-arkiv:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valgfrit basisnavn til WARC-arkivet; lad feltet stå tomt for automatisk navngivning i outputmappen.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Query-Schlüssel entfernen:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Kommagetrennte Query-Schlüssel, die bei der Benennung gespeicherter Dateien entfallen (z. B. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
WARC-Archiv des Crawls schreiben
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Jede heruntergeladene Antwort zusätzlich in einem ISO-28500-WARC/1.1-Archiv neben dem Spiegel speichern.
|
||||
WARC archive name:
|
||||
Name des WARC-Archivs:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optionaler Basisname für das WARC-Archiv; leer lassen, um es automatisch im Ausgabeverzeichnis zu benennen.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Eemalda päringuvõtmed:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Komadega eraldatud päringuvõtmed, mis jäetakse salvestatud faili nimest välja (nt sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Kirjuta läbimise WARC-arhiiv
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Salvesta iga alla laaditud vastus ka ISO-28500 WARC/1.1 arhiivi peegli kõrvale.
|
||||
WARC archive name:
|
||||
WARC-arhiivi nimi:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC-arhiivi valikuline põhinimi; jäta tühjaks, et see väljundkataloogis automaatselt nimetada.
|
||||
|
||||
@@ -1004,11 +1004,3 @@ Strip query keys:
|
||||
Strip query keys:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Write a WARC archive of the crawl
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
WARC archive name:
|
||||
WARC archive name:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
|
||||
@@ -958,11 +958,3 @@ Strip query keys:
|
||||
Poista kyselyavaimet:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Pilkuin erotellut kyselyavaimet, jotka jätetään pois tallennettujen tiedostojen nimeämisestä (esim. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Kirjoita imuroinnin WARC-arkisto
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Tallenna myös jokainen noudettu vastaus ISO-28500 WARC/1.1 -arkistoon peilin viereen.
|
||||
WARC archive name:
|
||||
WARC-arkiston nimi:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valinnainen WARC-arkiston perusnimi; jätä tyhjäksi, jotta se nimetään automaattisesti tulostehakemistoon.
|
||||
|
||||
@@ -1004,11 +1004,3 @@ Strip query keys:
|
||||
Supprimer les clés de query string :
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Clés de query string à retirer du nommage des fichiers enregistrés, séparées par des virgules (par ex. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Écrire une archive WARC du crawl
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Enregistrer aussi chaque réponse téléchargée dans une archive WARC/1.1 (ISO-28500), à côté du miroir.
|
||||
WARC archive name:
|
||||
Nom de l'archive WARC :
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nom de base optionnel pour l'archive WARC ; laissez vide pour le générer automatiquement dans le répertoire de sortie.
|
||||
|
||||
@@ -958,11 +958,3 @@ Strip query keys:
|
||||
Αφαίρεση κλειδιών ερωτήματος:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Κλειδιά ερωτήματος χωρισμένα με κόμμα, που θα αφαιρεθούν από την ονομασία των αποθηκευμένων αρχείων (π.χ. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Εγγραφή αρχείου WARC της ανίχνευσης
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Αποθήκευση κάθε ληφθείσας απόκρισης και σε αρχείο WARC/1.1 ISO-28500, δίπλα στο είδωλο.
|
||||
WARC archive name:
|
||||
Όνομα αρχείου WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Προαιρετικό βασικό όνομα για το αρχείο WARC. Αφήστε το κενό για αυτόματη ονομασία στον κατάλογο εξόδου.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Rimuovi chiavi della query string:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Chiavi della query string, separate da virgole, da rimuovere dai nomi dei file salvati (ad es. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Scrivi un archivio WARC della scansione
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Salva anche ogni risposta scaricata in un archivio WARC/1.1 ISO-28500, accanto al mirror.
|
||||
WARC archive name:
|
||||
Nome dell'archivio WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nome di base facoltativo per l'archivio WARC; lascia vuoto per assegnarlo automaticamente nella directory di output.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
削除するクエリキー:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
保存ファイル名の生成から除外するクエリキーをカンマ区切りで指定します (例: sid,utm_source)。
|
||||
Write a WARC archive of the crawl
|
||||
クロールの WARC アーカイブを書き出す
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
取得した各レスポンスを ISO-28500 WARC/1.1 アーカイブとしてミラーの隣にも保存します。
|
||||
WARC archive name:
|
||||
WARC アーカイブ名:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC アーカイブの任意のベース名。空欄にすると出力ディレクトリ内で自動的に名前が付けられます。
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
¾âáâàÐÝØ ÚÛãçÕÒØ ÞÔ ÑÐàÐúÕâÞ:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
ºÛãçÕÒØ ÞÔ ÑÐàÐúÕâÞ, ÞÔÔÕÛÕÝØ áÞ ×ÐßØàÚÐ, èâÞ áÕ ÞâáâàÐÝãÒÐÐâ ÞÔ ØÜÕâÞ ÝÐ ×ÐçãÒÐÝÐâÐ ÔÐâÞâÕÚÐ (ÝÐ ßàØÜÕà sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
·ÐßØèØ WARC ÐàåØÒÐ ÝÐ ßàÕÑÐàãÒÐúÕâÞ
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
·ÐçãÒÐø ÓÞ áÕÚÞø ßàÕ×ÕÜÕÝ ÞÔÓÞÒÞà Ø ÒÞ ISO-28500 WARC/1.1 ÐàåØÒÐ, ßÞÚàÐø ÞÓÛÕÔÐÛÞâÞ.
|
||||
WARC archive name:
|
||||
¸ÜÕ ÝÐ WARC ÐàåØÒÐâÐ:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
¸×ÑÞàÝÞ ÞáÝÞÒÝÞ ØÜÕ ×Ð WARC ÐàåØÒÐâÐ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð ÐÒâÞÜÐâáÚÞ ØÜÕÝãÒÐúÕ ÒÞ Ø×ÛÕ×ÝØÞâ ÔØàÕÚâÞàØãÜ.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Lekérdezési kulcsok eltávolítása:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Vesszõvel elválasztott lekérdezési kulcsok, amelyeket el kell hagyni a mentett fájl elnevezésébõl (pl. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
A bejárás WARC archívumának írása
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Minden letöltött válasz mentése ISO-28500 WARC/1.1 archívumba is, a tükör mellé.
|
||||
WARC archive name:
|
||||
WARC archívum neve:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
A WARC archívum opcionális alapneve; hagyja üresen az automatikus elnevezéshez a kimeneti könyvtárban.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Query-sleutels verwijderen:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Door komma's gescheiden query-sleutels die bij het benoemen van opgeslagen bestanden worden weggelaten (bijv. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Schrijf een WARC-archief van de crawl
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Sla ook elke opgehaalde respons op in een ISO-28500 WARC/1.1-archief, naast de mirror.
|
||||
WARC archive name:
|
||||
Naam van WARC-archief:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Optionele basisnaam voor het WARC-archief; laat leeg om het automatisch een naam te geven in de uitvoermap.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Fjern spørrenøkler:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Kommaseparerte spørrenøkler som utelates i navngivingen av lagrede filer (f.eks. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Skriv et WARC-arkiv av gjennomgangen
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Lagre også hvert nedlastet svar i et ISO-28500 WARC/1.1-arkiv ved siden av speilet.
|
||||
WARC archive name:
|
||||
Navn på WARC-arkiv:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valgfritt basisnavn for WARC-arkivet; la feltet stå tomt for automatisk navngivning i utdatamappen.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Usuñ klucze zapytania:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Rozdzielone przecinkami klucze zapytania pomijane przy nazywaniu zapisanych plików (np. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Zapisz archiwum WARC z indeksowania
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Zapisz te¿ ka¿d± pobran± odpowied¼ do archiwum WARC/1.1 ISO-28500, obok kopii lustrzanej.
|
||||
WARC archive name:
|
||||
Nazwa archiwum WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Opcjonalna nazwa bazowa archiwum WARC; pozostaw puste, aby nazwaæ je automatycznie w katalogu wyj¶ciowym.
|
||||
|
||||
@@ -1004,11 +1004,3 @@ Strip query keys:
|
||||
Remover chaves da query string:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Chaves da query string, separadas por vírgulas, a serem removidas da nomeação dos arquivos salvos (ex.: sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Gravar um arquivo WARC do rastreamento
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Salvar também cada resposta baixada em um arquivo WARC/1.1 ISO-28500, ao lado do espelho.
|
||||
WARC archive name:
|
||||
Nome do arquivo WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nome base opcional para o arquivo WARC; deixe em branco para nomeá-lo automaticamente no diretório de saída.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Remover chaves da query string:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Chaves da query string, separadas por vírgulas, a remover da nomeação dos ficheiros guardados (por ex. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Escrever um arquivo WARC do rastreio
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Guardar também cada resposta transferida num arquivo WARC/1.1 ISO-28500, ao lado do espelho.
|
||||
WARC archive name:
|
||||
Nome do arquivo WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nome base opcional para o arquivo WARC; deixe em branco para o nomear automaticamente no diretório de saída.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Elimina cheile din query string:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Chei din query string, separate prin virgula, de eliminat din denumirea fisierelor salvate (de ex. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Scrie o arhiva WARC a parcurgerii
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Salveaza si fiecare raspuns descarcat intr-o arhiva WARC/1.1 ISO-28500, langa oglinda.
|
||||
WARC archive name:
|
||||
Numele arhivei WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Nume de baza optional pentru arhiva WARC; lasati gol pentru a-l denumi automat in directorul de iesire.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Óäàëÿòü êëþ÷è çàïðîñà:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Êëþ÷è çàïðîñà ÷åðåç çàïÿòóþ, óäàëÿåìûå èç èìåíè ñîõðàíÿåìîãî ôàéëà (íàïðèìåð, sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Çàïèñàòü WARC-àðõèâ îáõîäà
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Ñîõðàíÿòü êàæäûé çàãðóæåííûé îòâåò òàêæå â àðõèâ WARC/1.1 ISO-28500 ðÿäîì ñ çåðêàëîì.
|
||||
WARC archive name:
|
||||
Èìÿ WARC-àðõèâà:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Íåîáÿçàòåëüíîå áàçîâîå èìÿ WARC-àðõèâà; îñòàâüòå ïóñòûì äëÿ àâòîìàòè÷åñêîãî èìåíîâàíèÿ â âûõîäíîì êàòàëîãå.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Odstráni» kµúèe dotazu:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Kµúèe dotazu oddelené èiarkami, ktoré sa vynechajú z pomenovania ukladaných súborov (napr. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Zapísa» archív WARC z prehµadávania
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Ulo¾i» aj ka¾dú stiahnutú odpoveï do archívu WARC/1.1 ISO-28500 vedµa zrkadla.
|
||||
WARC archive name:
|
||||
Názov archívu WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Voliteµný základný názov archívu WARC; ponechajte prázdne pre automatické pomenovanie vo výstupnom adresári.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Odstrani kljuce poizvedbe:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Z vejicami loceni kljuci poizvedbe, ki se izpustijo pri poimenovanju shranjenih datotek (npr. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Zapisi arhiv WARC iz pregledovanja
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Shrani tudi vsak preneseni odgovor v arhiv WARC/1.1 ISO-28500 poleg zrcala.
|
||||
WARC archive name:
|
||||
Ime arhiva WARC:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Neobvezno osnovno ime arhiva WARC; pustite prazno za samodejno poimenovanje v izhodni mapi.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Ta bort frågenycklar:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Kommaseparerade frågenycklar som utelämnas vid namngivningen av sparade filer (t.ex. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Skriv ett WARC-arkiv av genomsökningen
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Spara även varje hämtat svar i ett ISO-28500 WARC/1.1-arkiv, bredvid spegeln.
|
||||
WARC archive name:
|
||||
WARC-arkivets namn:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Valfritt basnamn för WARC-arkivet; lämna tomt för att namnge det automatiskt i utdatakatalogen.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Sorgu anahtarlarýný çýkar:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Kaydedilen dosya adlandýrmasýndan çýkarýlacak, virgülle ayrýlmýþ sorgu anahtarlarý (örn. sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Taramanýn WARC arþivini yaz
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Ýndirilen her yanýtý ayrýca aynanýn yanýna bir ISO-28500 WARC/1.1 arþivine kaydet.
|
||||
WARC archive name:
|
||||
WARC arþivi adý:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC arþivi için isteðe baðlý temel ad; çýktý dizininde otomatik adlandýrma için boþ býrakýn.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Âèëó÷àòè êëþ÷³ çàïèòó:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Êëþ÷³ çàïèòó ÷åðåç êîìó, ÿê³ âèëó÷àþòüñÿ ç ³ìåí³ çáåðåæåíîãî ôàéëó (íàïðèêëàä, sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Çàïèñàòè WARC-àðõ³â îáõîäó
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Çáåð³ãàòè êîæíó çàâàíòàæåíó â³äïîâ³äü òàêîæ ó àðõ³â WARC/1.1 ISO-28500 ïîðÿä ³ç äçåðêàëîì.
|
||||
WARC archive name:
|
||||
²ì'ÿ WARC-àðõ³âó:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
Íåîáîâ'ÿçêîâà áàçîâà íàçâà WARC-àðõ³âó; çàëèøòå ïîðîæí³ì äëÿ àâòîìàòè÷íîãî íàéìåíóâàííÿ ó âèõ³äíîìó êàòàëîç³.
|
||||
|
||||
@@ -956,11 +956,3 @@ Strip query keys:
|
||||
Olib tashlanadigan so’rov kalitlari:
|
||||
Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
|
||||
Saqlangan fayl nomidan olib tashlanadigan, vergul bilan ajratilgan so’rov kalitlari (masalan, sid,utm_source).
|
||||
Write a WARC archive of the crawl
|
||||
Qidiruvning WARC arxivini yozish
|
||||
Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
|
||||
Har bir yuklab olingan javobni ISO-28500 WARC/1.1 arxiviga ham, ko'zgu yonida saqlash.
|
||||
WARC archive name:
|
||||
WARC arxivi nomi:
|
||||
Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
|
||||
WARC arxivi uchun ixtiyoriy asosiy nom; chiqish katalogida avtomatik nomlash uchun bo'sh qoldiring.
|
||||
|
||||
@@ -40,7 +40,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%N, \-\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
|
||||
[ \fB\-%M, \-\-mime\-html\fR ]
|
||||
[ \fB\-%r, \-\-warc\fR ]
|
||||
[ \fB\-LN, \-\-long\-names[=N]\fR ]
|
||||
[ \fB\-KN, \-\-keep\-links[=N]\fR ]
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
@@ -59,7 +58,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-sN, \-\-robots[=N]\fR ]
|
||||
[ \fB\-%h, \-\-http\-10\fR ]
|
||||
[ \fB\-%k, \-\-keep\-alive\fR ]
|
||||
[ \fB\-%z, \-\-disable\-compression\fR ]
|
||||
[ \fB\-%B, \-\-tolerant\fR ]
|
||||
[ \fB\-%s, \-\-updatehack\fR ]
|
||||
[ \fB\-%u, \-\-urlhack\fR ]
|
||||
@@ -100,7 +98,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-%!, \-\-disable\-security\-limits\fR ]
|
||||
[ \fB\-V, \-\-userdef\-cmd\fR ]
|
||||
[ \fB\-%W, \-\-callback\fR ]
|
||||
[ \fB\-y, \-\-background\-on\-suspend\fR ]
|
||||
[ \fB\-K, \-\-keep\-links[=N]\fR ]
|
||||
.SH DESCRIPTION
|
||||
.B httrack
|
||||
@@ -198,10 +195,6 @@ delayed type check, don't make any link test but wait for files download to star
|
||||
cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
|
||||
.IP \-%M
|
||||
generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
|
||||
.IP \-%r
|
||||
write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name, \-\-warc\-max\-size N rotates segments past N bytes (\-\-warc)
|
||||
.IP \-%t
|
||||
keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
|
||||
.IP \-LN
|
||||
long names (L1 *long names / L0 8\-3 conversion / L2 ISO9660 compatible) (\-\-long\-names[=N])
|
||||
.IP \-KN
|
||||
@@ -239,8 +232,6 @@ follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always
|
||||
force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (\-\-http\-10)
|
||||
.IP \-%k
|
||||
use keep\-alive if possible, greately reducing latency for small files and test requests (%k0 don't use) (\-\-keep\-alive)
|
||||
.IP \-%z
|
||||
do not request compressed content (%z0 request) (\-\-disable\-compression)
|
||||
.IP \-%B
|
||||
tolerant requests (accept bogus responses on some servers, but not standard!) (\-\-tolerant)
|
||||
.IP \-%s
|
||||
@@ -267,7 +258,7 @@ default referer field sent in HTTP headers (\-\-referer <param>)
|
||||
.IP \-%E
|
||||
from email address sent in HTTP headers (\-\-from <param>)
|
||||
.IP \-%F
|
||||
footer string in Html code (\-%F "Mirrored from {url} on {date}"; fields {addr} {path} {url} {date} {lastmodified} {version} {mime} {charset} {status} {size}, or legacy %s) (\-\-footer <param>)
|
||||
footer string in Html code (\-%F "Mirrored [from host %s [file %s [at %s]]]" (\-\-footer <param>)
|
||||
.IP \-%l
|
||||
preferred language (\-%l "fr, en, jp, *" (\-\-language <param>)
|
||||
.IP \-%a
|
||||
@@ -335,6 +326,8 @@ go everywhere on the web (\-\-go\-everywhere)
|
||||
.IP \-%H
|
||||
debug HTTP headers in logfile (\-\-debug\-headers)
|
||||
.SS Guru options: (do NOT use if possible)
|
||||
.IP \-#X
|
||||
*use optimized engine (limited memory boundary checks) (\-\-fast\-engine)
|
||||
.IP \-#test
|
||||
list engine self\-tests (run one with \-#test=NAME [args])
|
||||
.IP \-#C
|
||||
@@ -359,6 +352,8 @@ maximum number of links (\-#L1000000) (\-\-advanced\-maxlinks[=N])
|
||||
display ugly progress information (\-\-advanced\-progressinfo)
|
||||
.IP \-#P
|
||||
catch URL (\-\-catch\-url)
|
||||
.IP \-#R
|
||||
old FTP routines (debug) (\-\-repair\-cache)
|
||||
.IP \-#T
|
||||
generate transfer ops. log every minutes (\-\-debug\-xfrstats)
|
||||
.IP \-#u
|
||||
@@ -377,8 +372,6 @@ USE IT WITH EXTREME CARE
|
||||
execute system command after each files ($0 is the filename: \-V "rm \\$0") (\-\-userdef\-cmd <param>)
|
||||
.IP \-%W
|
||||
use an external library function as a wrapper (\-%W myfoo.so[,myparameters]) (\-\-callback <param>)
|
||||
.IP \-y
|
||||
go to background when suspended (y0 don't) (\-\-background\-on\-suspend)
|
||||
.SS Details: Option N
|
||||
.IP \-N0
|
||||
Site\-structure (default)
|
||||
|
||||
@@ -63,7 +63,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.c htslib.c htsurlport.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htswarc.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
@@ -74,7 +74,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htsproxy.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
|
||||
@@ -114,10 +114,6 @@ const char *hts_optalias[][4] = {
|
||||
"strip [host/pattern=]key1,key2,... from URLs"},
|
||||
{"cookies-file", "-%K", "param1",
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
|
||||
{"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
|
||||
{"warc-max-size", "-%rs", "param1",
|
||||
"rotate the WARC archive once a segment passes N bytes (0: single file)"},
|
||||
{"why", "-%Y", "param1",
|
||||
"explain which filter rule accepts or rejects a URL, then exit"},
|
||||
{"pause", "-%G", "param1",
|
||||
|
||||
@@ -37,7 +37,6 @@ Please visit our Website: http://www.httrack.com
|
||||
/* specific definitions */
|
||||
#include "htsnet.h"
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
#include "htsthread.h"
|
||||
#include <time.h>
|
||||
/* END specific definitions */
|
||||
@@ -980,10 +979,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
// status finished callback
|
||||
RUN_CALLBACK1(opt, xfrstatus, &back[p]);
|
||||
|
||||
// WARC archive of the transaction (request + response/revisit)
|
||||
if (StringNotEmpty(opt->warc_file))
|
||||
warc_write_backtransaction(opt, &back[p]);
|
||||
|
||||
return 0;
|
||||
} else { // testmode
|
||||
if (back[p].r.statuscode / 100 >= 3) { /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */
|
||||
@@ -1060,9 +1055,6 @@ void back_copy_static(const lien_back * src, lien_back * dst) {
|
||||
dst->r.soc = INVALID_SOCKET;
|
||||
dst->r.adr = NULL;
|
||||
dst->r.headers = NULL;
|
||||
dst->r.warc_reqhdr = NULL;
|
||||
dst->r.warc_resphdr = NULL;
|
||||
dst->r.warc_truncated = 0;
|
||||
dst->r.out = NULL;
|
||||
dst->r.location = dst->location_buffer;
|
||||
dst->r.fp = NULL;
|
||||
@@ -1126,9 +1118,6 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
|
||||
(*dst)->chunk_adr = NULL;
|
||||
(*dst)->r.adr = NULL;
|
||||
(*dst)->r.out = NULL;
|
||||
(*dst)->r.warc_reqhdr = NULL;
|
||||
(*dst)->r.warc_resphdr = NULL;
|
||||
(*dst)->r.warc_truncated = 0;
|
||||
(*dst)->r.location = (*dst)->location_buffer;
|
||||
(*dst)->r.fp = NULL;
|
||||
(*dst)->r.soc = INVALID_SOCKET;
|
||||
@@ -1596,7 +1585,6 @@ int back_clear_entry(lien_back * back) {
|
||||
freet(back->r.headers);
|
||||
back->r.headers = NULL;
|
||||
}
|
||||
warc_free_request(&back->r);
|
||||
// Tout nettoyer
|
||||
memset(back, 0, sizeof(lien_back));
|
||||
back->r.soc = INVALID_SOCKET;
|
||||
@@ -2521,19 +2509,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
for (i = 0; i < (unsigned int) back_max; i++) {
|
||||
if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
|
||||
/* A cap-truncated body is deliberate, not broken: archive what arrived
|
||||
with WARC-Truncated before the abort overwrites the slot's real 2xx
|
||||
status. HTTrack still treats the slot as incomplete afterwards. */
|
||||
if (StringNotEmpty(opt->warc_file) && back[i].r.statuscode > 0 &&
|
||||
back[i].r.warc_resphdr != NULL && back[i].r.size > 0 &&
|
||||
!(back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))) {
|
||||
if (back[i].r.is_write && back[i].r.out != NULL)
|
||||
fflush(back[i].r.out);
|
||||
back[i].r.warc_truncated = (limit == HTS_MIRROR_LIMIT_SIZE)
|
||||
? WARC_TRUNC_LENGTH
|
||||
: WARC_TRUNC_TIME;
|
||||
warc_write_backtransaction(opt, &back[i]);
|
||||
}
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
@@ -3656,10 +3631,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
deleteaddr(&back[i].r);
|
||||
back[i].r.headers = block;
|
||||
}
|
||||
// Stash the raw response headers for WARC (deletehttp frees
|
||||
// r.headers when the socket closes, before back_finalize)
|
||||
if (StringNotEmpty(opt->warc_file))
|
||||
warc_stash_response(&back[i].r, back[i].r.headers);
|
||||
|
||||
/*
|
||||
Status code and header-response hacks
|
||||
|
||||
@@ -39,7 +39,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
/* File defs */
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -2246,7 +2245,6 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
|
||||
// ending
|
||||
usercommand(opt, 0, NULL, NULL, NULL, NULL);
|
||||
warc_close_opt(opt);
|
||||
|
||||
// désallocation mémoire & buffers
|
||||
XH_uninit;
|
||||
@@ -3630,10 +3628,6 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
|
||||
if (StringNotEmpty(from->cookies_file))
|
||||
StringCopyS(to->cookies_file, from->cookies_file);
|
||||
|
||||
if (StringNotEmpty(from->warc_file))
|
||||
StringCopyS(to->warc_file, from->warc_file);
|
||||
to->warc_max_size = from->warc_max_size;
|
||||
|
||||
if (from->pause_max_ms > 0) {
|
||||
to->pause_min_ms = from->pause_min_ms;
|
||||
to->pause_max_ms = from->pause_max_ms;
|
||||
|
||||
@@ -40,7 +40,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htsdefines.h"
|
||||
#include "htsalias.h"
|
||||
#include "htswarc.h"
|
||||
#include "htsbauth.h"
|
||||
#include "htswrap.h"
|
||||
#include "htsmodules.h"
|
||||
@@ -1623,9 +1622,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF
|
||||
("Option %F needs to be followed by a blank space, and a footer string");
|
||||
printf("Example: -%%F \"<!-- Mirrored from {addr}{path} by "
|
||||
"HTTrack Website Copier/"
|
||||
"{version} " HTTRACK_AFF_AUTHORS ", {date} -->\"\n");
|
||||
printf
|
||||
("Example: -%%F \"<!-- Mirrored from %%s by HTTrack Website Copier/"
|
||||
HTTRACK_AFF_VERSION " " HTTRACK_AFF_AUTHORS
|
||||
", %%s -->\"\n");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
@@ -1790,45 +1790,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
StringCopy(opt->cookies_file, argv[na]);
|
||||
}
|
||||
break;
|
||||
case 'r': // warc / warc-file: write an ISO-28500 WARC archive
|
||||
if (*(com + 1) == 'f') { // --warc-file NAME: explicit basename
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option warc-file needs a blank space and a WARC name");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
if (strlen(argv[na]) >= 1024) {
|
||||
HTS_PANIC_PRINTF("WARC file name too long");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
StringCopy(opt->warc_file, argv[na]);
|
||||
} else if (*(com + 1) == 's') { // --warc-max-size N: rotation
|
||||
com++;
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF(
|
||||
"Option warc-max-size needs a blank space and a size");
|
||||
htsmain_free();
|
||||
return -1;
|
||||
}
|
||||
na++;
|
||||
{ // reject non-numeric/negative/overflow; keep default 0
|
||||
// (single file)
|
||||
char *end;
|
||||
LLint v;
|
||||
errno = 0;
|
||||
v = strtoll(argv[na], &end, 10);
|
||||
if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
|
||||
errno != ERANGE)
|
||||
opt->warc_max_size = v;
|
||||
}
|
||||
} else { // --warc: auto-named archive under the output dir
|
||||
StringCopy(opt->warc_file, WARC_AUTONAME);
|
||||
}
|
||||
break;
|
||||
case 'Y': // why: explain the filter verdict for a URL, no crawl
|
||||
if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
|
||||
HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
|
||||
|
||||
@@ -524,10 +524,6 @@ void help(const char *app, int more) {
|
||||
infomsg
|
||||
(" %D cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
|
||||
infomsg(" %M generate a RFC MIME-encapsulated full-archive (.mht)");
|
||||
infomsg(" %r write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
|
||||
"output name, --warc-max-size N rotates segments past N bytes");
|
||||
infomsg(" %t keep the original file extension, don't rewrite it from the "
|
||||
"MIME type (%t0 rewrite)");
|
||||
infomsg
|
||||
(" LN long names (L1 *long names / L0 8-3 conversion / L2 ISO9660 compatible)");
|
||||
infomsg
|
||||
@@ -558,7 +554,6 @@ void help(const char *app, int more) {
|
||||
(" %h force HTTP/1.0 requests (reduce update features, only for old servers or proxies)");
|
||||
infomsg
|
||||
(" %k use keep-alive if possible, greately reducing latency for small files and test requests (%k0 don't use)");
|
||||
infomsg(" %z do not request compressed content (%z0 request)");
|
||||
infomsg
|
||||
(" %B tolerant requests (accept bogus responses on some servers, but not standard!)");
|
||||
infomsg
|
||||
@@ -583,10 +578,8 @@ void help(const char *app, int more) {
|
||||
(" F user-agent field sent in HTTP headers (-F \"user-agent name\")");
|
||||
infomsg(" %R default referer field sent in HTTP headers");
|
||||
infomsg(" %E from email address sent in HTTP headers");
|
||||
infomsg(
|
||||
" %F footer string in Html code (-%F \"Mirrored from {url} on "
|
||||
"{date}\"; fields {addr} {path} {url} {date} {lastmodified} {version} "
|
||||
"{mime} {charset} {status} {size}, or legacy %s)");
|
||||
infomsg
|
||||
(" %F footer string in Html code (-%F \"Mirrored [from host %s [file %s [at %s]]]\"");
|
||||
infomsg(" %l preferred language (-%l \"fr, en, jp, *\"");
|
||||
infomsg(" %a accepted formats (-%a \"text/html,image/png;q=0.9,*/*;q=0.1\"");
|
||||
infomsg(" %X additional HTTP header line (-%X \"X-Magic: 42\"");
|
||||
@@ -627,6 +620,7 @@ void help(const char *app, int more) {
|
||||
infomsg(" %H debug HTTP headers in logfile");
|
||||
infomsg("");
|
||||
infomsg("Guru options: (do NOT use if possible)");
|
||||
infomsg(" #X *use optimized engine (limited memory boundary checks)");
|
||||
infomsg(" #test list engine self-tests (run one with -#test=NAME [args])");
|
||||
infomsg(" #C cache list (-#C '*.com/spider*.gif'");
|
||||
infomsg(" #R cache repair (damaged cache)");
|
||||
@@ -639,6 +633,7 @@ void help(const char *app, int more) {
|
||||
infomsg(" #L maximum number of links (-#L1000000)");
|
||||
infomsg(" #p display ugly progress information");
|
||||
infomsg(" #P catch URL");
|
||||
infomsg(" #R old FTP routines (debug)");
|
||||
infomsg(" #T generate transfer ops. log every minutes");
|
||||
infomsg(" #u wait time");
|
||||
infomsg(" #Z generate transfer rate statistics every minutes");
|
||||
@@ -655,7 +650,6 @@ void help(const char *app, int more) {
|
||||
(" V execute system command after each files ($0 is the filename: -V \"rm \\$0\")");
|
||||
infomsg(" %W use an external library function as a wrapper (-%W "
|
||||
"myfoo.so[,myparameters])");
|
||||
infomsg(" y go to background when suspended (y0 don't)");
|
||||
infomsg("");
|
||||
infomsg("Details: Option N");
|
||||
infomsg(" N0 Site-structure (default)");
|
||||
|
||||
11
src/htslib.c
11
src/htslib.c
@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
|
||||
// Fichier librairie .c
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htswarc.h"
|
||||
|
||||
/* specific definitions */
|
||||
#include "htsbase.h"
|
||||
@@ -1202,11 +1201,6 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
} // Fin test pas postfile
|
||||
//
|
||||
|
||||
// Stash the raw request for the WARC request record (freed at
|
||||
// back_clear_entry)
|
||||
if (StringNotEmpty(opt->warc_file))
|
||||
warc_stash_request(retour, bstr.buffer);
|
||||
|
||||
// Callback
|
||||
{
|
||||
int test_head =
|
||||
@@ -2120,8 +2114,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
|
||||
#if HTS_DEBUG_CLOSESOCK
|
||||
DEBUG_W("http_test: deletehttp\n");
|
||||
#endif
|
||||
// this probe's htsblk is discarded by callers, so free any WARC stash here
|
||||
warc_free_request(&retour);
|
||||
deletehttp(&retour);
|
||||
retour.soc = INVALID_SOCKET;
|
||||
}
|
||||
@@ -6018,8 +6010,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
|
||||
StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
|
||||
StringCopy(opt->strip_query, "");
|
||||
StringCopy(opt->cookies_file, "");
|
||||
StringCopy(opt->warc_file, "");
|
||||
opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
|
||||
StringCopy(opt->why_url, "");
|
||||
opt->pause_min_ms = 0;
|
||||
opt->pause_max_ms = 0;
|
||||
@@ -6171,7 +6161,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
|
||||
StringFree(opt->strip_query);
|
||||
StringFree(opt->cookies_file);
|
||||
StringFree(opt->why_url);
|
||||
StringFree(opt->warc_file);
|
||||
|
||||
StringFree(opt->path_html);
|
||||
StringFree(opt->path_html_utf8);
|
||||
|
||||
10
src/htsopt.h
10
src/htsopt.h
@@ -256,7 +256,6 @@ struct htsoptstate {
|
||||
unsigned int debug_state;
|
||||
unsigned int tmpnameid; /**< counter for temporary file names */
|
||||
int is_ended; /**< mirror has finished */
|
||||
void *warc; /**< open WARC writer (warc_writer*), or NULL */
|
||||
};
|
||||
|
||||
/* Library handles */
|
||||
@@ -539,10 +538,6 @@ struct httrackp {
|
||||
int pause_max_ms; /**< inter-file pause upper bound, ms */
|
||||
String why_url; /**< URL to diagnose (--why): print the deciding filter rule
|
||||
and exit without crawling */
|
||||
String warc_file; /**< WARC output: WARC_AUTONAME for --warc, or the
|
||||
--warc-file basename (appended at the tail: ABI) */
|
||||
LLint warc_max_size; /**< --warc-max-size: rotate the archive past this many
|
||||
bytes (<=0: single file). Tail: ABI */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
@@ -666,11 +661,6 @@ struct htsblk {
|
||||
/* Restart-whole signal: a resume this response rejected (unusable 206) must
|
||||
retry with no Range, else a surviving partial/temp-ref loops (#581). */
|
||||
hts_boolean refetch_wholefile;
|
||||
char *warc_reqhdr; /**< stashed raw request header block for WARC (or NULL) */
|
||||
char *
|
||||
warc_resphdr; /**< stashed raw response header block for WARC (or NULL) */
|
||||
int warc_truncated; /**< WARC-Truncated reason for a cap-truncated body
|
||||
(WARC_TRUNC_*, 0=none). Tail: ABI */
|
||||
/*char digest[32+2]; // md5 digest generated by the engine ("" if none) */
|
||||
};
|
||||
|
||||
|
||||
@@ -792,72 +792,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
|
||||
char gmttime[256];
|
||||
char BIGSTK safe_adr[HTS_URLMAXSIZE * 3 + 4];
|
||||
char BIGSTK safe_fil[HTS_URLMAXSIZE * 3 + 4];
|
||||
char BIGSTK safe_url[HTS_URLMAXSIZE * 6 + 16];
|
||||
char safe_lastmod[sizeof(r->lastmodified) * 3 + 4];
|
||||
char safe_ctype[sizeof(r->contenttype) * 3 + 4];
|
||||
char safe_charset[sizeof(r->charset) * 3 + 4];
|
||||
char status_str[16];
|
||||
char size_str[32];
|
||||
// {url} scheme: jump_identification_const strips it for
|
||||
// http/https/ftp, so re-add it (bare host is http); for any
|
||||
// other scheme the host keeps its "scheme://" and we add
|
||||
// none.
|
||||
const char *const url_host =
|
||||
jump_identification_const(urladr());
|
||||
const char *const url_scheme =
|
||||
strstr(url_host, "://") != NULL ? ""
|
||||
: strfield(urladr(), "https://") ? "https://"
|
||||
: strfield(urladr(), "ftp://") ? "ftp://"
|
||||
: "http://";
|
||||
|
||||
// {addr}/{path} are the escaped host and remote path; {url}
|
||||
// prepends the scheme to them (credentials already stripped
|
||||
// by jump_identification_const, and the scheme is a safe
|
||||
// literal, so no re-escaping is needed).
|
||||
html_inline_safe(url_host, safe_adr, sizeof(safe_adr));
|
||||
html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil));
|
||||
snprintf(safe_url, sizeof(safe_url), "%s%s%s", url_scheme,
|
||||
safe_adr, safe_fil);
|
||||
snprintf(status_str, sizeof(status_str), "%d", r->statuscode);
|
||||
snprintf(size_str, sizeof(size_str), LLintP, (LLint) r->size);
|
||||
tempo[0] = '\0';
|
||||
time_gmt_rfc822(gmttime);
|
||||
|
||||
{
|
||||
// Every network-derived string is html_inline_safe()'d: the
|
||||
// footer sits inside an HTML comment, so a value holding
|
||||
// "-->" would otherwise close it and inject markup (#165).
|
||||
// status/size are formatted integers and need no escaping.
|
||||
const hts_footer_field fields[] = {
|
||||
{"addr", safe_adr},
|
||||
{"path", safe_fil},
|
||||
{"url", safe_url},
|
||||
{"date", gmttime},
|
||||
{"lastmodified",
|
||||
html_inline_safe(r->lastmodified, safe_lastmod,
|
||||
sizeof(safe_lastmod))},
|
||||
{"version", HTTRACK_VERSIONID},
|
||||
{"mime", html_inline_safe(r->contenttype, safe_ctype,
|
||||
sizeof(safe_ctype))},
|
||||
{"charset", html_inline_safe(r->charset, safe_charset,
|
||||
sizeof(safe_charset))},
|
||||
{"status", status_str},
|
||||
{"size", size_str},
|
||||
};
|
||||
|
||||
tempo[0] = '\0';
|
||||
strcatbuff(tempo, eol);
|
||||
// hts_footer_format returns <0 on overflow, leaving tempo
|
||||
// unterminated; emitting it would abort in strcatbuff
|
||||
// below.
|
||||
if (hts_footer_format(tempo + strlen(tempo),
|
||||
sizeof(tempo) - strlen(tempo),
|
||||
StringBuff(opt->footer), fields,
|
||||
sizeof(fields) / sizeof(fields[0])) >=
|
||||
0) {
|
||||
strcatbuff(tempo, eol);
|
||||
HT_ADD(tempo);
|
||||
}
|
||||
}
|
||||
strcatbuff(tempo, eol);
|
||||
hts_template_format_str(tempo + strlen(tempo), sizeof(tempo) - strlen(tempo),
|
||||
StringBuff(opt->footer),
|
||||
html_inline_safe(jump_identification_const(urladr()), safe_adr, sizeof(safe_adr)),
|
||||
html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
|
||||
HTTRACK_VERSIONID, /* EOF */ NULL);
|
||||
strcatbuff(tempo, eol);
|
||||
HT_ADD(tempo);
|
||||
}
|
||||
// Emit charset ?
|
||||
if (emited_footer == 1 && strnotempty(r->charset)) {
|
||||
|
||||
@@ -57,7 +57,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htssniff.h"
|
||||
#include "htscodec.h"
|
||||
#include "htsproxy.h"
|
||||
#include "htswarc.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
@@ -972,47 +971,6 @@ static int st_entities(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// -#test=footerfmt <template>: expand a -%F footer with fixed fields (drives
|
||||
// tests/01_engine-footerfmt.test). Also asserts the overflow/zero-size returns
|
||||
// the CLI cap keeps out of reach.
|
||||
static int st_footerfmt(httrackp *opt, int argc, char **argv) {
|
||||
static const hts_footer_field fields[] = {
|
||||
{"addr", "host.example"},
|
||||
{"path", "/dir/page.html"},
|
||||
{"url", "http://host.example/dir/page.html"},
|
||||
{"date", "DATE"},
|
||||
{"lastmodified", "LASTMOD"},
|
||||
{"version", "VER"},
|
||||
{"mime", "text/html"},
|
||||
{"charset", "utf-8"},
|
||||
{"status", "200"},
|
||||
{"size", "1234"},
|
||||
};
|
||||
const size_t nfields = sizeof(fields) / sizeof(fields[0]);
|
||||
char out[1024];
|
||||
char tiny[4];
|
||||
|
||||
(void) opt;
|
||||
// Overflow (named and legacy) and a zero-size buffer must return <0, never
|
||||
// truncate silently or write out of bounds.
|
||||
assertf(hts_footer_format(tiny, sizeof(tiny), "{addr}", fields, nfields) < 0);
|
||||
assertf(hts_footer_format(tiny, sizeof(tiny), "a %s b", fields, nfields) < 0);
|
||||
assertf(hts_footer_format(out, 0, "", fields, nfields) < 0);
|
||||
// An empty template yields an empty, terminated string.
|
||||
assertf(hts_footer_format(out, sizeof(out), "", fields, nfields) == 1 &&
|
||||
out[0] == '\0');
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "footerfmt: needs a template\n");
|
||||
return 1;
|
||||
}
|
||||
if (hts_footer_format(out, sizeof(out), argv[0], fields, nfields) < 0) {
|
||||
fprintf(stderr, "footerfmt: overflow\n");
|
||||
return 1;
|
||||
}
|
||||
printf("%s\n", out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* The unescapers must reserve one byte for the trailing NUL: a 'max'-byte
|
||||
dest holding 'max' output chars pre-fix wrote dest[max] (1-byte OOB, caught
|
||||
by ASan). Both unescapeEntities and unescapeUrl share the guard. */
|
||||
@@ -1302,13 +1260,6 @@ static int st_copyopt(httrackp *opt, int argc, char **argv) {
|
||||
if (strcmp(StringBuff(to->cookies_file), "/tmp/jar.txt") != 0)
|
||||
err = 1;
|
||||
|
||||
/* warc_file: same String deep-copy path as cookies_file */
|
||||
StringCopy(from->warc_file, "run.warc.gz");
|
||||
StringCopy(to->warc_file, "");
|
||||
copy_htsopt(from, to);
|
||||
if (strcmp(StringBuff(to->warc_file), "run.warc.gz") != 0)
|
||||
err = 1;
|
||||
|
||||
/* #185 pause pair: copied when enabled (max>0), the 0 sentinel skips */
|
||||
from->pause_min_ms = 5000;
|
||||
from->pause_max_ms = 10000;
|
||||
@@ -3264,498 +3215,6 @@ static int st_ftpuser(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Bounded substring search (records carry NUL bytes; strstr won't do). */
|
||||
static const char *warc_memstr(const char *hay, const char *needle,
|
||||
size_t haylen, size_t nlen) {
|
||||
if (nlen == 0 || haylen < nlen)
|
||||
return NULL;
|
||||
{
|
||||
size_t i;
|
||||
for (i = 0; i + nlen <= haylen; i++) {
|
||||
if (memcmp(hay + i, needle, nlen) == 0)
|
||||
return hay + i;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Slurp a whole file into a malloc'd buffer; sets *len. NULL on error. */
|
||||
static unsigned char *warc_slurp(const char *path, size_t *len) {
|
||||
FILE *f = FOPEN(path, "rb");
|
||||
unsigned char *buf;
|
||||
long sz;
|
||||
if (f == NULL)
|
||||
return NULL;
|
||||
if (fseek(f, 0, SEEK_END) != 0 || (sz = ftell(f)) < 0) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
rewind(f);
|
||||
buf = malloct((size_t) sz + 1);
|
||||
if (buf == NULL) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
*len = fread(buf, 1, (size_t) sz, f);
|
||||
fclose(f);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Inflate one gzip member at *in (limit end); returns the decompressed record
|
||||
in a malloc'd buffer (*out_len), advancing *in past the member. NULL at end
|
||||
or on error (*out_len distinguishes: 0 and NULL = clean end). */
|
||||
static unsigned char *warc_next_member(const unsigned char **in,
|
||||
const unsigned char *end,
|
||||
size_t *out_len) {
|
||||
z_stream zs;
|
||||
unsigned char *out = NULL;
|
||||
size_t len = 0;
|
||||
int zerr;
|
||||
*out_len = 0;
|
||||
if (*in >= end)
|
||||
return NULL;
|
||||
memset(&zs, 0, sizeof(zs));
|
||||
if (inflateInit2(&zs, 15 + 32) != Z_OK)
|
||||
return NULL;
|
||||
zs.next_in = (const Bytef *) *in;
|
||||
zs.avail_in = (uInt) (end - *in);
|
||||
do {
|
||||
unsigned char tmp[8192];
|
||||
size_t got;
|
||||
zs.next_out = tmp;
|
||||
zs.avail_out = sizeof(tmp);
|
||||
zerr = inflate(&zs, Z_NO_FLUSH);
|
||||
if (zerr != Z_OK && zerr != Z_STREAM_END) {
|
||||
freet(out);
|
||||
inflateEnd(&zs);
|
||||
return NULL;
|
||||
}
|
||||
got = sizeof(tmp) - zs.avail_out;
|
||||
if (got > 0) {
|
||||
unsigned char *n = realloct(out, len + got + 1);
|
||||
if (n == NULL) {
|
||||
freet(out);
|
||||
inflateEnd(&zs);
|
||||
return NULL;
|
||||
}
|
||||
out = n;
|
||||
memcpy(out + len, tmp, got);
|
||||
len += got;
|
||||
}
|
||||
} while (zerr != Z_STREAM_END);
|
||||
*in = (const unsigned char *) zs.next_in; /* start of the next member */
|
||||
inflateEnd(&zs);
|
||||
if (out != NULL)
|
||||
out[len] = '\0';
|
||||
*out_len = len;
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Feed a synthetic transaction and validate the resulting .warc.gz against the
|
||||
WARC/1.1 spec: each record a self-standing gzip member starting WARC/1.,
|
||||
Content-Length == block length, the \r\n\r\n trailer intact, the response
|
||||
body round-trips, and the encoding headers are stripped (strategy B). */
|
||||
static int st_warc(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *end;
|
||||
int err = 0, nrec = 0, nresp = 0, nreq = 0, nrevisit = 0, ninfo = 0;
|
||||
int seen_a_body = 0, body_occurrences = 0, a2_bodyless = 0, nm_cl_ok = 0;
|
||||
static const char a_body[] = "Hello, WARC!\n";
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-selftest.warc.gz");
|
||||
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
|
||||
/* 200 HTML: bogus Content-Length + gzip/chunked encodings must be stripped.
|
||||
*/
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/a.html", "127.0.0.1",
|
||||
"GET /a.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
|
||||
"gzip\r\nTransfer-Encoding: chunked\r\nContent-Length: 999\r\n\r\n",
|
||||
a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
|
||||
|
||||
/* 302 redirect: header-only, no body. */
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/r", "127.0.0.1",
|
||||
"GET /r HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 302 Found\r\nLocation: http://test.local/a.html\r\n\r\n", NULL,
|
||||
0, NULL, 302, 0, 0);
|
||||
|
||||
/* 200 binary, chunked coding on the wire (already de-chunked here). */
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/b.bin", "127.0.0.1",
|
||||
"GET /b.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n"
|
||||
"Transfer-Encoding: chunked\r\n\r\n",
|
||||
"\x00\x01\x02\x03\x04", 5, NULL, 200, 0, 0);
|
||||
|
||||
/* 200 with a body shorter than the declared Content-Length (rewritten). */
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/trunc", "127.0.0.1",
|
||||
"GET /trunc HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: "
|
||||
"100\r\n\r\n",
|
||||
"short", 5, NULL, 200, 0, 0);
|
||||
|
||||
/* Same payload as a.html at a new URL: identical-payload-digest revisit
|
||||
(OpenSSL builds only; a plain build writes a second full response). */
|
||||
warc_write_transaction(w, "http://test.local/a2.html", "127.0.0.1",
|
||||
"GET /a2.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
|
||||
a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
|
||||
|
||||
/* 304 revisit with an EMPTY response-header block: the block is just the
|
||||
2-byte separator, so declared Content-Length must be exactly 2 (F3). */
|
||||
warc_write_transaction(w, "http://test.local/nm", "127.0.0.1",
|
||||
"GET /nm HTTP/1.1\r\nHost: test.local\r\n\r\n", "",
|
||||
NULL, 0, NULL, 304, 1, 0);
|
||||
|
||||
warc_close(w);
|
||||
|
||||
data = warc_slurp(path, &data_len);
|
||||
assertf(data != NULL);
|
||||
p = data;
|
||||
end = data + data_len;
|
||||
|
||||
while (p < end) {
|
||||
size_t rlen = 0;
|
||||
unsigned char *rec = warc_next_member(&p, end, &rlen);
|
||||
const char *sep, *cl;
|
||||
long long block_len = 0; /* 0 on a parse failure; err is already set */
|
||||
size_t hdr_len;
|
||||
if (rec == NULL) {
|
||||
if (rlen == 0)
|
||||
break; /* clean end */
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
nrec++;
|
||||
/* magic */
|
||||
if (rlen < 8 || memcmp(rec, "WARC/1.", 7) != 0)
|
||||
err = 1;
|
||||
/* record header ends at the first blank line */
|
||||
sep = warc_memstr((char *) rec, "\r\n\r\n", rlen, 4);
|
||||
if (sep == NULL) {
|
||||
err = 1;
|
||||
freet(rec);
|
||||
continue;
|
||||
}
|
||||
hdr_len = (size_t) ((const unsigned char *) sep - rec) + 4;
|
||||
/* Content-Length must equal the actual block length */
|
||||
cl = warc_memstr((char *) rec, "Content-Length:", hdr_len, 15);
|
||||
if (cl == NULL || sscanf(cl + 15, "%lld", &block_len) != 1)
|
||||
err = 1;
|
||||
else {
|
||||
if (hdr_len + (size_t) block_len + 4 != rlen)
|
||||
err = 1; /* header + block + trailing CRLFCRLF */
|
||||
else if (memcmp(rec + hdr_len + block_len, "\r\n\r\n", 4) != 0)
|
||||
err = 1; /* trailer intact */
|
||||
}
|
||||
if (warc_memstr((char *) rec, "WARC-Type: warcinfo", hdr_len, 19) != NULL)
|
||||
ninfo++;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: request", hdr_len, 18) != NULL)
|
||||
nreq++;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL)
|
||||
nresp++;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: revisit", hdr_len, 18) != NULL)
|
||||
nrevisit++;
|
||||
/* F1: the full body must appear exactly once across the whole file (a
|
||||
revisit must not re-embed it). */
|
||||
if (warc_memstr((char *) rec, a_body, rlen, sizeof(a_body) - 1) != NULL)
|
||||
body_occurrences++;
|
||||
/* F1: the a2.html identical-payload-digest revisit carries no body. */
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/a2.html",
|
||||
hdr_len, 42) != NULL &&
|
||||
warc_memstr((char *) rec, "WARC-Type: revisit", hdr_len, 18) != NULL)
|
||||
a2_bodyless =
|
||||
(warc_memstr((char *) rec, a_body, rlen, sizeof(a_body) - 1) == NULL);
|
||||
/* F3: the empty-header 304 revisit block is exactly the 2-byte separator
|
||||
(the request record shares this target URI, so match the revisit only).
|
||||
*/
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/nm",
|
||||
hdr_len, 37) != NULL &&
|
||||
warc_memstr((char *) rec, "WARC-Type: revisit", hdr_len, 18) != NULL)
|
||||
nm_cl_ok = (block_len == 2);
|
||||
/* a.html response body must round-trip and carry no encoding headers */
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/a.html",
|
||||
hdr_len, 41) != NULL &&
|
||||
warc_memstr((char *) rec, "msgtype=response", hdr_len, 16) != NULL) {
|
||||
const char *bsep = warc_memstr((char *) rec + hdr_len, "\r\n\r\n",
|
||||
(size_t) block_len, 4);
|
||||
if (bsep == NULL)
|
||||
err = 1;
|
||||
else {
|
||||
size_t bodyoff = (size_t) (bsep - (char *) rec) + 4;
|
||||
size_t got = rlen - 4 - bodyoff; /* minus record trailer */
|
||||
if (got != sizeof(a_body) - 1 ||
|
||||
memcmp(rec + bodyoff, a_body, got) != 0)
|
||||
err = 1;
|
||||
seen_a_body = 1;
|
||||
}
|
||||
if (warc_memstr((char *) rec, "Content-Encoding", hdr_len + block_len,
|
||||
16) != NULL ||
|
||||
warc_memstr((char *) rec, "Transfer-Encoding", hdr_len + block_len,
|
||||
17) != NULL)
|
||||
err = 1;
|
||||
}
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
|
||||
/* warcinfo + 6 transactions (response/revisit + request each) = 13 records.
|
||||
*/
|
||||
if (ninfo != 1 || nreq != 6 || nrec != 13 || !seen_a_body || !nm_cl_ok)
|
||||
err = 1;
|
||||
#if HTS_USEOPENSSL
|
||||
/* a.html + b.bin + trunc + 302 are full responses; a2.html deduped to a
|
||||
revisit (bodyless), nm is the 304 revisit; the body appears exactly once.
|
||||
*/
|
||||
if (nrevisit != 2 || nresp != 4 || !a2_bodyless || body_occurrences != 1)
|
||||
err = 1;
|
||||
#else
|
||||
/* No digests: a2.html is a second full response, so the body appears twice
|
||||
and only the 304 nm is a revisit. */
|
||||
if (nrevisit != 1 || nresp != 5 || body_occurrences != 2)
|
||||
err = 1;
|
||||
(void) a2_bodyless; /* only meaningful with digests */
|
||||
#endif
|
||||
|
||||
printf("warc: %d records (%d response, %d request, %d revisit): %s\n", nrec,
|
||||
nresp, nreq, nrevisit, err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* Parse a record's header/block split; sets *hdr_len and *block_len, returns 0
|
||||
when Content-Length matches the actual block bytes, -1 otherwise. */
|
||||
static int warc_rec_split(const unsigned char *rec, size_t rlen,
|
||||
size_t *hdr_len, long long *block_len) {
|
||||
const char *sep = warc_memstr((const char *) rec, "\r\n\r\n", rlen, 4);
|
||||
const char *cl;
|
||||
*block_len = 0;
|
||||
if (sep == NULL)
|
||||
return -1;
|
||||
*hdr_len = (size_t) ((const unsigned char *) sep - rec) + 4;
|
||||
cl = warc_memstr((const char *) rec, "Content-Length:", *hdr_len, 15);
|
||||
if (cl == NULL || sscanf(cl + 15, "%lld", block_len) != 1 ||
|
||||
*hdr_len + (size_t) *block_len + 4 != rlen)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* A cap-truncated body is still archived, tagged WARC-Truncated (v1.1). Assert
|
||||
the sole response carries "WARC-Truncated: length" and every record parses.
|
||||
*/
|
||||
static int st_warc_trunc(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *end;
|
||||
int err = 0, found_trunc = 0, nresp = 0;
|
||||
static const char body[] = "partial body bytes\n";
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-trunc: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-trunc.warc.gz");
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
warc_write_transaction(
|
||||
w, "http://test.local/big.bin", "127.0.0.1",
|
||||
"GET /big.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n", body,
|
||||
sizeof(body) - 1, NULL, 200, 0, WARC_TRUNC_LENGTH);
|
||||
warc_close(w);
|
||||
|
||||
data = warc_slurp(path, &data_len);
|
||||
assertf(data != NULL);
|
||||
p = data;
|
||||
end = data + data_len;
|
||||
while (p < end) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, end, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
|
||||
err = 1;
|
||||
else if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) !=
|
||||
NULL) {
|
||||
nresp++;
|
||||
if (warc_memstr((char *) rec, "WARC-Truncated: length", hdr_len, 22) !=
|
||||
NULL)
|
||||
found_trunc = 1;
|
||||
}
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (!found_trunc || nresp != 1)
|
||||
err = 1;
|
||||
printf("warc-trunc: %s\n", err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* An ftp:// capture is ONE resource record: WARC-Type: resource, the payload's
|
||||
own Content-Type, block == payload, and no request/response pair. */
|
||||
static int st_warc_ftp(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *end;
|
||||
int err = 0, nresource = 0, nresp = 0, nreq = 0;
|
||||
static const char body[] = "\x00\x01"
|
||||
"FTP payload"
|
||||
"\x02\x03";
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-ftp: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-ftp.warc.gz");
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
warc_write_resource(w, "ftp://ftp.local/file.bin", "127.0.0.1",
|
||||
"application/octet-stream", body, sizeof(body) - 1, NULL,
|
||||
0);
|
||||
warc_close(w);
|
||||
|
||||
data = warc_slurp(path, &data_len);
|
||||
assertf(data != NULL);
|
||||
p = data;
|
||||
end = data + data_len;
|
||||
while (p < end) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, end, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
|
||||
err = 1;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: resource", hdr_len, 19) != NULL) {
|
||||
nresource++;
|
||||
if ((size_t) block_len != sizeof(body) - 1 ||
|
||||
memcmp(rec + hdr_len, body, sizeof(body) - 1) != 0)
|
||||
err = 1; /* block is the raw payload, no HTTP envelope */
|
||||
if (warc_memstr((char *) rec, "WARC-Target-URI: ftp://ftp.local/file.bin",
|
||||
hdr_len, 41) == NULL ||
|
||||
warc_memstr((char *) rec, "Content-Type: application/octet-stream",
|
||||
hdr_len, 38) == NULL)
|
||||
err = 1;
|
||||
}
|
||||
if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL)
|
||||
nresp++;
|
||||
if (warc_memstr((char *) rec, "WARC-Type: request", hdr_len, 18) != NULL)
|
||||
nreq++;
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (nresource != 1 || nresp != 0 || nreq != 0)
|
||||
err = 1;
|
||||
printf("warc-ftp: resource=%d response=%d request=%d: %s\n", nresource, nresp,
|
||||
nreq, err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* --warc-max-size rotates into <base>-00000.warc.gz, -00001, ...; each segment
|
||||
is independently valid and begins with its own warcinfo. */
|
||||
static int st_warc_rotate(httrackp *opt, int argc, char **argv) {
|
||||
char path[HTS_URLMAXSIZE];
|
||||
char seg[HTS_URLMAXSIZE];
|
||||
warc_writer *w;
|
||||
LLint saved_max;
|
||||
unsigned char body[600];
|
||||
unsigned int rng = 0x12345678u;
|
||||
int err = 0, nseg = 0, i;
|
||||
size_t j;
|
||||
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "warc-rotate: needs a writable directory\n");
|
||||
return 1;
|
||||
}
|
||||
for (j = 0; j < sizeof(body);
|
||||
j++) { /* incompressible: gzip can't shrink it */
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 17;
|
||||
rng ^= rng << 5;
|
||||
body[j] = (unsigned char) (rng >> 24);
|
||||
}
|
||||
fconcat(path, sizeof(path), argv[0], "warc-rot.warc.gz");
|
||||
saved_max = opt->warc_max_size;
|
||||
opt->warc_max_size =
|
||||
1000; /* a couple records per segment => several segments */
|
||||
w = warc_open(opt, path);
|
||||
assertf(w != NULL);
|
||||
for (i = 0; i < 8; i++) {
|
||||
char uri[64];
|
||||
snprintf(uri, sizeof(uri), "http://test.local/f%d.bin", i);
|
||||
warc_write_transaction(
|
||||
w, uri, "127.0.0.1", "GET / HTTP/1.1\r\nHost: test.local\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n",
|
||||
(const char *) body, sizeof(body), NULL, 200, 0, 0);
|
||||
}
|
||||
warc_close(w);
|
||||
opt->warc_max_size = saved_max;
|
||||
|
||||
for (i = 0;; i++) {
|
||||
char fname[64];
|
||||
unsigned char *data;
|
||||
size_t data_len = 0;
|
||||
const unsigned char *p, *pend;
|
||||
int first = 1;
|
||||
snprintf(fname, sizeof(fname), "warc-rot-%05d.warc.gz", i);
|
||||
fconcat(seg, sizeof(seg), argv[0], fname);
|
||||
data = warc_slurp(seg, &data_len);
|
||||
if (data == NULL)
|
||||
break; /* past the last segment */
|
||||
nseg++;
|
||||
p = data;
|
||||
pend = data + data_len;
|
||||
while (p < pend) {
|
||||
size_t rlen = 0, hdr_len = 0;
|
||||
long long block_len = 0;
|
||||
unsigned char *rec = warc_next_member(&p, pend, &rlen);
|
||||
if (rec == NULL) {
|
||||
if (rlen != 0)
|
||||
err = 1;
|
||||
break;
|
||||
}
|
||||
if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
|
||||
err = 1;
|
||||
if (first) { /* each segment leads with its own warcinfo */
|
||||
if (warc_memstr((char *) rec, "WARC-Type: warcinfo", hdr_len, 19) ==
|
||||
NULL)
|
||||
err = 1;
|
||||
first = 0;
|
||||
}
|
||||
freet(rec);
|
||||
}
|
||||
freet(data);
|
||||
if (first) /* empty segment */
|
||||
err = 1;
|
||||
}
|
||||
if (nseg < 2)
|
||||
err = 1;
|
||||
printf("warc-rotate: %d segments: %s\n", nseg, err ? "FAIL" : "OK");
|
||||
return err;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* Registry: name -> handler, with a usage hint and a one-line description. */
|
||||
/* ------------------------------------------------------------ */
|
||||
@@ -3798,8 +3257,6 @@ static const struct selftest_entry {
|
||||
{"idna-decode", "<host>", "decode an IDNA/punycode hostname",
|
||||
st_idna_decode},
|
||||
{"entities", "<string> [encoding]", "unescape HTML entities", st_entities},
|
||||
{"footerfmt", "<template>", "-%F footer positional/named expansion",
|
||||
st_footerfmt},
|
||||
{"unescape-bounds", "", "unescapers reserve the NUL byte (no 1-byte OOB)",
|
||||
st_unescape_bounds},
|
||||
{"hashtable", "<count|file>", "coucal hashtable stress test", st_hashtable},
|
||||
@@ -3875,14 +3332,6 @@ static const struct selftest_entry {
|
||||
{"ftp-line", "", "get_ftp_line bounds a hostile FTP reply line",
|
||||
st_ftpline},
|
||||
{"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
|
||||
{"warc", "<dir>", "WARC/1.1 writer: framing, digests, revisit dedup",
|
||||
st_warc},
|
||||
{"warc-trunc", "<dir>", "WARC-Truncated on a cap-truncated body",
|
||||
st_warc_trunc},
|
||||
{"warc-ftp", "<dir>", "ftp resource record (no HTTP envelope)",
|
||||
st_warc_ftp},
|
||||
{"warc-rotate", "<dir>", "--warc-max-size segment rotation",
|
||||
st_warc_rotate},
|
||||
};
|
||||
|
||||
static void list_selftests(void) {
|
||||
|
||||
129
src/htstools.c
129
src/htstools.c
@@ -754,48 +754,35 @@ typedef struct hts_template_format_buf {
|
||||
size_t offset;
|
||||
} hts_template_format_buf;
|
||||
|
||||
// Bounded append to a template buffer (or FILE); returns -1 on overflow/error.
|
||||
static int htsfmt_putc(hts_template_format_buf *buf, char c) {
|
||||
if (buf->fp != NULL) {
|
||||
assertf(buf->buffer == NULL);
|
||||
if (fputc(c, buf->fp) < 0)
|
||||
return -1;
|
||||
} else {
|
||||
assertf(buf->buffer != NULL);
|
||||
if (buf->offset + 1 < buf->size)
|
||||
buf->buffer[buf->offset++] = c;
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int htsfmt_puts(hts_template_format_buf *buf, const char *s) {
|
||||
size_t i;
|
||||
assertf(s != NULL);
|
||||
for (i = 0; s[i] != '\0'; i++) {
|
||||
if (htsfmt_putc(buf, s[i]) < 0)
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// note: upstream arg list MUST be NULL-terminated for safety
|
||||
// returns a negative value upon error
|
||||
static int hts_template_formatv(hts_template_format_buf *buf,
|
||||
static int hts_template_formatv(hts_template_format_buf *buf,
|
||||
const char *format, va_list args) {
|
||||
#undef FPUTC
|
||||
#undef FPUTS
|
||||
#define FPUTC(C) \
|
||||
do { \
|
||||
if (htsfmt_putc(buf, (C)) < 0) \
|
||||
return -1; \
|
||||
} while (0)
|
||||
#define FPUTS(S) \
|
||||
do { \
|
||||
if (htsfmt_puts(buf, (S)) < 0) \
|
||||
return -1; \
|
||||
} while (0)
|
||||
#define FPUTC(C) do { \
|
||||
if (buf->fp != NULL) { \
|
||||
assertf(buf->buffer == NULL); \
|
||||
if (fputc(C, buf->fp) < 0) { \
|
||||
return -1; \
|
||||
} \
|
||||
} else { \
|
||||
assertf(buf->buffer != NULL); \
|
||||
if (buf->offset + 1 < buf->size) { \
|
||||
buf->buffer[buf->offset++] = (C); \
|
||||
} else { \
|
||||
return -1; \
|
||||
} \
|
||||
} \
|
||||
} while(0)
|
||||
#define FPUTS(S) do { \
|
||||
size_t i; \
|
||||
const char *const str_ = (S); \
|
||||
assertf(str_ != NULL); \
|
||||
for(i = 0 ; str_[i] != '\0' ; i++) { \
|
||||
FPUTC(str_[i]); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
if (buf != NULL && format != NULL) {
|
||||
const char *arg_expanded[32];
|
||||
@@ -872,74 +859,6 @@ int hts_template_format_str(char *buffer, size_t size, const char *format, ...)
|
||||
return success;
|
||||
}
|
||||
|
||||
// Value of the named field, or "" if absent (never NULL, so callers can pass it
|
||||
// straight to a formatter).
|
||||
static const char *footer_field_value(const hts_footer_field *fields,
|
||||
size_t nfields, const char *name) {
|
||||
size_t j;
|
||||
for (j = 0; j < nfields; j++) {
|
||||
if (strcmp(fields[j].name, name) == 0)
|
||||
return fields[j].value != NULL ? fields[j].value : "";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
int hts_footer_format(char *buffer, size_t size, const char *footer,
|
||||
const hts_footer_field *fields, size_t nfields) {
|
||||
hts_template_format_buf buf = {NULL, buffer, size, 0};
|
||||
size_t i;
|
||||
|
||||
if (footer == NULL || buffer == NULL || size == 0)
|
||||
return -1;
|
||||
// %s keeps the legacy positional model, byte-for-byte for existing -%F
|
||||
// strings: addr, path, date, version, looked up by name and
|
||||
// order-independent.
|
||||
if (strstr(footer, "%s") != NULL)
|
||||
return hts_template_format_str(
|
||||
buffer, size, footer, footer_field_value(fields, nfields, "addr"),
|
||||
footer_field_value(fields, nfields, "path"),
|
||||
footer_field_value(fields, nfields, "date"),
|
||||
footer_field_value(fields, nfields, "version"), /* EOF */ NULL);
|
||||
// "{{"/"}}" emit a literal brace; an unknown "{...}" is left verbatim so
|
||||
// typos stay visible.
|
||||
for (i = 0; footer[i] != '\0'; i++) {
|
||||
const char c = footer[i];
|
||||
if (c == '{' && footer[i + 1] == '{') {
|
||||
if (htsfmt_putc(&buf, '{') < 0)
|
||||
return -1;
|
||||
i++;
|
||||
} else if (c == '}' && footer[i + 1] == '}') {
|
||||
if (htsfmt_putc(&buf, '}') < 0)
|
||||
return -1;
|
||||
i++;
|
||||
} else if (c == '{') {
|
||||
const char *const end = strchr(footer + i + 1, '}');
|
||||
int matched = 0;
|
||||
if (end != NULL) {
|
||||
const size_t namelen = (size_t) (end - (footer + i + 1));
|
||||
size_t j;
|
||||
for (j = 0; j < nfields; j++) {
|
||||
if (strlen(fields[j].name) == namelen &&
|
||||
strncmp(fields[j].name, footer + i + 1, namelen) == 0) {
|
||||
if (htsfmt_puts(&buf,
|
||||
fields[j].value != NULL ? fields[j].value : "") < 0)
|
||||
return -1;
|
||||
i += namelen + 1; // consume the name and its closing '}'
|
||||
matched = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!matched && htsfmt_putc(&buf, '{') < 0)
|
||||
return -1;
|
||||
} else if (htsfmt_putc(&buf, c) < 0) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
buffer[buf.offset] = '\0';
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Note: NOT utf-8 */
|
||||
HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
|
||||
const char *binpath) {
|
||||
|
||||
@@ -73,20 +73,6 @@ HTS_INLINE int rech_tageq_all(const char *adr, const char *s);
|
||||
int hts_template_format(FILE *const out, const char *format, ...);
|
||||
int hts_template_format_str(char *buffer, size_t size, const char *format, ...);
|
||||
|
||||
// A footer named field and its already-context-escaped value.
|
||||
typedef struct hts_footer_field {
|
||||
const char *name;
|
||||
const char *value;
|
||||
} hts_footer_field;
|
||||
|
||||
// Expand a footer template. A "%s" in it selects the legacy positional model,
|
||||
// consuming the "addr"/"path"/"date"/"version" fields in that order; otherwise
|
||||
// "{name}" is substituted from fields by name ("{{"/"}}" emit a literal brace,
|
||||
// an unknown "{...}" is left verbatim). Values must already be escaped for the
|
||||
// target context by the caller. Returns <0 on overflow.
|
||||
int hts_footer_format(char *buffer, size_t size, const char *footer,
|
||||
const hts_footer_field *fields, size_t nfields);
|
||||
|
||||
#define rech_tageq(adr,s) \
|
||||
( \
|
||||
( (*((adr)-1)=='<') || (is_space(*((adr)-1))) ) ? \
|
||||
|
||||
995
src/htswarc.c
995
src/htswarc.c
@@ -1,995 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* HTTrack WARC/1.1 output writer (ISO 28500). See warc.h.
|
||||
Strategy B (see design): the response record stores the decoded body and a
|
||||
normalized header block (Content-Encoding/Transfer-Encoding stripped,
|
||||
Content-Length rewritten to the decoded length). Valid, replayable WARC. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#define HTS_INTERNAL_BYTECODE
|
||||
|
||||
#include "htswarc.h"
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htslib.h"
|
||||
#include "htstools.h"
|
||||
#include "htssafe.h"
|
||||
#include "htszlib.h"
|
||||
#include "coucal/coucal.h"
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
#include <time.h>
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/rand.h>
|
||||
#endif
|
||||
|
||||
/* opt->state.warc value meaning "open failed once, do not retry". */
|
||||
#define WARC_DISABLED ((void *) ~(uintptr_t) 0)
|
||||
|
||||
struct warc_writer {
|
||||
FILE *f;
|
||||
int gz; /* 1: one gzip member per record; 0: raw */
|
||||
uint64_t offset; /* running byte offset (member starts, for a future index) */
|
||||
uint64_t counter; /* monotonic record counter */
|
||||
uint64_t rng; /* PRNG state for the UUID fallback */
|
||||
char info_id[64]; /* warcinfo WARC-Record-ID, referenced by every record */
|
||||
coucal seen; /* base32 payload digest -> "uri\001date" (revisit dedup) */
|
||||
/* --warc-max-size rotation: NAME-00000.warc.gz, -00001, ... (wget-style). */
|
||||
uint64_t max_size; /* rotate once a segment reaches this; 0: single file */
|
||||
char *seg_base; /* segment path without the .warc[.gz] suffix, or NULL */
|
||||
const char *seg_ext; /* ".warc.gz" or ".warc" */
|
||||
unsigned seg; /* current segment number */
|
||||
char *info_fields; /* warcinfo body, re-emitted at each new segment */
|
||||
};
|
||||
|
||||
const char *warc_truncated_reason(int code) {
|
||||
switch (code) {
|
||||
case WARC_TRUNC_LENGTH:
|
||||
return "length";
|
||||
case WARC_TRUNC_TIME:
|
||||
return "time";
|
||||
case WARC_TRUNC_DISCONNECT:
|
||||
return "disconnect";
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- growable byte buffer (overflow-safe, project allocators) ---- */
|
||||
|
||||
typedef struct {
|
||||
char *data;
|
||||
size_t len;
|
||||
size_t cap;
|
||||
} wbuf;
|
||||
|
||||
static void wbuf_free(wbuf *b) {
|
||||
freet(b->data);
|
||||
b->len = b->cap = 0;
|
||||
}
|
||||
|
||||
/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
|
||||
static int wbuf_add(wbuf *b, const void *p, size_t n) {
|
||||
if (n > (size_t) -1 - b->len)
|
||||
return -1;
|
||||
if (b->len + n > b->cap) {
|
||||
size_t ncap = b->cap ? b->cap : 256;
|
||||
char *nd;
|
||||
while (ncap < b->len + n) {
|
||||
if (ncap > (size_t) -1 / 2)
|
||||
return -1;
|
||||
ncap *= 2;
|
||||
}
|
||||
nd = realloct(b->data, ncap);
|
||||
if (nd == NULL)
|
||||
return -1;
|
||||
b->data = nd;
|
||||
b->cap = ncap;
|
||||
}
|
||||
memcpy(b->data + b->len, p, n);
|
||||
b->len += n;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int wbuf_puts(wbuf *b, const char *s) {
|
||||
return wbuf_add(b, s, strlen(s));
|
||||
}
|
||||
|
||||
static int wbuf_printf(wbuf *b, const char *fmt, ...) HTS_PRINTF_FUN(2, 3);
|
||||
|
||||
static int wbuf_printf(wbuf *b, const char *fmt, ...) {
|
||||
char tmp[1024];
|
||||
int n;
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
n = vsnprintf(tmp, sizeof(tmp), fmt, ap);
|
||||
va_end(ap);
|
||||
if (n < 0 || (size_t) n >= sizeof(tmp))
|
||||
return -1;
|
||||
return wbuf_add(b, tmp, (size_t) n);
|
||||
}
|
||||
|
||||
/* ---- gzip-per-record member writer (mirrors ae_write_packed) ---- */
|
||||
|
||||
typedef struct {
|
||||
warc_writer *w;
|
||||
z_stream strm;
|
||||
int active; /* deflate stream initialized */
|
||||
} member;
|
||||
|
||||
static int member_begin(member *m, warc_writer *w) {
|
||||
m->w = w;
|
||||
m->active = 0;
|
||||
if (w->gz) {
|
||||
memset(&m->strm, 0, sizeof(m->strm));
|
||||
/* windowBits=31 => full RFC1952 gzip member */
|
||||
if (deflateInit2(&m->strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8,
|
||||
Z_DEFAULT_STRATEGY) != Z_OK)
|
||||
return -1;
|
||||
m->active = 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int member_write(member *m, const void *p, size_t n) {
|
||||
if (!m->w->gz)
|
||||
return (n == 0 || fwrite(p, 1, n, m->w->f) == n) ? 0 : -1;
|
||||
m->strm.next_in = (const Bytef *) p;
|
||||
while (n > 0) {
|
||||
unsigned char out[8192];
|
||||
size_t got;
|
||||
uInt chunk = (n > (uInt) -1) ? (uInt) -1 : (uInt) n;
|
||||
m->strm.avail_in = chunk;
|
||||
do {
|
||||
m->strm.next_out = out;
|
||||
m->strm.avail_out = sizeof(out);
|
||||
if (deflate(&m->strm, Z_NO_FLUSH) != Z_OK)
|
||||
return -1;
|
||||
got = sizeof(out) - m->strm.avail_out;
|
||||
if (got > 0 && fwrite(out, 1, got, m->w->f) != got)
|
||||
return -1;
|
||||
} while (m->strm.avail_out == 0);
|
||||
n -= chunk;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int member_end(member *m) {
|
||||
int rc = 0;
|
||||
if (m->active) {
|
||||
unsigned char out[8192];
|
||||
int zerr;
|
||||
m->strm.avail_in = 0;
|
||||
do {
|
||||
m->strm.next_out = out;
|
||||
m->strm.avail_out = sizeof(out);
|
||||
zerr = deflate(&m->strm, Z_FINISH);
|
||||
{
|
||||
size_t got = sizeof(out) - m->strm.avail_out;
|
||||
if (got > 0 && fwrite(out, 1, got, m->w->f) != got)
|
||||
rc = -1;
|
||||
}
|
||||
} while (zerr == Z_OK);
|
||||
if (zerr != Z_STREAM_END)
|
||||
rc = -1;
|
||||
deflateEnd(&m->strm);
|
||||
m->active = 0;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ---- SHA-1 + Base32 (digests are OpenSSL-only; omitted otherwise) ---- */
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
/* Streaming SHA-1 over the block (all regions) and the payload (body only). */
|
||||
typedef struct {
|
||||
EVP_MD_CTX *block;
|
||||
EVP_MD_CTX *payload;
|
||||
} digester;
|
||||
|
||||
static void base32_20(const unsigned char in[20], char out[33]) {
|
||||
static const char a[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
||||
int i, o = 0;
|
||||
uint64_t buf = 0;
|
||||
int bits = 0;
|
||||
for (i = 0; i < 20; i++) {
|
||||
buf = (buf << 8) | in[i];
|
||||
bits += 8;
|
||||
while (bits >= 5) {
|
||||
bits -= 5;
|
||||
out[o++] = a[(buf >> bits) & 0x1F];
|
||||
}
|
||||
}
|
||||
out[o] = '\0'; /* 20 bytes => exactly 32 base32 chars, no padding */
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Stream a block to a sink. When http_section, the HTTP header bytes (may be
|
||||
empty) plus their terminating CRLF are emitted first; the separator is bound
|
||||
to http_section, not to http_hdr being non-NULL, so an empty header still
|
||||
emits (and is counted in) the 2-byte separator (F3). The on-disk body is
|
||||
written as EXACTLY body_len octets — capped if the file grew, zero-padded if
|
||||
it shrank — so the declared Content-Length always equals the bytes written
|
||||
across every pass (F2). region 0=header, 1=body. Returns 0 on success. */
|
||||
typedef int (*warc_sink)(void *ctx, int region, const void *p, size_t n);
|
||||
|
||||
static int stream_body_pad(warc_sink sink, void *ctx, size_t remaining) {
|
||||
static const char zeros[4096] = {0};
|
||||
while (remaining > 0) {
|
||||
size_t chunk = (remaining < sizeof(zeros)) ? remaining : sizeof(zeros);
|
||||
if (sink(ctx, 1, zeros, chunk) != 0)
|
||||
return -1;
|
||||
remaining -= chunk;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int stream_block(int http_section, const char *http_hdr,
|
||||
size_t http_hdr_len, int has_body, const char *body,
|
||||
size_t body_len, const char *body_path, warc_sink sink,
|
||||
void *ctx) {
|
||||
if (http_section) {
|
||||
if (http_hdr != NULL && http_hdr_len > 0 &&
|
||||
sink(ctx, 0, http_hdr, http_hdr_len) != 0)
|
||||
return -1;
|
||||
if (sink(ctx, 0, "\r\n", 2) != 0)
|
||||
return -1;
|
||||
}
|
||||
if (has_body) {
|
||||
if (body != NULL) {
|
||||
if (body_len > 0 && sink(ctx, 1, body, body_len) != 0)
|
||||
return -1;
|
||||
} else if (body_path != NULL) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
size_t remaining = body_len;
|
||||
FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), body_path), "rb");
|
||||
if (fp == NULL)
|
||||
return -1;
|
||||
while (remaining > 0) {
|
||||
char b[32768];
|
||||
size_t want = (remaining < sizeof(b)) ? remaining : sizeof(b);
|
||||
size_t nl = fread(b, 1, want, fp);
|
||||
if (nl == 0)
|
||||
break; /* short file: pad below so written == declared */
|
||||
if (sink(ctx, 1, b, nl) != 0) {
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
remaining -= nl;
|
||||
}
|
||||
fclose(fp);
|
||||
if (stream_body_pad(sink, ctx, remaining) != 0)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
static int digest_sink(void *ctx, int region, const void *p, size_t n) {
|
||||
digester *d = (digester *) ctx;
|
||||
if (d->block != NULL && EVP_DigestUpdate(d->block, p, n) != 1)
|
||||
return -1;
|
||||
if (region == 1 && d->payload != NULL &&
|
||||
EVP_DigestUpdate(d->payload, p, n) != 1)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int write_sink(void *ctx, int region, const void *p, size_t n) {
|
||||
(void) region;
|
||||
return member_write((member *) ctx, p, n);
|
||||
}
|
||||
|
||||
/* Base32 SHA-1 of a transaction payload (body only), for revisit dedup.
|
||||
Returns 1 and fills out[33] on success, 0 without OpenSSL or on error. */
|
||||
static int payload_digest_b32(const char *body, size_t body_len,
|
||||
const char *body_path, char out[33]) {
|
||||
#if HTS_USEOPENSSL
|
||||
digester d;
|
||||
unsigned char md[EVP_MAX_MD_SIZE];
|
||||
unsigned int mdlen = 0;
|
||||
int ok;
|
||||
d.block = NULL;
|
||||
d.payload = EVP_MD_CTX_new();
|
||||
if (d.payload == NULL)
|
||||
return 0;
|
||||
if (EVP_DigestInit_ex(d.payload, EVP_sha1(), NULL) != 1) {
|
||||
EVP_MD_CTX_free(d.payload);
|
||||
return 0;
|
||||
}
|
||||
ok = stream_block(0, NULL, 0, 1, body, body_len, body_path, digest_sink,
|
||||
&d) == 0 &&
|
||||
EVP_DigestFinal_ex(d.payload, md, &mdlen) == 1 && mdlen == 20;
|
||||
EVP_MD_CTX_free(d.payload);
|
||||
if (!ok)
|
||||
return 0;
|
||||
base32_20(md, out);
|
||||
return 1;
|
||||
#else
|
||||
(void) body;
|
||||
(void) body_len;
|
||||
(void) body_path;
|
||||
(void) out;
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* ---- misc record helpers ---- */
|
||||
|
||||
static void warc_fill_random(warc_writer *w, unsigned char *b, size_t n) {
|
||||
size_t i;
|
||||
#if HTS_USEOPENSSL
|
||||
if (n <= (size_t) 0x7fffffff && RAND_bytes(b, (int) n) == 1)
|
||||
return;
|
||||
#endif
|
||||
for (i = 0; i < n; i++) {
|
||||
w->rng ^= w->rng << 13;
|
||||
w->rng ^= w->rng >> 7;
|
||||
w->rng ^= w->rng << 17;
|
||||
b[i] = (unsigned char) (w->rng >> 24);
|
||||
}
|
||||
}
|
||||
|
||||
/* urn:uuid: v4-shaped record id (uniqueness within the run is what matters). */
|
||||
static void warc_make_id(warc_writer *w, char out[64]) {
|
||||
unsigned char b[16];
|
||||
w->counter++;
|
||||
warc_fill_random(w, b, sizeof(b));
|
||||
b[6] = (unsigned char) ((b[6] & 0x0F) | 0x40);
|
||||
b[8] = (unsigned char) ((b[8] & 0x3F) | 0x80);
|
||||
snprintf(out, 64,
|
||||
"<urn:uuid:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-"
|
||||
"%02x%02x%02x%02x%02x%02x>",
|
||||
b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10],
|
||||
b[11], b[12], b[13], b[14], b[15]);
|
||||
}
|
||||
|
||||
static void warc_now_iso8601(char out[32]) {
|
||||
time_t t = time(NULL);
|
||||
struct tm tmv;
|
||||
#if defined(_WIN32)
|
||||
struct tm *g = gmtime(&t);
|
||||
if (g != NULL)
|
||||
tmv = *g;
|
||||
else
|
||||
memset(&tmv, 0, sizeof(tmv));
|
||||
#else
|
||||
if (gmtime_r(&t, &tmv) == NULL)
|
||||
memset(&tmv, 0, sizeof(tmv));
|
||||
#endif
|
||||
strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &tmv);
|
||||
}
|
||||
|
||||
/* Case-insensitive "does line start with name:" test. */
|
||||
static int header_is(const char *line, size_t line_len, const char *name) {
|
||||
size_t nl = strlen(name);
|
||||
if (line_len < nl + 1)
|
||||
return 0;
|
||||
if (strncasecmp(line, name, nl) != 0)
|
||||
return 0;
|
||||
return line[nl] == ':';
|
||||
}
|
||||
|
||||
/* Build the normalized HTTP header block from raw resp_hdr into out (no
|
||||
trailing CRLF terminator). Always drops Transfer-Encoding (hop-by-hop).
|
||||
When set_cl>=0, also drops Content-Encoding and the original Content-Length
|
||||
and appends "Content-Length: <set_cl>". Returns 0 on success. */
|
||||
static int normalize_http_headers(const char *resp_hdr, long long set_cl,
|
||||
wbuf *out) {
|
||||
const char *p = resp_hdr;
|
||||
int first = 1;
|
||||
if (resp_hdr == NULL)
|
||||
return -1;
|
||||
while (*p != '\0') {
|
||||
const char *eol = strchr(p, '\n');
|
||||
size_t len = (eol != NULL) ? (size_t) (eol - p) : strlen(p);
|
||||
size_t raw = len;
|
||||
if (len > 0 && p[len - 1] == '\r')
|
||||
len--; /* strip CR; re-added as CRLF below */
|
||||
if (len == 0)
|
||||
break; /* blank line: end of headers */
|
||||
if (first) {
|
||||
first = 0; /* status line: keep verbatim */
|
||||
} else if (header_is(p, len, "Transfer-Encoding")) {
|
||||
goto next;
|
||||
} else if (set_cl >= 0 && (header_is(p, len, "Content-Encoding") ||
|
||||
header_is(p, len, "Content-Length"))) {
|
||||
goto next;
|
||||
}
|
||||
if (wbuf_add(out, p, len) != 0 || wbuf_add(out, "\r\n", 2) != 0)
|
||||
return -1;
|
||||
next:
|
||||
if (eol == NULL)
|
||||
break;
|
||||
p = eol + 1;
|
||||
(void) raw;
|
||||
}
|
||||
if (set_cl >= 0 && wbuf_printf(out, "Content-Length: %lld\r\n", set_cl) != 0)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Close the current segment and open the next; writes its warcinfo. */
|
||||
static int warc_rotate(warc_writer *w);
|
||||
|
||||
/* Emit one full WARC record. When http_section, the block carries an HTTP
|
||||
header block (http_hdr, possibly empty) + a CRLF separator; body follows when
|
||||
has_body. block_len is derived here (single source of truth: separator and
|
||||
payload are counted exactly as stream_block emits them), so a declared
|
||||
Content-Length can never desync from the written bytes. The payload digest
|
||||
(body-only) is passed in when already known. truncated is a WARC-Truncated
|
||||
reason token or NULL. On success the record id is copied to out_id (may be
|
||||
NULL). */
|
||||
static int warc_emit(warc_writer *w, const char *type, const char *content_type,
|
||||
const char *target_uri, const char *ip,
|
||||
const char *concurrent_to, const char *refers_uri,
|
||||
const char *refers_date, const char *profile,
|
||||
const char *payload_digest, const char *truncated,
|
||||
int http_section, const char *http_hdr,
|
||||
size_t http_hdr_len, int has_body, const char *body,
|
||||
size_t body_len, const char *body_path, char out_id[64]) {
|
||||
wbuf hdr;
|
||||
member m;
|
||||
char id[64], date[32];
|
||||
size_t sep = http_section ? 2 : 0;
|
||||
size_t payload = has_body ? body_len : 0;
|
||||
size_t block_len;
|
||||
int rc = -1;
|
||||
#if HTS_USEOPENSSL
|
||||
digester d;
|
||||
unsigned char md[EVP_MAX_MD_SIZE];
|
||||
unsigned int mdlen = 0;
|
||||
char block_b32[33];
|
||||
int have_block_digest = 0;
|
||||
#endif
|
||||
|
||||
/* Rotate to the next segment before this record when the current one is full;
|
||||
never split a record, and never rotate a warcinfo (it opens a segment). */
|
||||
if (w->max_size > 0 && w->seg_base != NULL && w->offset >= w->max_size &&
|
||||
strcmp(type, "warcinfo") != 0) {
|
||||
if (warc_rotate(w) != 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* F4: overflow-safe block length; http_hdr_len+sep is provably small. */
|
||||
if (payload > (size_t) -1 - http_hdr_len - sep)
|
||||
return -1;
|
||||
block_len = http_hdr_len + sep + payload;
|
||||
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
warc_make_id(w, id);
|
||||
warc_now_iso8601(date);
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
/* Block digest over the whole block, in one streaming pass. */
|
||||
d.block = EVP_MD_CTX_new();
|
||||
d.payload = NULL;
|
||||
if (d.block != NULL && EVP_DigestInit_ex(d.block, EVP_sha1(), NULL) == 1 &&
|
||||
stream_block(http_section, http_hdr, http_hdr_len, has_body, body,
|
||||
body_len, body_path, digest_sink, &d) == 0 &&
|
||||
EVP_DigestFinal_ex(d.block, md, &mdlen) == 1 && mdlen == 20) {
|
||||
base32_20(md, block_b32);
|
||||
have_block_digest = 1;
|
||||
}
|
||||
if (d.block != NULL)
|
||||
EVP_MD_CTX_free(d.block);
|
||||
#endif
|
||||
|
||||
if (wbuf_puts(&hdr, "WARC/1.1\r\n") != 0 ||
|
||||
wbuf_printf(&hdr, "WARC-Type: %s\r\n", type) != 0 ||
|
||||
wbuf_printf(&hdr, "WARC-Record-ID: %s\r\n", id) != 0 ||
|
||||
wbuf_printf(&hdr, "WARC-Date: %s\r\n", date) != 0)
|
||||
goto done;
|
||||
if (content_type != NULL &&
|
||||
wbuf_printf(&hdr, "Content-Type: %s\r\n", content_type) != 0)
|
||||
goto done;
|
||||
if (wbuf_printf(&hdr, "Content-Length: %llu\r\n",
|
||||
(unsigned long long) block_len) != 0)
|
||||
goto done;
|
||||
if (w->info_id[0] != '\0' && strcmp(type, "warcinfo") != 0 &&
|
||||
wbuf_printf(&hdr, "WARC-Warcinfo-ID: %s\r\n", w->info_id) != 0)
|
||||
goto done;
|
||||
if (target_uri != NULL && target_uri[0] != '\0' &&
|
||||
wbuf_printf(&hdr, "WARC-Target-URI: %s\r\n", target_uri) != 0)
|
||||
goto done;
|
||||
if (ip != NULL && ip[0] != '\0' &&
|
||||
wbuf_printf(&hdr, "WARC-IP-Address: %s\r\n", ip) != 0)
|
||||
goto done;
|
||||
if (concurrent_to != NULL && concurrent_to[0] != '\0' &&
|
||||
wbuf_printf(&hdr, "WARC-Concurrent-To: %s\r\n", concurrent_to) != 0)
|
||||
goto done;
|
||||
if (profile != NULL &&
|
||||
wbuf_printf(&hdr, "WARC-Profile: %s\r\n", profile) != 0)
|
||||
goto done;
|
||||
if (refers_uri != NULL && refers_uri[0] != '\0' &&
|
||||
wbuf_printf(&hdr, "WARC-Refers-To-Target-URI: %s\r\n", refers_uri) != 0)
|
||||
goto done;
|
||||
if (refers_date != NULL && refers_date[0] != '\0' &&
|
||||
wbuf_printf(&hdr, "WARC-Refers-To-Date: %s\r\n", refers_date) != 0)
|
||||
goto done;
|
||||
#if HTS_USEOPENSSL
|
||||
if (have_block_digest &&
|
||||
wbuf_printf(&hdr, "WARC-Block-Digest: sha1:%s\r\n", block_b32) != 0)
|
||||
goto done;
|
||||
#endif
|
||||
if (payload_digest != NULL && payload_digest[0] != '\0' &&
|
||||
wbuf_printf(&hdr, "WARC-Payload-Digest: sha1:%s\r\n", payload_digest) !=
|
||||
0)
|
||||
goto done;
|
||||
if (truncated != NULL &&
|
||||
wbuf_printf(&hdr, "WARC-Truncated: %s\r\n", truncated) != 0)
|
||||
goto done;
|
||||
if (wbuf_puts(&hdr, "\r\n") != 0)
|
||||
goto done;
|
||||
|
||||
if (member_begin(&m, w) != 0)
|
||||
goto done;
|
||||
if (member_write(&m, hdr.data, hdr.len) != 0 ||
|
||||
stream_block(http_section, http_hdr, http_hdr_len, has_body, body,
|
||||
body_len, body_path, write_sink, &m) != 0 ||
|
||||
member_write(&m, "\r\n\r\n", 4) != 0) {
|
||||
member_end(&m);
|
||||
goto done;
|
||||
}
|
||||
if (member_end(&m) != 0)
|
||||
goto done;
|
||||
|
||||
{
|
||||
long pos = ftell(w->f);
|
||||
if (pos >= 0)
|
||||
w->offset = (uint64_t) pos;
|
||||
}
|
||||
if (out_id != NULL)
|
||||
strlcpybuff(out_id, id, 64);
|
||||
rc = 0;
|
||||
done:
|
||||
wbuf_free(&hdr);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ---- segment rotation (--warc-max-size) ---- */
|
||||
|
||||
/* Emit the warcinfo that heads a segment; sets w->info_id for its records. */
|
||||
static int warc_write_warcinfo_record(warc_writer *w) {
|
||||
w->info_id[0] = '\0'; /* warcinfo itself carries no WARC-Warcinfo-ID */
|
||||
return warc_emit(w, "warcinfo", "application/warc-fields", NULL, NULL, NULL,
|
||||
NULL, NULL, NULL, NULL, NULL, 0, NULL, 0, 1, w->info_fields,
|
||||
w->info_fields != NULL ? strlen(w->info_fields) : 0, NULL,
|
||||
w->info_id);
|
||||
}
|
||||
|
||||
static int warc_rotate(warc_writer *w) {
|
||||
char namebuf[HTS_URLMAXSIZE * 2];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
if (w->f != NULL) {
|
||||
fclose(w->f);
|
||||
w->f = NULL;
|
||||
}
|
||||
w->seg++;
|
||||
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
|
||||
w->seg_ext);
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), namebuf), "wb");
|
||||
if (w->f == NULL)
|
||||
return -1;
|
||||
w->offset = 0;
|
||||
return warc_write_warcinfo_record(w);
|
||||
}
|
||||
|
||||
/* ---- request stash (engine hooks) ---- */
|
||||
|
||||
void warc_stash_request(htsblk *r, const char *reqhdr) {
|
||||
if (r == NULL)
|
||||
return;
|
||||
freet(r->warc_reqhdr);
|
||||
if (reqhdr != NULL)
|
||||
r->warc_reqhdr = strdupt(reqhdr);
|
||||
}
|
||||
|
||||
void warc_stash_response(htsblk *r, const char *resphdr) {
|
||||
if (r == NULL)
|
||||
return;
|
||||
freet(r->warc_resphdr);
|
||||
if (resphdr != NULL)
|
||||
r->warc_resphdr = strdupt(resphdr);
|
||||
}
|
||||
|
||||
void warc_free_request(htsblk *r) {
|
||||
if (r != NULL) {
|
||||
freet(r->warc_reqhdr);
|
||||
freet(r->warc_resphdr);
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- open / close ---- */
|
||||
|
||||
warc_writer *warc_open(httrackp *opt, const char *path) {
|
||||
warc_writer *w;
|
||||
char namebuf[HTS_URLMAXSIZE * 2];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
wbuf info;
|
||||
const char *robots;
|
||||
size_t plen;
|
||||
|
||||
if (path == NULL)
|
||||
return NULL;
|
||||
|
||||
/* --warc with no name: <output>/httrack-<timestamp>.warc.gz */
|
||||
if (strcmp(path, WARC_AUTONAME) == 0) {
|
||||
char ts[32];
|
||||
time_t t = time(NULL);
|
||||
struct tm tmv;
|
||||
#if defined(_WIN32)
|
||||
struct tm *g = gmtime(&t);
|
||||
if (g != NULL)
|
||||
tmv = *g;
|
||||
else
|
||||
memset(&tmv, 0, sizeof(tmv));
|
||||
#else
|
||||
if (gmtime_r(&t, &tmv) == NULL)
|
||||
memset(&tmv, 0, sizeof(tmv));
|
||||
#endif
|
||||
strftime(ts, sizeof(ts), "%Y%m%d%H%M%S", &tmv);
|
||||
snprintf(catbuff, sizeof(catbuff), "httrack-%s.warc.gz", ts);
|
||||
path =
|
||||
fconcat(namebuf, sizeof(namebuf), StringBuff(opt->path_html), catbuff);
|
||||
} else {
|
||||
/* --warc-file NAME: append .warc.gz unless already a .warc/.warc.gz name;
|
||||
place bare basenames under the output directory (like the auto name). */
|
||||
size_t l = strlen(path);
|
||||
int has_warc = (l >= 5 && strcasecmp(path + l - 5, ".warc") == 0);
|
||||
int has_gz = (l >= 3 && strcasecmp(path + l - 3, ".gz") == 0);
|
||||
char named[HTS_URLMAXSIZE];
|
||||
if (has_warc || has_gz)
|
||||
strlcpybuff(named, path, sizeof(named));
|
||||
else
|
||||
snprintf(named, sizeof(named), "%s.warc.gz", path);
|
||||
if (strchr(named, '/') == NULL && strchr(named, '\\') == NULL) {
|
||||
path =
|
||||
fconcat(namebuf, sizeof(namebuf), StringBuff(opt->path_html), named);
|
||||
} else {
|
||||
strlcpybuff(namebuf, named, sizeof(namebuf));
|
||||
path = namebuf;
|
||||
}
|
||||
}
|
||||
|
||||
w = calloct(1, sizeof(*w));
|
||||
if (w == NULL)
|
||||
return NULL;
|
||||
plen = strlen(path);
|
||||
w->gz = (plen >= 3 && strcasecmp(path + plen - 3, ".gz") == 0);
|
||||
w->rng = (uint64_t) time(NULL) ^ ((uint64_t) (uintptr_t) w << 16) ^
|
||||
0x9e3779b97f4a7c15ULL;
|
||||
w->seen = coucal_new(0);
|
||||
if (w->seen != NULL)
|
||||
coucal_value_is_malloc(w->seen, 1);
|
||||
w->max_size = (opt->warc_max_size > 0) ? (uint64_t) opt->warc_max_size : 0;
|
||||
|
||||
/* Build the warcinfo body once; each segment re-emits it. */
|
||||
robots = (opt->robots == HTS_ROBOTS_NEVER) ? "ignore" : "obey";
|
||||
memset(&info, 0, sizeof(info));
|
||||
if (wbuf_printf(&info,
|
||||
"software: HTTrack/%s (+https://www.httrack.com/)\r\n"
|
||||
"format: WARC file version 1.1\r\n"
|
||||
"conformsTo: http://iipc.github.io/warc-specifications/"
|
||||
"specifications/warc-format/warc-1.1/\r\n"
|
||||
"robots: %s\r\n",
|
||||
HTTRACK_VERSION, robots) != 0 ||
|
||||
(StringNotEmpty(opt->path_html) &&
|
||||
wbuf_printf(&info, "isPartOf: %s\r\n", StringBuff(opt->path_html)) !=
|
||||
0) ||
|
||||
wbuf_add(&info, "", 1) != 0) { /* NUL-terminate for info_fields */
|
||||
wbuf_free(&info);
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
w->info_fields = strdupt(info.data);
|
||||
wbuf_free(&info);
|
||||
if (w->info_fields == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Rotation on: the first segment is <base>-00000<ext> (wget-style); split the
|
||||
resolved path into base + suffix so later segments reuse the base. */
|
||||
if (w->max_size > 0) {
|
||||
size_t l = strlen(path);
|
||||
size_t baselen;
|
||||
if (l >= 8 && strcasecmp(path + l - 8, ".warc.gz") == 0) {
|
||||
baselen = l - 8;
|
||||
w->seg_ext = ".warc.gz";
|
||||
} else if (l >= 5 && strcasecmp(path + l - 5, ".warc") == 0) {
|
||||
baselen = l - 5;
|
||||
w->seg_ext = ".warc";
|
||||
} else {
|
||||
baselen = l;
|
||||
w->seg_ext = w->gz ? ".warc.gz" : ".warc";
|
||||
}
|
||||
w->seg_base = malloct(baselen + 1);
|
||||
if (w->seg_base == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(w->seg_base, path, baselen);
|
||||
w->seg_base[baselen] = '\0';
|
||||
w->seg = 0;
|
||||
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
|
||||
w->seg_ext);
|
||||
path = namebuf;
|
||||
}
|
||||
|
||||
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
|
||||
if (w->f == NULL) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (warc_write_warcinfo_record(w) != 0) {
|
||||
warc_close(w);
|
||||
return NULL;
|
||||
}
|
||||
return w;
|
||||
}
|
||||
|
||||
void warc_close(warc_writer *w) {
|
||||
if (w == NULL)
|
||||
return;
|
||||
if (w->f != NULL)
|
||||
fclose(w->f);
|
||||
if (w->seen != NULL)
|
||||
coucal_delete(&w->seen);
|
||||
freet(w->seg_base);
|
||||
freet(w->info_fields);
|
||||
freet(w);
|
||||
}
|
||||
|
||||
void warc_close_opt(httrackp *opt) {
|
||||
if (opt->state.warc != NULL && opt->state.warc != WARC_DISABLED) {
|
||||
warc_close((warc_writer *) opt->state.warc);
|
||||
}
|
||||
opt->state.warc = NULL;
|
||||
}
|
||||
|
||||
/* ---- one transaction ---- */
|
||||
|
||||
int warc_write_transaction(warc_writer *w, const char *target_uri,
|
||||
const char *ip, const char *req_hdr,
|
||||
const char *resp_hdr, const char *body,
|
||||
size_t body_len, const char *body_path,
|
||||
int statuscode, int is_update_unchanged,
|
||||
int truncated) {
|
||||
wbuf http;
|
||||
char resp_id[64];
|
||||
char pdig[33];
|
||||
int have_pdig;
|
||||
int is_revisit = 0;
|
||||
const char *profile = NULL;
|
||||
const char *refers_uri = NULL;
|
||||
const char *refers_date = NULL;
|
||||
char refers_buf[HTS_URLMAXSIZE * 2 + 64];
|
||||
int has_payload;
|
||||
int emit_body;
|
||||
int rc = -1;
|
||||
|
||||
if (resp_hdr == NULL)
|
||||
return -1;
|
||||
|
||||
/* A payload exists (for digesting) unless this is a bodyless 304. */
|
||||
has_payload = (body_len > 0 && (body != NULL || body_path != NULL) &&
|
||||
!is_update_unchanged);
|
||||
|
||||
/* Payload digest drives identical-payload-digest dedup (OpenSSL only). */
|
||||
have_pdig =
|
||||
has_payload ? payload_digest_b32(body, body_len, body_path, pdig) : 0;
|
||||
|
||||
if (is_update_unchanged) {
|
||||
is_revisit = 1;
|
||||
profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
|
||||
} else if (have_pdig && w->seen != NULL) {
|
||||
void *prev = NULL;
|
||||
if (coucal_read_pvoid(w->seen, pdig, &prev) && prev != NULL) {
|
||||
char *slot = (char *) prev;
|
||||
char *sep = strchr(slot, '\001');
|
||||
is_revisit = 1;
|
||||
profile =
|
||||
"http://netpreserve.org/warc/1.1/revisit/identical-payload-digest";
|
||||
if (sep != NULL) {
|
||||
size_t n = (size_t) (sep - slot);
|
||||
if (n < sizeof(refers_buf)) {
|
||||
memcpy(refers_buf, slot, n);
|
||||
refers_buf[n] = '\0';
|
||||
refers_uri = refers_buf;
|
||||
refers_date = sep + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Both revisit kinds (server-304 and identical-payload-digest) are bodyless;
|
||||
only a full response carries the payload (F1). */
|
||||
emit_body = has_payload && !is_revisit;
|
||||
|
||||
/* Normalize headers: full response rewrites Content-Length to the decoded
|
||||
body length and strips Content-Encoding; a revisit keeps them (no body). */
|
||||
memset(&http, 0, sizeof(http));
|
||||
if (normalize_http_headers(resp_hdr, emit_body ? (long long) body_len : -1,
|
||||
&http) != 0) {
|
||||
wbuf_free(&http);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Response first: its id links the request via WARC-Concurrent-To. A revisit
|
||||
is a deliberate dedup, not a truncation, so tag WARC-Truncated only on a
|
||||
full (body-carrying) response. */
|
||||
resp_id[0] = '\0';
|
||||
if (warc_emit(w, is_revisit ? "revisit" : "response",
|
||||
"application/http;msgtype=response", target_uri, ip, NULL,
|
||||
refers_uri, refers_date, profile, have_pdig ? pdig : NULL,
|
||||
emit_body ? warc_truncated_reason(truncated) : NULL, 1,
|
||||
http.data, http.len, emit_body, body, body_len, body_path,
|
||||
resp_id) != 0) {
|
||||
wbuf_free(&http);
|
||||
return -1;
|
||||
}
|
||||
wbuf_free(&http);
|
||||
|
||||
if (req_hdr != NULL && req_hdr[0] != '\0') {
|
||||
size_t rlen = strlen(req_hdr);
|
||||
if (warc_emit(w, "request", "application/http;msgtype=request", target_uri,
|
||||
NULL, resp_id, NULL, NULL, NULL, NULL, NULL, 0, NULL, 0, 1,
|
||||
req_hdr, rlen, NULL, NULL) != 0)
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Record this payload for later identical-payload-digest revisits. */
|
||||
if (!is_revisit && have_pdig && w->seen != NULL && target_uri != NULL) {
|
||||
char date[32];
|
||||
char *slot;
|
||||
size_t need;
|
||||
warc_now_iso8601(date);
|
||||
need = strlen(target_uri) + 1 + strlen(date) + 1;
|
||||
slot = malloct(need);
|
||||
if (slot != NULL) {
|
||||
snprintf(slot, need, "%s\001%s", target_uri, date);
|
||||
if (coucal_write_pvoid(w->seen, pdig, slot) == 0) {
|
||||
/* replaced an existing entry: coucal freed the old value */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(void) statuscode;
|
||||
rc = 0;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ---- one non-HTTP capture ---- */
|
||||
|
||||
int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
|
||||
const char *content_type, const char *body,
|
||||
size_t body_len, const char *body_path, int truncated) {
|
||||
char pdig[33];
|
||||
int has_body = (body_len > 0 && (body != NULL || body_path != NULL));
|
||||
int have_pdig =
|
||||
has_body ? payload_digest_b32(body, body_len, body_path, pdig) : 0;
|
||||
/* resource: the block is the raw payload, its own MIME is the record's
|
||||
Content-Type, and there is no HTTP request/response envelope. */
|
||||
return warc_emit(w, "resource",
|
||||
(content_type != NULL && content_type[0] != '\0')
|
||||
? content_type
|
||||
: "application/octet-stream",
|
||||
target_uri, ip, NULL, NULL, NULL, NULL,
|
||||
have_pdig ? pdig : NULL, warc_truncated_reason(truncated), 0,
|
||||
NULL, 0, has_body, body, body_len, body_path, NULL);
|
||||
}
|
||||
|
||||
/* ---- engine emit hook ---- */
|
||||
|
||||
void warc_write_backtransaction(httrackp *opt, lien_back *back) {
|
||||
warc_writer *w;
|
||||
char uri[HTS_URLMAXSIZE * 4 + 16];
|
||||
char ip[128];
|
||||
const char *body;
|
||||
size_t body_len;
|
||||
const char *body_path;
|
||||
const char *resp_hdr;
|
||||
char synth[512];
|
||||
int is_unchanged;
|
||||
int is_ftp;
|
||||
|
||||
if (opt->state.warc == WARC_DISABLED)
|
||||
return;
|
||||
if (opt->state.warc == NULL) {
|
||||
w = warc_open(opt, StringBuff(opt->warc_file));
|
||||
if (w == NULL) {
|
||||
opt->state.warc = WARC_DISABLED;
|
||||
hts_log_print(opt, LOG_ERROR, "could not create WARC archive %s",
|
||||
StringBuff(opt->warc_file));
|
||||
return;
|
||||
}
|
||||
opt->state.warc = w;
|
||||
}
|
||||
w = (warc_writer *) opt->state.warc;
|
||||
|
||||
if (back->r.statuscode <= 0)
|
||||
return;
|
||||
|
||||
is_ftp = strfield(back->url_adr, "ftp://") != 0;
|
||||
snprintf(uri, sizeof(uri), "%s%s%s",
|
||||
link_has_authority(back->url_adr) ? "" : "http://", back->url_adr,
|
||||
back->url_fil);
|
||||
|
||||
ip[0] = '\0';
|
||||
SOCaddr_inetntoa(ip, sizeof(ip), back->r.address);
|
||||
|
||||
if (!back->r.is_write) {
|
||||
body = back->r.adr;
|
||||
body_len = (back->r.size > 0) ? (size_t) back->r.size : 0;
|
||||
body_path = NULL;
|
||||
} else {
|
||||
LLint fs;
|
||||
body = NULL;
|
||||
body_path = back->url_sav;
|
||||
fs = fsize_utf8(body_path);
|
||||
/* F4: an on-disk body past size_t (LLP32/ILP32, >4GB) would wrap the length
|
||||
used as Content-Length; drop the body rather than desync the record. */
|
||||
if (fs > 0 && (uint64_t) fs <= (uint64_t) (size_t) -1)
|
||||
body_len = (size_t) fs;
|
||||
else
|
||||
body_len = 0;
|
||||
}
|
||||
|
||||
/* FTP has no HTTP envelope: one resource record carrying the payload. */
|
||||
if (is_ftp) {
|
||||
warc_write_resource(w, uri, ip, back->r.contenttype, body, body_len,
|
||||
body_path, back->r.warc_truncated);
|
||||
return;
|
||||
}
|
||||
|
||||
is_unchanged = (back->r.notmodified && opt->is_update) ? 1 : 0;
|
||||
|
||||
/* Prefer the stashed raw headers; synthesize a minimal status line for the
|
||||
header-less (HTTP/0.9-style) responses that never carried a header block.
|
||||
*/
|
||||
resp_hdr = back->r.warc_resphdr;
|
||||
if (resp_hdr == NULL) {
|
||||
snprintf(synth, sizeof(synth), "HTTP/1.1 %d %s\r\nContent-Type: %s\r\n\r\n",
|
||||
back->r.statuscode, back->r.msg[0] ? back->r.msg : "OK",
|
||||
back->r.contenttype[0] ? back->r.contenttype
|
||||
: "application/octet-stream");
|
||||
resp_hdr = synth;
|
||||
}
|
||||
|
||||
warc_write_transaction(w, uri, ip, back->r.warc_reqhdr, resp_hdr, body,
|
||||
body_len, body_path, back->r.statuscode, is_unchanged,
|
||||
back->r.warc_truncated);
|
||||
}
|
||||
118
src/htswarc.h
118
src/htswarc.h
@@ -1,118 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* HTTrack WARC/1.1 output writer (ISO 28500). Internal, not installed.
|
||||
All WARC record serialization, gzip-member framing, digests, UUID and
|
||||
revisit dedup live here; the engine only stashes the request, frees it,
|
||||
and calls one entry point per finished transaction. */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_WARC_DEFH
|
||||
#define HTS_WARC_DEFH
|
||||
|
||||
#include "htsopt.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* opt->warc_file sentinel: --warc with no argument => auto-name the archive
|
||||
under the project's output directory at open time. */
|
||||
#define WARC_AUTONAME "\001auto"
|
||||
|
||||
/* htsblk.warc_truncated / WARC-Truncated reason tokens (ISO 28500 sec 5.13).
|
||||
A cap-truncated body is still archived, tagged with why it was cut short. */
|
||||
#define WARC_TRUNC_NONE 0
|
||||
#define WARC_TRUNC_LENGTH 1 /* hit a size cap (-M mirror / -m per-file) */
|
||||
#define WARC_TRUNC_TIME 2 /* hit the mirror time cap (-E/--max-time) */
|
||||
#define WARC_TRUNC_DISCONNECT 3 /* connection dropped mid-body */
|
||||
|
||||
/* WARC-Truncated token for a warc_truncated code, or NULL for none. */
|
||||
const char *warc_truncated_reason(int code);
|
||||
|
||||
typedef struct warc_writer warc_writer;
|
||||
|
||||
/* Stash the raw request header block (bstr.buffer) on r for the later WARC
|
||||
request record; frees any prior stash. No-op when reqhdr is NULL. */
|
||||
void warc_stash_request(htsblk *r, const char *reqhdr);
|
||||
|
||||
/* Stash the raw response header block: deletehttp frees r.headers when the
|
||||
socket closes, before back_finalize, so keep a WARC-owned copy. */
|
||||
void warc_stash_response(htsblk *r, const char *resphdr);
|
||||
|
||||
/* Free both stashed header blocks (idempotent, NULL-safe). */
|
||||
void warc_free_request(htsblk *r);
|
||||
|
||||
/* Emit the request + response (or revisit) records for one finished
|
||||
transaction. Lazily opens the writer into opt->state.warc; a no-op (logged
|
||||
once) if the archive cannot be created. */
|
||||
void warc_write_backtransaction(httrackp *opt, lien_back *back);
|
||||
|
||||
/* Close and free the writer held in opt->state.warc, if any. */
|
||||
void warc_close_opt(httrackp *opt);
|
||||
|
||||
/* --- Direct writer API (used by the hooks above and the self-test). --- */
|
||||
|
||||
/* Create the archive at path (auto-named when path is WARC_AUTONAME), writing
|
||||
the warcinfo record. .warc.gz => one gzip member per record; .warc => raw.
|
||||
Returns NULL on failure. */
|
||||
warc_writer *warc_open(httrackp *opt, const char *path);
|
||||
|
||||
/* Flush, close and free the writer (NULL-safe). */
|
||||
void warc_close(warc_writer *w);
|
||||
|
||||
/* Write one transaction's request + response (or revisit) records.
|
||||
target_uri: absolute URL fetched.
|
||||
ip: numeric peer IP, or NULL/"" to omit.
|
||||
req_hdr: exact request header block sent, or NULL to skip the request.
|
||||
resp_hdr: raw received response header block (status line + headers).
|
||||
body/body_len: decoded in-memory body, or NULL when on disk.
|
||||
body_path: file re-read for the body when body==NULL (may be NULL).
|
||||
is_update_unchanged: nonzero for a 304 server-not-modified revisit.
|
||||
truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
|
||||
Returns 0 on success, -1 on error. */
|
||||
int warc_write_transaction(warc_writer *w, const char *target_uri,
|
||||
const char *ip, const char *req_hdr,
|
||||
const char *resp_hdr, const char *body,
|
||||
size_t body_len, const char *body_path,
|
||||
int statuscode, int is_update_unchanged,
|
||||
int truncated);
|
||||
|
||||
/* Write one non-HTTP capture as a single WARC 'resource' record: the block is
|
||||
the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.
|
||||
Used for ftp:// transfers. truncated is a WARC_TRUNC_* reason or 0.
|
||||
Returns 0 on success, -1 on error. */
|
||||
int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
|
||||
const char *content_type, const char *body,
|
||||
size_t body_len, const char *body_path, int truncated);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -135,7 +135,6 @@
|
||||
<ClCompile Include="htswizard.c" />
|
||||
<ClCompile Include="htswrap.c" />
|
||||
<ClCompile Include="htszlib.c" />
|
||||
<ClCompile Include="htswarc.c" />
|
||||
<ClCompile Include="md5.c" />
|
||||
<ClCompile Include="minizip\ioapi.c" />
|
||||
<ClCompile Include="minizip\iowin32.c" />
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Keep this POSIX-portable: the harness runs it via $(BASH), which is a plain
|
||||
# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms despite the
|
||||
# #!/bin/bash above.
|
||||
|
||||
# A -%F footer whose expansion overflows the on-page buffer must be dropped, not
|
||||
# crash the crawl. Before the fix the unchecked hts_footer_format return left the
|
||||
# buffer unterminated and the next strcatbuff aborted (SIGABRT).
|
||||
|
||||
set -eu
|
||||
|
||||
# The overflow needs a path longer than Windows MAX_PATH (260), so both the
|
||||
# source tree and the mirror output blow past it there; run on POSIX only. The
|
||||
# fix itself is platform-independent and covered on Linux/macOS.
|
||||
case "$(uname -s 2>/dev/null)" in
|
||||
MINGW* | MSYS* | CYGWIN*) exit 77 ;;
|
||||
esac
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# A few-hundred-char file path (kept well under the URL length limit) makes the
|
||||
# {path} field long.
|
||||
seg=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
deep="$dir"
|
||||
i=0
|
||||
while [ "$i" -lt 6 ]; do
|
||||
deep="$deep/$seg"
|
||||
i=$((i + 1))
|
||||
done
|
||||
mkdir -p "$deep"
|
||||
printf '<html><body>hi</body></html>' >"$deep/index.html"
|
||||
|
||||
# 40 {path} references (~240 chars, under the 254-char -%F cap); expanded against
|
||||
# a few-hundred-char path this far exceeds the per-page footer buffer.
|
||||
footer=""
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
footer="$footer{path}"
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
mir="$dir/mir"
|
||||
httrack "file://$deep/index.html" -O "$mir" -%F "$footer" -q -s0 -%v0 \
|
||||
>/dev/null 2>&1 || {
|
||||
echo "crawl failed/aborted (exit $?) on an oversized footer" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The crawled page must exist (proves the URL wasn't rejected for length, so the
|
||||
# footer path ran). Look under file/, not $mir, to skip the makeindex top index.
|
||||
find "$mir/file" -name index.html | grep -q . || {
|
||||
echo "page not mirrored; the oversized-footer path was not exercised" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# -%F footer expansion (hts_footer_format via -#test=footerfmt). Fixed fields:
|
||||
# addr=host.example path=/dir/page.html date=DATE version=VER.
|
||||
ftr() {
|
||||
out="$(httrack -O /dev/null -#test=footerfmt "$1")"
|
||||
test "$out" == "$2" || {
|
||||
echo "FAIL: '$1' -> '$out' (want '$2')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Legacy positional model: a template with %s keeps the historic addr/path/date
|
||||
# order byte-for-byte (backward compatibility).
|
||||
ftr '<!-- Mirrored from %s%s by X, %s -->' \
|
||||
'<!-- Mirrored from host.example/dir/page.html by X, DATE -->'
|
||||
# Legacy arg overflow past the four fields yields the historic "???" marker.
|
||||
ftr 'a %s b %s c %s d %s e %s' 'a host.example b /dir/page.html c DATE d VER e ???'
|
||||
# Legacy %% is a literal percent; an unknown %x passes through verbatim.
|
||||
ftr '100%% %d %s' '100% %d host.example'
|
||||
|
||||
# Named model (no %s): fields substitute by name, in any order or subset.
|
||||
ftr '{addr}{path} {version} {date}' 'host.example/dir/page.html VER DATE'
|
||||
ftr 'built {date}' 'built DATE'
|
||||
# A literal char abutting a field's '}' survives (guards the index advance).
|
||||
ftr '{date}!{addr}' 'DATE!host.example'
|
||||
# {{ and }} escape to a literal brace.
|
||||
ftr 'a {{ b }} {addr}' 'a { b } host.example'
|
||||
# Unknown, empty, or unterminated braces are emitted verbatim (typos stay visible).
|
||||
ftr '{bogus} {} {addr' '{bogus} {} {addr'
|
||||
# A literal percent is untouched in named mode (no positional interpretation).
|
||||
ftr '100% {path}' '100% /dir/page.html'
|
||||
# %s anywhere forces legacy mode, so a mixed template leaves {addr} literal.
|
||||
ftr '{addr} %s' '{addr} host.example'
|
||||
|
||||
# The added named fields (url, lastmodified, mime, charset, status, size).
|
||||
ftr '{url}' 'http://host.example/dir/page.html'
|
||||
ftr 'src {lastmodified}, got {date}' 'src LASTMOD, got DATE'
|
||||
ftr '{mime}; {charset}; {status}; {size}' 'text/html; utf-8; 200; 1234'
|
||||
@@ -1,22 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# WARC/1.1 writer self-test: framing, Content-Length, gzip members, round-trip,
|
||||
# revisit dedup, plus v1.1 WARC-Truncated, ftp resource records, and
|
||||
# --warc-max-size rotation, over synthetic transactions.
|
||||
|
||||
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
|
||||
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
for t in warc warc-trunc warc-ftp warc-rotate; do
|
||||
out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")
|
||||
echo "$out"
|
||||
case "$out" in
|
||||
*": OK") ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
done
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Control for wait_bounded: a wedged fetch must be reaped at the harness deadline,
|
||||
# not hang the CI step to its 45-min cap.
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
start=$SECONDS
|
||||
out=$(CRAWL_DEADLINE=3 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
|
||||
httrack 'BASEURL/watchdog/stall' 2>&1) && rc=0 || rc=$?
|
||||
elapsed=$((SECONDS - start))
|
||||
|
||||
# Skip (not fail) on causes unrelated to the watchdog: no python3/server, or a
|
||||
# port-announce race (flaky-13).
|
||||
test "$rc" -ne 77 || {
|
||||
echo "SKIP: local crawl prerequisites missing" >&2
|
||||
exit 77
|
||||
}
|
||||
if printf '%s\n' "$out" | grep -q "could not discover server port"; then
|
||||
echo "SKIP: server port announce raced" >&2
|
||||
exit 77
|
||||
fi
|
||||
|
||||
# The message prints only on the 124 reap, and 25s < the engine's --timeout=30, so
|
||||
# together they pin the fast exit on the 3s watchdog, not httrack giving up.
|
||||
test "$rc" -ne 0 || fail "stalled crawl reported success"
|
||||
test "$elapsed" -lt 25 || fail "watchdog fired late (${elapsed}s)"
|
||||
printf '%s\n' "$out" | grep -q "watchdog fired" ||
|
||||
fail "crawl failed but not via the watchdog: $out"
|
||||
|
||||
echo "crawl watchdog OK (reaped in ${elapsed}s)"
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and an
|
||||
# --update re-crawl of an unchanged (all-304) site emits revisit records.
|
||||
# The stdlib validator (no warcio) is the real gate: it byte-compares the
|
||||
# fresh page.html response body against what the server served, asserts the
|
||||
# encoding headers were stripped and the payload digest matches, then checks
|
||||
# the update pass turned the unchanged assets into revisits (no full response).
|
||||
|
||||
set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# page.html body served by tests/local-server.py (route_mini304_page).
|
||||
export WARC_VALIDATE_BODY="page.html=3c68746d6c3e3c626f64793e74696e7920636163686561626c6520706167653c2f626f64793e3c2f68746d6c3e0a"
|
||||
export WARC_VALIDATE_NORESP="index.html page.html"
|
||||
|
||||
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun --warc-validate \
|
||||
--log-found 'no files updated' \
|
||||
--found 'mini304/index.html' --found 'mini304/page.html' \
|
||||
httrack 'BASEURL/mini304/index.html' --warc-file warc-out
|
||||
@@ -3,7 +3,7 @@
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py socks5-server.py proxy-connect-server.py \
|
||||
proxytestlib.py tls-stall-server.py warc-validate.py \
|
||||
proxytestlib.py tls-stall-server.py \
|
||||
local-crawl.sh local-server.py testlib.sh server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
@@ -38,8 +38,6 @@ TESTS = \
|
||||
01_engine-dnstimeout.test \
|
||||
01_engine-doitlog.test \
|
||||
01_engine-entities.test \
|
||||
01_engine-footerfmt.test \
|
||||
01_engine-footer-overflow.test \
|
||||
01_engine-filelist.test \
|
||||
01_engine-filter.test \
|
||||
01_engine-filterdual.test \
|
||||
@@ -80,7 +78,6 @@ TESTS = \
|
||||
01_engine-version-macros.test \
|
||||
01_engine-xfread.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_zlib-warc.test \
|
||||
01_zlib-contentcodings.test \
|
||||
01_zlib-cache.test \
|
||||
01_zlib-cache-corrupt.test \
|
||||
@@ -155,8 +152,6 @@ TESTS = \
|
||||
67_engine-delayed-truncate.test \
|
||||
68_webhttrack-outdir-charset.test \
|
||||
69_local-intl-logdir.test \
|
||||
71_local-crange-repaircache.test \
|
||||
72_watchdog-crawl.test \
|
||||
73_local-warc.test
|
||||
71_local-crange-repaircache.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -47,7 +47,6 @@ key="${testdir}/server.key"
|
||||
|
||||
tls=
|
||||
verbose=
|
||||
warc_validate=
|
||||
html_subdir=
|
||||
outdir_intl=
|
||||
rerun=
|
||||
@@ -115,8 +114,6 @@ while test "$pos" -lt "$nargs"; do
|
||||
--debug) verbose=1 ;;
|
||||
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
|
||||
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
|
||||
# validate the produced .warc.gz (see the validation block near the end)
|
||||
--warc-validate) warc_validate=1 ;;
|
||||
--no-purge)
|
||||
nopurge=1
|
||||
audit+=("--no-purge")
|
||||
@@ -186,12 +183,16 @@ debug "starting $python $server ${serverargs[*]}"
|
||||
"$python" "$(nativepath "$server")" "${serverargs[@]}" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
|
||||
# Wait for the "PORT <n>" line (server prints it once bound). A cold Python
|
||||
# start under a parallel `make check -jN` can lag past a second on a loaded Windows runner.
|
||||
# Wait for the "PORT <n>" line (server prints it once bound).
|
||||
port=
|
||||
for _ in $(seq 1 300); do
|
||||
# Match anywhere: a startup warning merged via 2>&1 could precede the PORT line.
|
||||
line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
|
||||
for _ in $(seq 1 50); do
|
||||
if test -s "$serverlog"; then
|
||||
line=$(head -n1 "$serverlog")
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || die "server exited early: $(cat "$serverlog")"
|
||||
sleep 0.1
|
||||
done
|
||||
@@ -242,17 +243,13 @@ fi
|
||||
# Localhost is fast; disable the rate/bandwidth safety limits but keep a
|
||||
# max-time backstop so a hang cannot wedge the suite.
|
||||
declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
|
||||
# Watchdog above --max-time so the engine limit fires first when healthy; only a
|
||||
# genuine wedge trips it. CRAWL_DEADLINE lets 72_watchdog-crawl drive it low.
|
||||
crawl_deadline=${CRAWL_DEADLINE:-180}
|
||||
log="${tmpdir}/log"
|
||||
info "running httrack ${hts[*]}"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline"
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
crawlpid=
|
||||
test "$crawlres" -ne 124 || warning "crawl watchdog fired after ${crawl_deadline}s"
|
||||
# httrack exits 0 even on hard connect/DNS errors, so this is a backstop only;
|
||||
# the real guard is the audit below (--errors 0 plus the host-root existence check).
|
||||
test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
|
||||
@@ -262,20 +259,13 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
|
||||
result "OK"
|
||||
grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2
|
||||
|
||||
# Snapshot the first-pass WARC before an update pass overwrites it: the fresh
|
||||
# crawl carries the full response bodies, the update pass only revisits.
|
||||
if test -n "$warc_validate"; then
|
||||
w1=$(find "$mirrorroot" -maxdepth 2 -name '*.warc.gz' 2>/dev/null | sort | tail -n1)
|
||||
test -z "$w1" || cp "$w1" "${tmpdir}/warc-pass1.gz"
|
||||
fi
|
||||
|
||||
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
|
||||
if test -n "$rerun"; then
|
||||
info "re-running httrack (update pass)"
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline"
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
crawlpid=
|
||||
test "$crawlres" -eq 0 || ! result "update pass exited $crawlres" || {
|
||||
@@ -303,7 +293,7 @@ if test -n "$rerun_args"; then
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline"
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
crawlpid=
|
||||
test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
|
||||
@@ -325,7 +315,7 @@ if test -n "$rerun_dead"; then
|
||||
httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait_bounded "$crawlpid" "$crawl_deadline" || true
|
||||
wait "$crawlpid" || true
|
||||
crawlpid=
|
||||
result "OK (dead pass ran)"
|
||||
# The dead pass must have gone through the no-data rollback, not bailed out
|
||||
@@ -360,44 +350,6 @@ done
|
||||
test -n "$hostroot" || die "could not find host root under $out"
|
||||
debug "host root: $hostroot"
|
||||
|
||||
# --- optional WARC validation (stdlib validator, no warcio) ------------------
|
||||
# WARC_VALIDATE_BODY="URLSUB=HEX" byte-checks a fresh-crawl response body;
|
||||
# WARC_VALIDATE_NORESP="URLSUB..." asserts those assets are revisits post-update.
|
||||
if test -n "$warc_validate"; then
|
||||
validator=$(nativepath "${testdir}/warc-validate.py")
|
||||
warc=$(find "$mirrorroot" -maxdepth 2 \( -name '*.warc.gz' -o -name '*.warc' \) 2>/dev/null | sort | tail -n1)
|
||||
test -n "$warc" || die "no WARC file produced under $mirrorroot"
|
||||
|
||||
# Fresh-crawl file (snapshot if an update pass overwrote it): full responses.
|
||||
fresh="${tmpdir}/warc-pass1.gz"
|
||||
test -f "$fresh" || fresh="$warc"
|
||||
declare -a bodyargs=()
|
||||
if test -n "${WARC_VALIDATE_BODY:-}"; then
|
||||
bodyargs=(--expect-body-hex "$WARC_VALIDATE_BODY")
|
||||
fi
|
||||
info "validating fresh WARC (response bodies)"
|
||||
"$python" "$validator" "$(nativepath "$fresh")" "${bodyargs[@]}" >&2 ||
|
||||
die "fresh WARC validation failed"
|
||||
result "OK"
|
||||
|
||||
# Final file: after an update pass the unchanged assets must be revisits.
|
||||
if test -n "$rerun"; then
|
||||
declare -a revargs=(--expect-revisit)
|
||||
for sub in ${WARC_VALIDATE_NORESP:-}; do
|
||||
revargs+=(--no-response-for "$sub")
|
||||
done
|
||||
info "validating update WARC (revisits)"
|
||||
"$python" "$validator" "$(nativepath "$warc")" "${revargs[@]}" >&2 ||
|
||||
die "update WARC validation failed"
|
||||
result "OK"
|
||||
fi
|
||||
|
||||
if command -v warcio >/dev/null 2>&1; then
|
||||
info "warcio check (optional)"
|
||||
if warcio check -v "$warc" >&2; then result "OK"; else die "warcio check failed"; fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# No crawl, even a cancelled one, may leave engine temporaries: .delayed (#107,
|
||||
# #483), or the .z/.u content-coding temps (#557).
|
||||
info "checking for leftover engine temporaries"
|
||||
|
||||
@@ -979,22 +979,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
# Always-stall endpoint for 72_watchdog-crawl: never finishes, so the harness
|
||||
# watchdog must reap it.
|
||||
def route_watchdog_stall(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", "1000000") # never delivered
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(b"STALL")
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# C7: stall the first GET (partial + temp-ref), then answer the resume's
|
||||
# Range with a bogus 304; httrack must drop the partial and refetch.
|
||||
RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
|
||||
@@ -1579,7 +1563,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/types/gen.php": route_types,
|
||||
"/intl/index.html": route_intl_index,
|
||||
"/intl/" + INTL_NAME: route_intl_page,
|
||||
"/watchdog/stall": route_watchdog_stall,
|
||||
"/resume/index.html": route_resume_index,
|
||||
"/resume/blob.txt": route_resume,
|
||||
"/resume304/index.html": route_resume304_index,
|
||||
|
||||
@@ -31,13 +31,15 @@ is_windows() {
|
||||
esac
|
||||
}
|
||||
|
||||
# On Windows MSYS can't signal a native python.exe, so kill_tree ends the whole
|
||||
# tree (a bare kill -9 leaves children). "|| true" throughout: callers run under
|
||||
# set -e and the reap makes wait return 143.
|
||||
# Stop a backgrounded server and reap it; MSYS cannot signal a native python.exe,
|
||||
# so only -9 lands. Every step is "|| true": callers run under set -e, and reaping
|
||||
# a server we just signalled makes wait return 143.
|
||||
stop_server() {
|
||||
test -n "${1:-}" || return 0
|
||||
kill "$1" 2>/dev/null || true
|
||||
if is_windows; then kill_tree "$1"; fi
|
||||
if is_windows; then
|
||||
kill -9 "$1" 2>/dev/null || true
|
||||
fi
|
||||
wait "$1" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
@@ -107,19 +109,3 @@ run_with_timeout() {
|
||||
done
|
||||
wait "$pid"
|
||||
}
|
||||
|
||||
# Bound an already-backgrounded crawl (pid $1) at $2s, reaping it and returning 124
|
||||
# on overrun: a wedge past --max-time would else block wait() forever and hang the CI step.
|
||||
wait_bounded() {
|
||||
local pid=$1 secs=$2 waited=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
if test "$waited" -ge "$secs"; then
|
||||
kill_tree "$pid"
|
||||
wait "$pid" 2>/dev/null || true
|
||||
return 124
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
wait "$pid"
|
||||
}
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Structural + semantic WARC/1.1 validator, Python stdlib only (no warcio):
|
||||
# walks the concatenated gzip members with zlib (gzip.decompress would fuse them
|
||||
# and lose per-record boundaries) and checks each record against the spec.
|
||||
#
|
||||
# Options:
|
||||
# --expect-revisit at least one revisit record must be present
|
||||
# --expect-body-hex SUB=HEX a response whose WARC-Target-URI contains SUB must
|
||||
# have an entity body byte-equal to bytes.fromhex(HEX),
|
||||
# no Content-Encoding/Transfer-Encoding header, and a
|
||||
# WARC-Payload-Digest matching sha1(body) when present
|
||||
# --no-response-for SUB the asset containing SUB must be a revisit: no
|
||||
# response may target it, and a revisit must
|
||||
import base64
|
||||
import hashlib
|
||||
import sys
|
||||
import zlib
|
||||
|
||||
|
||||
def records(data):
|
||||
"""Yield each record's decompressed bytes (one gzip member per record)."""
|
||||
if data[:2] != b"\x1f\x8b": # uncompressed .warc: split on the record magic
|
||||
parts = data.split(b"WARC/1.")
|
||||
for p in parts[1:]:
|
||||
yield b"WARC/1." + p
|
||||
return
|
||||
while data:
|
||||
d = zlib.decompressobj(zlib.MAX_WBITS | 16)
|
||||
block = d.decompress(data) + d.flush()
|
||||
yield block
|
||||
data = d.unused_data
|
||||
|
||||
|
||||
def field(header, name):
|
||||
for line in header.split(b"\r\n"):
|
||||
if line.lower().startswith(name.lower() + b":"):
|
||||
return line.split(b":", 1)[1].strip()
|
||||
return None
|
||||
|
||||
|
||||
def opt_values(argv, name):
|
||||
out = []
|
||||
for i, a in enumerate(argv):
|
||||
if a == name and i + 1 < len(argv):
|
||||
out.append(argv[i + 1])
|
||||
return out
|
||||
|
||||
|
||||
def check_body(rec, http_hdr, body, sub, want):
|
||||
if b"Content-Encoding" in http_hdr or b"Transfer-Encoding" in http_hdr:
|
||||
sys.exit("record for %s kept a content/transfer-encoding header" % sub)
|
||||
if body != want:
|
||||
sys.exit(
|
||||
"body mismatch for %s: got %d bytes, expected %d"
|
||||
% (sub, len(body), len(want))
|
||||
)
|
||||
pd = field(rec[: rec.find(b"\r\n\r\n")], b"WARC-Payload-Digest")
|
||||
if pd is not None and pd.startswith(b"sha1:"):
|
||||
want_b32 = base64.b32encode(hashlib.sha1(want).digest()).decode("ascii")
|
||||
if pd[5:].decode("ascii") != want_b32:
|
||||
sys.exit("WARC-Payload-Digest mismatch for %s" % sub)
|
||||
|
||||
|
||||
def main():
|
||||
argv = sys.argv[1:]
|
||||
expect_revisit = "--expect-revisit" in argv
|
||||
body_specs = [s.split("=", 1) for s in opt_values(argv, "--expect-body-hex")]
|
||||
no_resp = opt_values(argv, "--no-response-for")
|
||||
path = [a for a in argv if not a.startswith("--") and "=" not in a][0]
|
||||
data = open(path, "rb").read()
|
||||
|
||||
total = revisits = responses = infos = 0
|
||||
body_hits = {sub: False for sub, _ in body_specs}
|
||||
revisit_hits = {sub: False for sub in no_resp}
|
||||
for rec in records(data):
|
||||
total += 1
|
||||
if not rec.startswith(b"WARC/1."):
|
||||
sys.exit("record %d: bad magic %r" % (total, rec[:16]))
|
||||
sep = rec.find(b"\r\n\r\n")
|
||||
if sep < 0:
|
||||
sys.exit("record %d: no header terminator" % total)
|
||||
hdr_end = sep + 4
|
||||
header = rec[:sep]
|
||||
cl = field(header, b"Content-Length")
|
||||
if cl is None:
|
||||
sys.exit("record %d: no Content-Length" % total)
|
||||
block_len = int(cl)
|
||||
if hdr_end + block_len + 4 != len(rec):
|
||||
sys.exit(
|
||||
"record %d: Content-Length %d != block length %d"
|
||||
% (total, block_len, len(rec) - hdr_end - 4)
|
||||
)
|
||||
if rec[hdr_end + block_len :] != b"\r\n\r\n":
|
||||
sys.exit("record %d: missing \\r\\n\\r\\n trailer" % total)
|
||||
wtype = field(header, b"WARC-Type")
|
||||
uri = field(header, b"WARC-Target-URI") or b""
|
||||
if wtype == b"warcinfo":
|
||||
infos += 1
|
||||
elif wtype == b"response":
|
||||
responses += 1
|
||||
for sub in no_resp:
|
||||
if sub.encode() in uri:
|
||||
sys.exit("unexpected full response for %s (want revisit)" % sub)
|
||||
block = rec[hdr_end : hdr_end + block_len]
|
||||
bsep = block.find(b"\r\n\r\n")
|
||||
http_hdr, body = block[:bsep], block[bsep + 4 :]
|
||||
for sub, hexval in body_specs:
|
||||
if sub.encode() in uri:
|
||||
check_body(rec, http_hdr, body, sub, bytes.fromhex(hexval))
|
||||
body_hits[sub] = True
|
||||
elif wtype == b"revisit":
|
||||
revisits += 1
|
||||
for sub in no_resp:
|
||||
if sub.encode() in uri:
|
||||
revisit_hits[sub] = True
|
||||
|
||||
if total < 1:
|
||||
sys.exit("no records found")
|
||||
if infos != 1:
|
||||
sys.exit("expected exactly one warcinfo record, got %d" % infos)
|
||||
if expect_revisit and revisits < 1:
|
||||
sys.exit("expected at least one revisit record, found none")
|
||||
for sub, hit in body_hits.items():
|
||||
if not hit:
|
||||
sys.exit("no response record found for --expect-body-hex %s" % sub)
|
||||
for sub, hit in revisit_hits.items():
|
||||
if not hit:
|
||||
sys.exit("no revisit record found for unchanged asset %s" % sub)
|
||||
print(
|
||||
"warc-validate: %d records OK (%d response, %d revisit)"
|
||||
% (total, responses, revisits)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -46,12 +46,9 @@ cat >"$stubdir/x-www-browser" <<EOF
|
||||
echo "stub browser invoked with: \$1" >&2
|
||||
# Also fetch an option page and require a rendered title='' tooltip: proves the
|
||||
# option template expands and the \${html:} filter escapes into the attribute.
|
||||
# option9 additionally proves the WARC control renders with its expanded label.
|
||||
opturl="\${1%/}/server/option2.html"
|
||||
warcurl="\${1%/}/server/option9.html"
|
||||
if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html &&
|
||||
opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='" &&
|
||||
warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC; then
|
||||
opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='"; then
|
||||
echo PASS >"$marker"
|
||||
else
|
||||
echo "FAIL: unexpected response from \$1" >"$marker"
|
||||
|
||||
Reference in New Issue
Block a user