WARC: file the --warc help under Log/index/cache, not Build (#679)

WARC is a transaction-level archive of what was fetched (a sibling of the
hts-cache), not a browsable-mirror build format like MHTML, so it belongs in
the Log/index/cache group next to the cache options. Help text only; the
webhttrack GUI already places it on the Log/Index/Cache tab (option9).

Signed-off-by: Xavier Roche <xroche@gmail.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

html/httrack.man.html

@@ -90,35 +90,34 @@ offline browser : copy websites to a local directory</p>
 ] [ <b>-NN, --structure[=N]</b> ] [ <b>-%N,
 --delayed-type-check</b> ] [ <b>-%D,
 --cached-delayed-type-check</b> ] [ <b>-%M, --mime-html</b>
-] [ <b>-%r, --warc</b> ] [ <b>-LN, --long-names[=N]</b> ] [
-<b>-KN, --keep-links[=N]</b> ] [ <b>-x,
---replace-external</b> ] [ <b>-%x, --disable-passwords</b> ]
-[ <b>-%q, --include-query-string</b> ] [ <b>-%g,
---strip-query</b> ] [ <b>-o, --generate-errors</b> ] [
-<b>-X, --purge-old[=N]</b> ] [ <b>-%p, --preserve</b> ] [
-<b>-%T, --utf8-conversion</b> ] [ <b>-bN, --cookies[=N]</b>
-] [ <b>-%K, --cookies-file</b> ] [ <b>-%Y, --why</b> ] [
-<b>-u, --check-type[=N]</b> ] [ <b>-j, --parse-java[=N]</b>
-] [ <b>-sN, --robots[=N]</b> ] [ <b>-%h, --http-10</b> ] [
-<b>-%k, --keep-alive</b> ] [ <b>-%z,
---disable-compression</b> ] [ <b>-%B, --tolerant</b> ] [
-<b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
+] [ <b>-LN, --long-names[=N]</b> ] [ <b>-KN,
+--keep-links[=N]</b> ] [ <b>-x, --replace-external</b> ] [
+<b>-%x, --disable-passwords</b> ] [ <b>-%q,
+--include-query-string</b> ] [ <b>-%g, --strip-query</b> ] [
+<b>-o, --generate-errors</b> ] [ <b>-X, --purge-old[=N]</b>
+] [ <b>-%p, --preserve</b> ] [ <b>-%T, --utf8-conversion</b>
+] [ <b>-bN, --cookies[=N]</b> ] [ <b>-%K, --cookies-file</b>
+] [ <b>-%Y, --why</b> ] [ <b>-u, --check-type[=N]</b> ] [
+<b>-j, --parse-java[=N]</b> ] [ <b>-sN, --robots[=N]</b> ] [
+<b>-%h, --http-10</b> ] [ <b>-%k, --keep-alive</b> ] [
+<b>-%z, --disable-compression</b> ] [ <b>-%B, --tolerant</b>
+] [ <b>-%s, --updatehack</b> ] [ <b>-%u, --urlhack</b> ] [
 <b>-%A, --assume</b> ] [ <b>-@iN, --protocol[=N]</b> ] [
 <b>-%w, --disable-module</b> ] [ <b>-F, --user-agent</b> ] [
 <b>-%R, --referer</b> ] [ <b>-%E, --from</b> ] [ <b>-%F,
 --footer</b> ] [ <b>-%l, --language</b> ] [ <b>-%a,
 --accept</b> ] [ <b>-%X, --headers</b> ] [ <b>-C,
 --cache[=N]</b> ] [ <b>-k, --store-all-in-cache</b> ] [
-<b>-%n, --do-not-recatch</b> ] [ <b>-%v, --display</b> ] [
-<b>-Q, --do-not-log</b> ] [ <b>-q, --quiet</b> ] [ <b>-z,
---extra-log</b> ] [ <b>-Z, --debug-log</b> ] [ <b>-v,
---verbose</b> ] [ <b>-f, --file-log</b> ] [ <b>-f2,
---single-log</b> ] [ <b>-I, --index</b> ] [ <b>-%i,
---build-top-index</b> ] [ <b>-%I, --search-index</b> ] [
-<b>-pN, --priority[=N]</b> ] [ <b>-S, --stay-on-same-dir</b>
-] [ <b>-D, --can-go-down</b> ] [ <b>-U, --can-go-up</b> ] [
-<b>-B, --can-go-up-and-down</b> ] [ <b>-a,
---stay-on-same-address</b> ] [ <b>-d,
+<b>-%r, --warc</b> ] [ <b>-%n, --do-not-recatch</b> ] [
+<b>-%v, --display</b> ] [ <b>-Q, --do-not-log</b> ] [ <b>-q,
+--quiet</b> ] [ <b>-z, --extra-log</b> ] [ <b>-Z,
+--debug-log</b> ] [ <b>-v, --verbose</b> ] [ <b>-f,
+--file-log</b> ] [ <b>-f2, --single-log</b> ] [ <b>-I,
+--index</b> ] [ <b>-%i, --build-top-index</b> ] [ <b>-%I,
+--search-index</b> ] [ <b>-pN, --priority[=N]</b> ] [ <b>-S,
+--stay-on-same-dir</b> ] [ <b>-D, --can-go-down</b> ] [
+<b>-U, --can-go-up</b> ] [ <b>-B, --can-go-up-and-down</b> ]
+[ <b>-a, --stay-on-same-address</b> ] [ <b>-d,
 --stay-on-same-domain</b> ] [ <b>-l, --stay-on-same-tld</b>
 ] [ <b>-e, --go-everywhere</b> ] [ <b>-%H,
 --debug-headers</b> ] [ <b>-%!,
@@ -649,18 +648,6 @@ don&rsquo;t wait) (--cached-delayed-type-check)</p></td></tr>
 <td width="4%">
 
 
-<p>-%r</p></td>
-<td width="5%"></td>
-<td width="82%">
-
-
-<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
-sets the output name (--warc)</p></td></tr>
-<tr valign="top" align="left">
-<td width="9%"></td>
-<td width="4%">
-
-
 <p>-%t</p></td>
 <td width="5%"></td>
 <td width="82%">
@@ -1142,6 +1129,20 @@ update before) (--cache[=N])</p></td></tr>
 <td width="4%">
 
 
+<p>-%r</p></td>
+<td width="5%"></td>
+<td width="82%">
+
+
+<p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
+sets the output name, --warc-max-size N rotates segments
+past N bytes, --warc-cdx also writes a sorted CDXJ index,
+--wacz packages it all as a WACZ file (--warc)</p></td></tr>
+<tr valign="top" align="left">
+<td width="9%"></td>
+<td width="4%">
+
+
 <p>-%n</p></td>
 <td width="5%"></td>
 <td width="82%">

html/server/option9.html

@@ -103,6 +103,17 @@ ${do:end-if}
 > ${LANG_I61}
 <br><br>
 
+<input type="checkbox" name="warc" ${checked:warc}
+			title='${html:LANG_WARCTIP}' onMouseOver="info('${html:LANG_WARCTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+> ${LANG_WARC}
+<br><br>
+
+${LANG_WARCFILE}
+<input name="warcfile" value="${warcfile}" size="40"
+			title='${html:LANG_WARCFILETIP}' onMouseOver="info('${html:LANG_WARCFILETIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+>
+<br><br>
+
 <input type="checkbox" name="norecatch" ${checked:norecatch}
 			title='${html:LANG_I5b}' onMouseOver="info('${html:LANG_I5b}'); return true" onMouseOut="info('&nbsp;'); return true"
 > ${LANG_I34b}

html/server/step2.html

@@ -141,6 +141,8 @@ ${do:copy:KeepSlashes:keepslashes}
 ${do:copy:KeepQueryOrder:keepqueryorder}
 ${do:copy:StripQuery:stripquery}
 ${do:copy:StoreAllInCache:cache2}
+${do:copy:Warc:warc}
+${do:copy:WarcFile:warcfile}
 ${do:copy:LogType:logtype}
 ${do:copy:UseHTTPProxyForFTP:ftpprox}
 ${do:copy:ProxyType:proxytype}

html/server/step4.html

@@ -187,6 +187,8 @@ ${do:end-if}
 	${test:toler:--tolerant}
 	${test:http10:--http-10}
 	${test:cache2:--store-all-in-cache}
+	${test:warc:--warc}
+	${test:warcfile:--warc-file "}${html:warcfile}${test:warcfile:"}
 	${test:norecatch:--do-not-recatch}
 	${test:logf:--single-log}
 	${test:logtype:::--extra-log:--debug-log}
@@ -237,6 +239,8 @@ KeepSlashes=${ztest:keepslashes:0:1}
 KeepQueryOrder=${ztest:keepqueryorder:0:1}
 StripQuery=${stripquery}
 StoreAllInCache=${ztest:cache2:0:1}
+Warc=${ztest:warc:0:1}
+WarcFile=${warcfile}
 LogType=${logtype}
 UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
 ProxyType=${proxytype}

lang.def

@@ -1034,3 +1034,11 @@ LANG_STRIPQUERY
 Strip query keys:
 LANG_STRIPQUERYTIP
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
+LANG_WARC
+Write a WARC archive of the crawl
+LANG_WARCTIP
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+LANG_WARCFILE
+WARC archive name:
+LANG_WARCFILETIP
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.

lang/Bulgarian.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Ïðåìàõâàíå íà êëþ÷îâå îò çàÿâêàòà:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Êëþ÷îâå îò çàÿâêàòà, ðàçäåëåíè ñúñ çàïåòàÿ, êîèòî äà ñå ïðåìàõíàò îò èìåòî íà çàïèñàíèÿ ôàéë (íàïðèìåð sid,utm_source).
+Write a WARC archive of the crawl
+Çàïèñâàíå íà WARC àðõèâ íà îáõîæäàíåòî
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Çàïèñâàíå íà âñåêè èçòåãëåí îòãîâîð è â WARC/1.1 àðõèâ ïî ISO-28500, äî îãëåäàëîòî.
+WARC archive name:
+Èìå íà WARC àðõèâà:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Íåçàäúëæèòåëíî áàçîâî èìå çà WARC àðõèâà; îñòàâåòå ïðàçíî çà àâòîìàòè÷íî èìåíóâàíå â èçõîäíàòà äèðåêòîðèÿ.

lang/Castellano.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Eliminar claves de query string:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Claves de query string, separadas por comas, que se eliminarán del nombre de los archivos guardados (p. ej. sid,utm_source).
+Write a WARC archive of the crawl
+Escribir un archivo WARC del rastreo
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Guardar también cada respuesta descargada en un archivo WARC/1.1 ISO-28500, junto a la réplica.
+WARC archive name:
+Nombre del archivo WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Nombre base opcional para el archivo WARC; déjelo en blanco para nombrarlo automáticamente en el directorio de salida.

lang/Cesky.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Odebrat klíèe dotazu:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Klíèe dotazu oddìlené èárkami, které se vynechají z pojmenování ukládaných souborù (napø. sid,utm_source).
+Write a WARC archive of the crawl
+Zapsat archiv WARC z procházení
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Uložit také každou staženou odpovìï do archivu WARC/1.1 podle ISO-28500 vedle zrcadla.
+WARC archive name:
+Název archivu WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Volitelný základní název archivu WARC; ponechte prázdné pro automatické pojmenování ve výstupním adresáøi.

lang/Chinese-BIG5.txt

@@ -956,3 +956,11 @@ Strip query keys:
 移除查詢鍵:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 以逗號分隔的查詢鍵，將其從儲存檔案的命名中移除 (例如 sid,utm_source)。
+Write a WARC archive of the crawl
+寫入此次抓取的 WARC 封存檔
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+同時將每個已擷取的回應儲存為 ISO-28500 WARC/1.1 封存檔，置於鏡像網站旁。
+WARC archive name:
+WARC 封存檔名稱：
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+WARC 封存檔的選用基本名稱；留空則於輸出目錄中自動命名。

lang/Chinese-Simplified.txt

@@ -956,3 +956,11 @@ Strip query keys:
 剥离查询键:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 用逗号分隔的查询键，将其从保存文件的命名中删除 (例如 sid,utm_source)。
+Write a WARC archive of the crawl
+写入本次抓取的 WARC 归档
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+同时将每个已获取的响应保存为 ISO-28500 WARC/1.1 归档，置于镜像站点旁边。
+WARC archive name:
+WARC 归档名称：
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+WARC 归档的可选基本名称；留空则在输出目录中自动命名。

lang/Croatian.txt

@@ -958,3 +958,11 @@ Strip query keys:
 Ukloniti kljuèeve upita:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Zarezom odvojeni kljuèevi upita koji se izostavljaju iz naziva spremljenih datoteka (npr. sid,utm_source).
+Write a WARC archive of the crawl
+Zapi¹i WARC arhivu obilaska
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Spremi i svaki preuzeti odgovor u ISO-28500 WARC/1.1 arhivu, uz zrcalo.
+WARC archive name:
+Naziv WARC arhive:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Neobavezni osnovni naziv WARC arhive; ostavite prazno za automatsko imenovanje u izlaznom direktoriju.

lang/Dansk.txt

@@ -1004,3 +1004,11 @@ Strip query keys:
 Fjern forespørgselsnøgler:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Kommaseparerede forespørgselsnøgler, der udelades i navngivningen af gemte filer (f.eks. sid,utm_source).
+Write a WARC archive of the crawl
+Skriv et WARC-arkiv af gennemsøgningen
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Gem også hvert hentet svar i et ISO-28500 WARC/1.1-arkiv ved siden af spejlet.
+WARC archive name:
+Navn på WARC-arkiv:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Valgfrit basisnavn til WARC-arkivet; lad feltet stå tomt for automatisk navngivning i outputmappen.

lang/Deutsch.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Query-Schlüssel entfernen:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Kommagetrennte Query-Schlüssel, die bei der Benennung gespeicherter Dateien entfallen (z. B. sid,utm_source).
+Write a WARC archive of the crawl
+WARC-Archiv des Crawls schreiben
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Jede heruntergeladene Antwort zusätzlich in einem ISO-28500-WARC/1.1-Archiv neben dem Spiegel speichern.
+WARC archive name:
+Name des WARC-Archivs:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Optionaler Basisname für das WARC-Archiv; leer lassen, um es automatisch im Ausgabeverzeichnis zu benennen.

lang/Eesti.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Eemalda päringuvõtmed:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Komadega eraldatud päringuvõtmed, mis jäetakse salvestatud faili nimest välja (nt sid,utm_source).
+Write a WARC archive of the crawl
+Kirjuta läbimise WARC-arhiiv
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Salvesta iga alla laaditud vastus ka ISO-28500 WARC/1.1 arhiivi peegli kõrvale.
+WARC archive name:
+WARC-arhiivi nimi:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+WARC-arhiivi valikuline põhinimi; jäta tühjaks, et see väljundkataloogis automaatselt nimetada.

lang/English.txt

@@ -1004,3 +1004,11 @@ Strip query keys:
 Strip query keys:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
+Write a WARC archive of the crawl
+Write a WARC archive of the crawl
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+WARC archive name:
+WARC archive name:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.

lang/Finnish.txt

@@ -958,3 +958,11 @@ Strip query keys:
 Poista kyselyavaimet:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Pilkuin erotellut kyselyavaimet, jotka jätetään pois tallennettujen tiedostojen nimeämisestä (esim. sid,utm_source).
+Write a WARC archive of the crawl
+Kirjoita imuroinnin WARC-arkisto
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Tallenna myös jokainen noudettu vastaus ISO-28500 WARC/1.1 -arkistoon peilin viereen.
+WARC archive name:
+WARC-arkiston nimi:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Valinnainen WARC-arkiston perusnimi; jätä tyhjäksi, jotta se nimetään automaattisesti tulostehakemistoon.

lang/Francais.txt

@@ -1004,3 +1004,11 @@ Strip query keys:
 Supprimer les clés de query string :
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Clés de query string à retirer du nommage des fichiers enregistrés, séparées par des virgules (par ex. sid,utm_source).
+Write a WARC archive of the crawl
+Écrire une archive WARC du crawl
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Enregistrer aussi chaque réponse téléchargée dans une archive WARC/1.1 (ISO-28500), à côté du miroir.
+WARC archive name:
+Nom de l'archive WARC :
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Nom de base optionnel pour l'archive WARC ; laissez vide pour le générer automatiquement dans le répertoire de sortie.

lang/Greek.txt

@@ -958,3 +958,11 @@ Strip query keys:
 Αφαίρεση κλειδιών ερωτήματος:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Κλειδιά ερωτήματος χωρισμένα με κόμμα, που θα αφαιρεθούν από την ονομασία των αποθηκευμένων αρχείων (π.χ. sid,utm_source).
+Write a WARC archive of the crawl
+Εγγραφή αρχείου WARC της ανίχνευσης
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Αποθήκευση κάθε ληφθείσας απόκρισης και σε αρχείο WARC/1.1 ISO-28500, δίπλα στο είδωλο.
+WARC archive name:
+Όνομα αρχείου WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Προαιρετικό βασικό όνομα για το αρχείο WARC. Αφήστε το κενό για αυτόματη ονομασία στον κατάλογο εξόδου.

lang/Italiano.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Rimuovi chiavi della query string:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Chiavi della query string, separate da virgole, da rimuovere dai nomi dei file salvati (ad es. sid,utm_source).
+Write a WARC archive of the crawl
+Scrivi un archivio WARC della scansione
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Salva anche ogni risposta scaricata in un archivio WARC/1.1 ISO-28500, accanto al mirror.
+WARC archive name:
+Nome dell'archivio WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Nome di base facoltativo per l'archivio WARC; lascia vuoto per assegnarlo automaticamente nella directory di output.

lang/Japanese.txt

@@ -956,3 +956,11 @@ Strip query keys:
 削除するクエリキー:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 保存ファイル名の生成から除外するクエリキーをカンマ区切りで指定します (例: sid,utm_source)。
+Write a WARC archive of the crawl
+クロールの WARC アーカイブを書き出す
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+取得した各レスポンスを ISO-28500 WARC/1.1 アーカイブとしてミラーの隣にも保存します。
+WARC archive name:
+WARC アーカイブ名:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+WARC アーカイブの任意のベース名。空欄にすると出力ディレクトリ内で自動的に名前が付けられます。

lang/Macedonian.txt

@@ -956,3 +956,11 @@ Strip query keys:
 ¾âáâàÐÝØ ÚÛãçÕÒØ ÞÔ ÑÐàÐúÕâÞ:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 ºÛãçÕÒØ ÞÔ ÑÐàÐúÕâÞ, ÞÔÔÕÛÕÝØ áÞ ×ÐßØàÚÐ, èâÞ áÕ ÞâáâàÐÝãÒÐÐâ ÞÔ ØÜÕâÞ ÝÐ ×ÐçãÒÐÝÐâÐ ÔÐâÞâÕÚÐ (ÝÐ ßàØÜÕà sid,utm_source).
+Write a WARC archive of the crawl
+·ÐßØèØ WARC ÐàåØÒÐ ÝÐ ßàÕÑÐàãÒÐúÕâÞ
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+·ÐçãÒÐø ÓÞ áÕÚÞø ßàÕ×ÕÜÕÝ ÞÔÓÞÒÞà Ø ÒÞ ISO-28500 WARC/1.1 ÐàåØÒÐ, ßÞÚàÐø ÞÓÛÕÔÐÛÞâÞ.
+WARC archive name:
+¸ÜÕ ÝÐ WARC ÐàåØÒÐâÐ:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+¸×ÑÞàÝÞ ÞáÝÞÒÝÞ ØÜÕ ×Ð WARC ÐàåØÒÐâÐ; ÞáâÐÒÕâÕ ßàÐ×ÝÞ ×Ð ÐÒâÞÜÐâáÚÞ ØÜÕÝãÒÐúÕ ÒÞ Ø×ÛÕ×ÝØÞâ ÔØàÕÚâÞàØãÜ.

lang/Magyar.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Lekérdezési kulcsok eltávolítása:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Vesszõvel elválasztott lekérdezési kulcsok, amelyeket el kell hagyni a mentett fájl elnevezésébõl (pl. sid,utm_source).
+Write a WARC archive of the crawl
+A bejárás WARC archívumának írása
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Minden letöltött válasz mentése ISO-28500 WARC/1.1 archívumba is, a tükör mellé.
+WARC archive name:
+WARC archívum neve:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+A WARC archívum opcionális alapneve; hagyja üresen az automatikus elnevezéshez a kimeneti könyvtárban.

lang/Nederlands.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Query-sleutels verwijderen:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Door komma's gescheiden query-sleutels die bij het benoemen van opgeslagen bestanden worden weggelaten (bijv. sid,utm_source).
+Write a WARC archive of the crawl
+Schrijf een WARC-archief van de crawl
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Sla ook elke opgehaalde respons op in een ISO-28500 WARC/1.1-archief, naast de mirror.
+WARC archive name:
+Naam van WARC-archief:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Optionele basisnaam voor het WARC-archief; laat leeg om het automatisch een naam te geven in de uitvoermap.

lang/Norsk.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Fjern spørrenøkler:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Kommaseparerte spørrenøkler som utelates i navngivingen av lagrede filer (f.eks. sid,utm_source).
+Write a WARC archive of the crawl
+Skriv et WARC-arkiv av gjennomgangen
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Lagre også hvert nedlastet svar i et ISO-28500 WARC/1.1-arkiv ved siden av speilet.
+WARC archive name:
+Navn på WARC-arkiv:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Valgfritt basisnavn for WARC-arkivet; la feltet stå tomt for automatisk navngivning i utdatamappen.

lang/Polski.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Usuñ klucze zapytania:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Rozdzielone przecinkami klucze zapytania pomijane przy nazywaniu zapisanych plików (np. sid,utm_source).
+Write a WARC archive of the crawl
+Zapisz archiwum WARC z indeksowania
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Zapisz te¿ ka¿d± pobran± odpowied¼ do archiwum WARC/1.1 ISO-28500, obok kopii lustrzanej.
+WARC archive name:
+Nazwa archiwum WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Opcjonalna nazwa bazowa archiwum WARC; pozostaw puste, aby nazwaæ je automatycznie w katalogu wyj¶ciowym.

lang/Portugues-Brasil.txt

@@ -1004,3 +1004,11 @@ Strip query keys:
 Remover chaves da query string:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Chaves da query string, separadas por vírgulas, a serem removidas da nomeação dos arquivos salvos (ex.: sid,utm_source).
+Write a WARC archive of the crawl
+Gravar um arquivo WARC do rastreamento
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Salvar também cada resposta baixada em um arquivo WARC/1.1 ISO-28500, ao lado do espelho.
+WARC archive name:
+Nome do arquivo WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Nome base opcional para o arquivo WARC; deixe em branco para nomeá-lo automaticamente no diretório de saída.

lang/Portugues.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Remover chaves da query string:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Chaves da query string, separadas por vírgulas, a remover da nomeação dos ficheiros guardados (por ex. sid,utm_source).
+Write a WARC archive of the crawl
+Escrever um arquivo WARC do rastreio
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Guardar também cada resposta transferida num arquivo WARC/1.1 ISO-28500, ao lado do espelho.
+WARC archive name:
+Nome do arquivo WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Nome base opcional para o arquivo WARC; deixe em branco para o nomear automaticamente no diretório de saída.

lang/Romanian.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Elimina cheile din query string:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Chei din query string, separate prin virgula, de eliminat din denumirea fisierelor salvate (de ex. sid,utm_source).
+Write a WARC archive of the crawl
+Scrie o arhiva WARC a parcurgerii
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Salveaza si fiecare raspuns descarcat intr-o arhiva WARC/1.1 ISO-28500, langa oglinda.
+WARC archive name:
+Numele arhivei WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Nume de baza optional pentru arhiva WARC; lasati gol pentru a-l denumi automat in directorul de iesire.

lang/Russian.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Óäàëÿòü êëþ÷è çàïðîñà:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Êëþ÷è çàïðîñà ÷åðåç çàïÿòóþ, óäàëÿåìûå èç èìåíè ñîõðàíÿåìîãî ôàéëà (íàïðèìåð, sid,utm_source).
+Write a WARC archive of the crawl
+Çàïèñàòü WARC-àðõèâ îáõîäà
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Ñîõðàíÿòü êàæäûé çàãðóæåííûé îòâåò òàêæå â àðõèâ WARC/1.1 ISO-28500 ðÿäîì ñ çåðêàëîì.
+WARC archive name:
+Èìÿ WARC-àðõèâà:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Íåîáÿçàòåëüíîå áàçîâîå èìÿ WARC-àðõèâà; îñòàâüòå ïóñòûì äëÿ àâòîìàòè÷åñêîãî èìåíîâàíèÿ â âûõîäíîì êàòàëîãå.

lang/Slovak.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Odstráni» kµúèe dotazu:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Kµúèe dotazu oddelené èiarkami, ktoré sa vynechajú z pomenovania ukladaných súborov (napr. sid,utm_source).
+Write a WARC archive of the crawl
+Zapísa» archív WARC z prehµadávania
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Ulo¾i» aj ka¾dú stiahnutú odpoveï do archívu WARC/1.1 ISO-28500 vedµa zrkadla.
+WARC archive name:
+Názov archívu WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Voliteµný základný názov archívu WARC; ponechajte prázdne pre automatické pomenovanie vo výstupnom adresári.

lang/Slovenian.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Odstrani kljuce poizvedbe:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Z vejicami loceni kljuci poizvedbe, ki se izpustijo pri poimenovanju shranjenih datotek (npr. sid,utm_source).
+Write a WARC archive of the crawl
+Zapisi arhiv WARC iz pregledovanja
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Shrani tudi vsak preneseni odgovor v arhiv WARC/1.1 ISO-28500 poleg zrcala.
+WARC archive name:
+Ime arhiva WARC:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Neobvezno osnovno ime arhiva WARC; pustite prazno za samodejno poimenovanje v izhodni mapi.

lang/Svenska.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Ta bort frågenycklar:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Kommaseparerade frågenycklar som utelämnas vid namngivningen av sparade filer (t.ex. sid,utm_source).
+Write a WARC archive of the crawl
+Skriv ett WARC-arkiv av genomsökningen
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Spara även varje hämtat svar i ett ISO-28500 WARC/1.1-arkiv, bredvid spegeln.
+WARC archive name:
+WARC-arkivets namn:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Valfritt basnamn för WARC-arkivet; lämna tomt för att namnge det automatiskt i utdatakatalogen.

lang/Turkish.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Sorgu anahtarlarýný çýkar:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Kaydedilen dosya adlandýrmasýndan çýkarýlacak, virgülle ayrýlmýþ sorgu anahtarlarý (örn. sid,utm_source).
+Write a WARC archive of the crawl
+Taramanýn WARC arþivini yaz
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Ýndirilen her yanýtý ayrýca aynanýn yanýna bir ISO-28500 WARC/1.1 arþivine kaydet.
+WARC archive name:
+WARC arþivi adý:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+WARC arþivi için isteðe baðlý temel ad; çýktý dizininde otomatik adlandýrma için boþ býrakýn.

lang/Ukrainian.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Âèëó÷àòè êëþ÷³ çàïèòó:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Êëþ÷³ çàïèòó ÷åðåç êîìó, ÿê³ âèëó÷àþòüñÿ ç ³ìåí³ çáåðåæåíîãî ôàéëó (íàïðèêëàä, sid,utm_source).
+Write a WARC archive of the crawl
+Çàïèñàòè WARC-àðõ³â îáõîäó
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Çáåð³ãàòè êîæíó çàâàíòàæåíó â³äïîâ³äü òàêîæ ó àðõ³â WARC/1.1 ISO-28500 ïîðÿä ³ç äçåðêàëîì.
+WARC archive name:
+²ì'ÿ WARC-àðõ³âó:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+Íåîáîâ'ÿçêîâà áàçîâà íàçâà WARC-àðõ³âó; çàëèøòå ïîðîæí³ì äëÿ àâòîìàòè÷íîãî íàéìåíóâàííÿ ó âèõ³äíîìó êàòàëîç³.

lang/Uzbek.txt

@@ -956,3 +956,11 @@ Strip query keys:
 Olib tashlanadigan so’rov kalitlari:
 Comma-separated query keys to drop from the saved-file naming (e.g. sid,utm_source).
 Saqlangan fayl nomidan olib tashlanadigan, vergul bilan ajratilgan so’rov kalitlari (masalan, sid,utm_source).
+Write a WARC archive of the crawl
+Qidiruvning WARC arxivini yozish
+Also save every fetched response into an ISO-28500 WARC/1.1 archive, next to the mirror.
+Har bir yuklab olingan javobni ISO-28500 WARC/1.1 arxiviga ham, ko'zgu yonida saqlash.
+WARC archive name:
+WARC arxivi nomi:
+Optional base name for the WARC archive; leave blank to auto-name it under the output directory.
+WARC arxivi uchun ixtiyoriy asosiy nom; chiqish katalogida avtomatik nomlash uchun bo'sh qoldiring.

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "22 July 2026" "httrack website copier"
+.TH httrack 1 "23 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -40,7 +40,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%N, \-\-delayed\-type\-check\fR ]
 [ \fB\-%D, \-\-cached\-delayed\-type\-check\fR ]
 [ \fB\-%M, \-\-mime\-html\fR ]
-[ \fB\-%r, \-\-warc\fR ]
 [ \fB\-LN, \-\-long\-names[=N]\fR ]
 [ \fB\-KN, \-\-keep\-links[=N]\fR ]
 [ \fB\-x, \-\-replace\-external\fR ]
@@ -75,6 +74,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%X, \-\-headers\fR ]
 [ \fB\-C, \-\-cache[=N]\fR ]
 [ \fB\-k, \-\-store\-all\-in\-cache\fR ]
+[ \fB\-%r, \-\-warc\fR ]
 [ \fB\-%n, \-\-do\-not\-recatch\fR ]
 [ \fB\-%v, \-\-display\fR ]
 [ \fB\-Q, \-\-do\-not\-log\fR ]
@@ -198,8 +198,6 @@ delayed type check, don't make any link test but wait for files download to star
 cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait) (\-\-cached\-delayed\-type\-check)
 .IP \-%M
 generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
-.IP \-%r
-write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name (\-\-warc)
 .IP \-%t
 keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
 .IP \-LN
@@ -279,6 +277,8 @@ additional HTTP header line (\-%X "X\-Magic: 42" (\-\-headers <param>)
 create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before) (\-\-cache[=N])
 .IP \-k
 store all files in cache (not useful if files on disk) (\-\-store\-all\-in\-cache)
+.IP \-%r
+write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name, \-\-warc\-max\-size N rotates segments past N bytes, \-\-warc\-cdx also writes a sorted CDXJ index, \-\-wacz packages it all as a WACZ file (\-\-warc)
 .IP \-%n
 do not re\-download locally erased files (\-\-do\-not\-recatch)
 .IP \-%v

src/htsalias.c

@@ -116,6 +116,13 @@ const char *hts_optalias[][4] = {
    "load extra cookies from a Netscape cookies.txt"},
   {"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
   {"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
+  {"warc-max-size", "-%rs", "param1",
+   "rotate the WARC archive once a segment passes N bytes (0: single file)"},
+  {"warc-cdx", "-%rc", "single",
+   "write a sorted CDXJ index next to the WARC archive"},
+  {"warc-cdxj", "-%rc", "single", ""},
+  {"wacz", "-%rz", "single",
+   "package the WARC archive, CDXJ index and pages as a WACZ file"},
   {"why", "-%Y", "param1",
    "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",

src/htsback.c

@@ -750,9 +750,20 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                     fexist_utf8(back[p].url_sav))
                   filenote(&opt->state.strc, back[p].url_sav, NULL);
               }
+              /* Keep the compressed spool so the WARC record stores the body
+                 verbatim (Content-Encoding preserved) instead of unlinking it.
+               */
+              if (StringNotEmpty(opt->warc_file)) {
+                warc_adopt_rawspool(&back[p].r, back[p].tmpfile);
+                if (back[p].r.warc_rawpath != NULL)
+                  back[p].tmpfile =
+                      NULL; /* adopted: freed via warc_free_request */
+              }
               /* ensure that no remaining temporary file exists */
-              unlink(back[p].tmpfile);
-              back[p].tmpfile = NULL;
+              if (back[p].tmpfile != NULL) {
+                unlink(back[p].tmpfile);
+                back[p].tmpfile = NULL;
+              }
             }
             // stats
             HTS_STAT.total_packed += back[p].compressed_size;
@@ -1062,6 +1073,9 @@ void back_copy_static(const lien_back * src, lien_back * dst) {
   dst->r.headers = NULL;
   dst->r.warc_reqhdr = NULL;
   dst->r.warc_resphdr = NULL;
+  dst->r.warc_rawpath =
+      NULL; /* the spool stays owned by src (no double-unlink) */
+  dst->r.warc_truncated = 0;
   dst->r.out = NULL;
   dst->r.location = dst->location_buffer;
   dst->r.fp = NULL;
@@ -1127,6 +1141,8 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.out = NULL;
     (*dst)->r.warc_reqhdr = NULL;
     (*dst)->r.warc_resphdr = NULL;
+    (*dst)->r.warc_rawpath = NULL;
+    (*dst)->r.warc_truncated = 0;
     (*dst)->r.location = (*dst)->location_buffer;
     (*dst)->r.fp = NULL;
     (*dst)->r.soc = INVALID_SOCKET;
@@ -2519,6 +2535,19 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     for (i = 0; i < (unsigned int) back_max; i++) {
       if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        /* A cap-truncated body is deliberate, not broken: archive what arrived
+           with WARC-Truncated before the abort overwrites the slot's real 2xx
+           status. HTTrack still treats the slot as incomplete afterwards. */
+        if (StringNotEmpty(opt->warc_file) && back[i].r.statuscode > 0 &&
+            back[i].r.warc_resphdr != NULL && back[i].r.size > 0 &&
+            !(back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))) {
+          if (back[i].r.is_write && back[i].r.out != NULL)
+            fflush(back[i].r.out);
+          back[i].r.warc_truncated = (limit == HTS_MIRROR_LIMIT_SIZE)
+                                         ? WARC_TRUNC_LENGTH
+                                         : WARC_TRUNC_TIME;
+          warc_write_backtransaction(opt, &back[i]);
+        }
         if (back[i].r.soc != INVALID_SOCKET) {
           deletehttp(&back[i].r);
         }

src/htscore.c

@@ -3632,6 +3632,9 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
 
   if (StringNotEmpty(from->warc_file))
     StringCopyS(to->warc_file, from->warc_file);
+  to->warc_max_size = from->warc_max_size;
+  to->warc_cdx = from->warc_cdx;
+  to->warc_wacz = from->warc_wacz;
 
   if (from->pause_max_ms > 0) {
     to->pause_min_ms = from->pause_min_ms;

src/htscoremain.c

@@ -1806,6 +1806,34 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     return -1;
                   }
                   StringCopy(opt->warc_file, argv[na]);
+                } else if (*(com + 1) == 's') { // --warc-max-size N: rotation
+                  com++;
+                  if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                    HTS_PANIC_PRINTF(
+                        "Option warc-max-size needs a blank space and a size");
+                    htsmain_free();
+                    return -1;
+                  }
+                  na++;
+                  { // reject non-numeric/negative/overflow; keep default 0
+                    // (single file)
+                    char *end;
+                    LLint v;
+                    errno = 0;
+                    v = strtoll(argv[na], &end, 10);
+                    if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
+                        errno != ERANGE)
+                      opt->warc_max_size = v;
+                  }
+                } else if (*(com + 1) == 'c') { // --warc-cdx: sorted CDXJ index
+                  com++;
+                  opt->warc_cdx = 1;
+                } else if (*(com + 1) == 'z') { // --wacz: WACZ package
+                  com++;
+                  opt->warc_wacz = 1;
+                  opt->warc_cdx = 1; // WACZ embeds the CDXJ index
+                  if (!StringNotEmpty(opt->warc_file))
+                    StringCopy(opt->warc_file, WARC_AUTONAME);
                 } else { // --warc: auto-named archive under the output dir
                   StringCopy(opt->warc_file, WARC_AUTONAME);
                 }

src/htshelp.c

@@ -524,8 +524,6 @@ void help(const char *app, int more) {
   infomsg
     (" %D  cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
   infomsg(" %M  generate a RFC MIME-encapsulated full-archive (.mht)");
-  infomsg(" %r  write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
-          "output name");
   infomsg(" %t  keep the original file extension, don't rewrite it from the "
           "MIME type (%t0 rewrite)");
   infomsg
@@ -595,6 +593,10 @@ void help(const char *app, int more) {
   infomsg
     ("  C  create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)");
   infomsg("  k  store all files in cache (not useful if files on disk)");
+  infomsg(" %r  write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
+          "output name, --warc-max-size N rotates segments past N bytes, "
+          "--warc-cdx also writes a sorted CDXJ index, --wacz packages it all "
+          "as a WACZ file");
   infomsg(" %n  do not re-download locally erased files");
   infomsg
     (" %v  display on screen filenames downloaded (in realtime) - * %v1 short version - %v2 full animation");

src/htslib.c

@@ -6019,6 +6019,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
   StringCopy(opt->warc_file, "");
+  opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
   StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;

src/htsopt.h

@@ -541,6 +541,12 @@ struct httrackp {
                          and exit without crawling */
   String warc_file; /**< WARC output: WARC_AUTONAME for --warc, or the
                          --warc-file basename (appended at the tail: ABI) */
+  LLint warc_max_size;  /**< --warc-max-size: rotate the archive past this many
+                             bytes (<=0: single file). Tail: ABI */
+  hts_boolean warc_cdx; /**< --warc-cdx: write a sorted CDXJ index next to the
+                             archive. Tail: ABI */
+  hts_boolean warc_wacz; /**< --wacz: package archive+index+pages as a WACZ zip
+                              (implies --warc + --warc-cdx). Tail: ABI */
 };
 
 /* Running statistics for a mirror. */
@@ -667,6 +673,11 @@ struct htsblk {
   char *warc_reqhdr; /**< stashed raw request header block for WARC (or NULL) */
   char *
       warc_resphdr; /**< stashed raw response header block for WARC (or NULL) */
+  int warc_truncated; /**< WARC-Truncated reason for a cap-truncated body
+                           (WARC_TRUNC_*, 0=none). Tail: ABI */
+  char *warc_rawpath; /**< verbatim WARC: spooled compressed body path, or NULL
+                           (owns the file; unlinked on free). Tail: ABI */
+  LLint warc_rawsize; /**< byte length of warc_rawpath. Tail: ABI */
   /*char digest[32+2];   // md5 digest generated by the engine ("" if none) */
 };
 

src/htsselftest.c

@@ -64,6 +64,9 @@ Please visit our Website: http://www.httrack.com
 #if HTS_USEZSTD
 #include <zstd.h>
 #endif
+#if HTS_USEOPENSSL
+#include <openssl/evp.h>
+#endif
 #include "coucal/coucal.h"
 
 #include <ctype.h>
@@ -3354,7 +3357,8 @@ static unsigned char *warc_next_member(const unsigned char **in,
 /* Feed a synthetic transaction and validate the resulting .warc.gz against the
    WARC/1.1 spec: each record a self-standing gzip member starting WARC/1.,
    Content-Length == block length, the \r\n\r\n trailer intact, the response
-   body round-trips, and the encoding headers are stripped (strategy B). */
+   body round-trips, and the hop-by-hop Transfer-Encoding is dropped (a real
+   Content-Encoding is kept verbatim; see warc-verbatim). */
 static int st_warc(httrackp *opt, int argc, char **argv) {
   char path[HTS_URLMAXSIZE];
   warc_writer *w;
@@ -3374,21 +3378,22 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
   w = warc_open(opt, path);
   assertf(w != NULL);
 
-  /* 200 HTML: bogus Content-Length + gzip/chunked encodings must be stripped.
-   */
+  /* 200 HTML, plaintext body: bogus Content-Length rewritten, hop-by-hop
+     Transfer-Encoding dropped. The whitespace before its ':' exercises
+     header_is tolerating "Name : value". */
   warc_write_transaction(
       w, "http://test.local/a.html", "127.0.0.1",
       "GET /a.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
-      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
-      "gzip\r\nTransfer-Encoding: chunked\r\nContent-Length: 999\r\n\r\n",
-      a_body, sizeof(a_body) - 1, NULL, 200, 0);
+      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n"
+      "Transfer-Encoding : chunked\r\nContent-Length: 999\r\n\r\n",
+      a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
 
   /* 302 redirect: header-only, no body. */
   warc_write_transaction(
       w, "http://test.local/r", "127.0.0.1",
       "GET /r HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 302 Found\r\nLocation: http://test.local/a.html\r\n\r\n", NULL,
-      0, NULL, 302, 0);
+      0, NULL, 302, 0, 0);
 
   /* 200 binary, chunked coding on the wire (already de-chunked here). */
   warc_write_transaction(
@@ -3396,7 +3401,7 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
       "GET /b.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n"
       "Transfer-Encoding: chunked\r\n\r\n",
-      "\x00\x01\x02\x03\x04", 5, NULL, 200, 0);
+      "\x00\x01\x02\x03\x04", 5, NULL, 200, 0, 0);
 
   /* 200 with a body shorter than the declared Content-Length (rewritten). */
   warc_write_transaction(
@@ -3404,20 +3409,20 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
       "GET /trunc HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: "
       "100\r\n\r\n",
-      "short", 5, NULL, 200, 0);
+      "short", 5, NULL, 200, 0, 0);
 
   /* Same payload as a.html at a new URL: identical-payload-digest revisit
      (OpenSSL builds only; a plain build writes a second full response). */
   warc_write_transaction(w, "http://test.local/a2.html", "127.0.0.1",
                          "GET /a2.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
                          "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
-                         a_body, sizeof(a_body) - 1, NULL, 200, 0);
+                         a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
 
   /* 304 revisit with an EMPTY response-header block: the block is just the
      2-byte separator, so declared Content-Length must be exactly 2 (F3). */
   warc_write_transaction(w, "http://test.local/nm", "127.0.0.1",
                          "GET /nm HTTP/1.1\r\nHost: test.local\r\n\r\n", "",
-                         NULL, 0, NULL, 304, 1);
+                         NULL, 0, NULL, 304, 1, 0);
 
   warc_close(w);
 
@@ -3485,7 +3490,8 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
                     hdr_len, 37) != NULL &&
         warc_memstr((char *) rec, "WARC-Type: revisit", hdr_len, 18) != NULL)
       nm_cl_ok = (block_len == 2);
-    /* a.html response body must round-trip and carry no encoding headers */
+    /* a.html response body round-trips; no Content-Encoding (plaintext) and the
+       whitespaced Transfer-Encoding was dropped (header_is robustness). */
     if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/a.html",
                     hdr_len, 41) != NULL &&
         warc_memstr((char *) rec, "msgtype=response", hdr_len, 16) != NULL) {
@@ -3534,6 +3540,770 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* Parse a record's header/block split; sets *hdr_len and *block_len, returns 0
+   when Content-Length matches the actual block bytes, -1 otherwise. */
+static int warc_rec_split(const unsigned char *rec, size_t rlen,
+                          size_t *hdr_len, long long *block_len) {
+  const char *sep = warc_memstr((const char *) rec, "\r\n\r\n", rlen, 4);
+  const char *cl;
+  *block_len = 0;
+  if (sep == NULL)
+    return -1;
+  *hdr_len = (size_t) ((const unsigned char *) sep - rec) + 4;
+  cl = warc_memstr((const char *) rec, "Content-Length:", *hdr_len, 15);
+  if (cl == NULL || sscanf(cl + 15, "%lld", block_len) != 1 ||
+      *hdr_len + (size_t) *block_len + 4 != rlen)
+    return -1;
+  return 0;
+}
+
+/* A cap-truncated body is still archived, tagged WARC-Truncated (v1.1). A
+   compressed body cut short by a cap keeps its Content-Encoding (the stored
+   bytes are the coded partial), so the record's label matches its body: assert
+   the plaintext response carries "WARC-Truncated: length", and the gzip-coded
+   one carries "WARC-Truncated: time", keeps Content-Encoding, and stores the
+   coded bytes verbatim. */
+static int st_warc_trunc(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  warc_writer *w;
+  unsigned char *data;
+  size_t data_len = 0;
+  const unsigned char *p, *end;
+  int err = 0, trunc_len = 0, trunc_gz = 0, nresp = 0;
+  static const char body[] = "partial body bytes\n";
+  /* a valid gzip member (inflates to a known plaintext), as the coded partial
+   */
+  static const unsigned char gz[] = {
+      0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x0b, 0x2e,
+      0x29, 0x4a, 0x2c, 0x49, 0x4d, 0xaf, 0xd4, 0x75, 0x54, 0x28, 0x4b, 0x2d,
+      0x4a, 0x4a, 0x2c, 0xc9, 0xcc, 0x55, 0x08, 0x77, 0x0c, 0x72, 0x56, 0x48,
+      0xca, 0x4f, 0xa9, 0xb4, 0x52, 0x28, 0xc9, 0x48, 0x55, 0x28, 0x2c, 0xcd,
+      0x4c, 0xce, 0x56, 0x48, 0x2a, 0xca, 0x2f, 0xcf, 0x53, 0x48, 0xcb, 0xaf,
+      0x50, 0xc8, 0x2a, 0xcd, 0x2d, 0x28, 0xd6, 0xe3, 0x02, 0x00, 0x5e, 0xb8,
+      0xe7, 0x66, 0x3a, 0x00, 0x00, 0x00};
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-trunc: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-trunc.warc.gz");
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  warc_write_transaction(
+      w, "http://test.local/big.bin", "127.0.0.1",
+      "GET /big.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
+      "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n", body,
+      sizeof(body) - 1, NULL, 200, 0, WARC_TRUNC_LENGTH);
+  warc_write_transaction(
+      w, "http://test.local/big.gz", "127.0.0.1",
+      "GET /big.gz HTTP/1.1\r\nHost: test.local\r\n\r\n",
+      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
+      "gzip\r\n\r\n",
+      (const char *) gz, sizeof(gz), NULL, 200, 0, WARC_TRUNC_TIME);
+  warc_close(w);
+
+  data = warc_slurp(path, &data_len);
+  assertf(data != NULL);
+  p = data;
+  end = data + data_len;
+  while (p < end) {
+    size_t rlen = 0, hdr_len = 0;
+    long long block_len = 0;
+    unsigned char *rec = warc_next_member(&p, end, &rlen);
+    if (rec == NULL) {
+      if (rlen != 0)
+        err = 1;
+      break;
+    }
+    if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0) {
+      err = 1;
+      freet(rec);
+      continue;
+    }
+    if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL) {
+      nresp++;
+      if (warc_memstr((char *) rec,
+                      "WARC-Target-URI: http://test.local/big.bin", hdr_len,
+                      42) != NULL &&
+          warc_memstr((char *) rec, "WARC-Truncated: length", hdr_len, 22) !=
+              NULL)
+        trunc_len = 1;
+      if (warc_memstr((char *) rec, "WARC-Target-URI: http://test.local/big.gz",
+                      hdr_len, 41) != NULL) {
+        const char *bsep = warc_memstr((char *) rec + hdr_len, "\r\n\r\n",
+                                       (size_t) block_len, 4);
+        size_t bodyoff = bsep ? (size_t) (bsep - (char *) rec) + 4 : 0;
+        size_t got = bsep ? rlen - 4 - bodyoff : 0;
+        /* WARC-Truncated: time, Content-Encoding kept, stored body == coded. */
+        if (bsep != NULL &&
+            warc_memstr((char *) rec, "WARC-Truncated: time", hdr_len, 20) !=
+                NULL &&
+            warc_memstr((char *) rec + hdr_len,
+                        "Content-Encoding:", (size_t) block_len, 17) != NULL &&
+            got == sizeof(gz) && memcmp(rec + bodyoff, gz, sizeof(gz)) == 0)
+          trunc_gz = 1;
+      }
+    }
+    freet(rec);
+  }
+  freet(data);
+  if (!trunc_len || !trunc_gz || nresp != 2)
+    err = 1;
+  printf("warc-trunc: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+/* An ftp:// capture is ONE resource record: WARC-Type: resource, the payload's
+   own Content-Type, block == payload, and no request/response pair. */
+static int st_warc_ftp(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  warc_writer *w;
+  unsigned char *data;
+  size_t data_len = 0;
+  const unsigned char *p, *end;
+  int err = 0, nresource = 0, nresp = 0, nreq = 0;
+  static const char body[] = "\x00\x01"
+                             "FTP payload"
+                             "\x02\x03";
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-ftp: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-ftp.warc.gz");
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  warc_write_resource(w, "ftp://ftp.local/file.bin", "127.0.0.1",
+                      "application/octet-stream", body, sizeof(body) - 1, NULL,
+                      0);
+  warc_close(w);
+
+  data = warc_slurp(path, &data_len);
+  assertf(data != NULL);
+  p = data;
+  end = data + data_len;
+  while (p < end) {
+    size_t rlen = 0, hdr_len = 0;
+    long long block_len = 0;
+    unsigned char *rec = warc_next_member(&p, end, &rlen);
+    if (rec == NULL) {
+      if (rlen != 0)
+        err = 1;
+      break;
+    }
+    if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
+      err = 1;
+    if (warc_memstr((char *) rec, "WARC-Type: resource", hdr_len, 19) != NULL) {
+      nresource++;
+      if ((size_t) block_len != sizeof(body) - 1 ||
+          memcmp(rec + hdr_len, body, sizeof(body) - 1) != 0)
+        err = 1; /* block is the raw payload, no HTTP envelope */
+      if (warc_memstr((char *) rec, "WARC-Target-URI: ftp://ftp.local/file.bin",
+                      hdr_len, 41) == NULL ||
+          warc_memstr((char *) rec, "Content-Type: application/octet-stream",
+                      hdr_len, 38) == NULL)
+        err = 1;
+    }
+    if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL)
+      nresp++;
+    if (warc_memstr((char *) rec, "WARC-Type: request", hdr_len, 18) != NULL)
+      nreq++;
+    freet(rec);
+  }
+  freet(data);
+  if (nresource != 1 || nresp != 0 || nreq != 0)
+    err = 1;
+  printf("warc-ftp: resource=%d response=%d request=%d: %s\n", nresource, nresp,
+         nreq, err ? "FAIL" : "OK");
+  return err;
+}
+
+/* --warc-max-size rotates into <base>-00000.warc.gz, -00001, ...; each segment
+   is independently valid and begins with its own warcinfo. */
+static int st_warc_rotate(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  char seg[HTS_URLMAXSIZE];
+  warc_writer *w;
+  LLint saved_max;
+  unsigned char body[600];
+  unsigned int rng = 0x12345678u;
+  int err = 0, nseg = 0, i;
+  size_t j;
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-rotate: needs a writable directory\n");
+    return 1;
+  }
+  for (j = 0; j < sizeof(body);
+       j++) { /* incompressible: gzip can't shrink it */
+    rng ^= rng << 13;
+    rng ^= rng >> 17;
+    rng ^= rng << 5;
+    body[j] = (unsigned char) (rng >> 24);
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-rot.warc.gz");
+  saved_max = opt->warc_max_size;
+  opt->warc_max_size =
+      1000; /* a couple records per segment => several segments */
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  for (i = 0; i < 8; i++) {
+    char uri[64];
+    snprintf(uri, sizeof(uri), "http://test.local/f%d.bin", i);
+    warc_write_transaction(
+        w, uri, "127.0.0.1", "GET / HTTP/1.1\r\nHost: test.local\r\n\r\n",
+        "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n",
+        (const char *) body, sizeof(body), NULL, 200, 0, 0);
+  }
+  warc_close(w);
+  opt->warc_max_size = saved_max;
+
+  for (i = 0;; i++) {
+    char fname[64];
+    unsigned char *data;
+    size_t data_len = 0;
+    const unsigned char *p, *pend;
+    int first = 1;
+    snprintf(fname, sizeof(fname), "warc-rot-%05d.warc.gz", i);
+    fconcat(seg, sizeof(seg), argv[0], fname);
+    data = warc_slurp(seg, &data_len);
+    if (data == NULL)
+      break; /* past the last segment */
+    nseg++;
+    p = data;
+    pend = data + data_len;
+    while (p < pend) {
+      size_t rlen = 0, hdr_len = 0;
+      long long block_len = 0;
+      unsigned char *rec = warc_next_member(&p, pend, &rlen);
+      if (rec == NULL) {
+        if (rlen != 0)
+          err = 1;
+        break;
+      }
+      if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
+        err = 1;
+      if (first) { /* each segment leads with its own warcinfo */
+        if (warc_memstr((char *) rec, "WARC-Type: warcinfo", hdr_len, 19) ==
+            NULL)
+          err = 1;
+        first = 0;
+      }
+      freet(rec);
+    }
+    freet(data);
+    if (first) /* empty segment */
+      err = 1;
+  }
+  if (nseg < 2)
+    err = 1;
+  printf("warc-rotate: %d segments: %s\n", nseg, err ? "FAIL" : "OK");
+  return err;
+}
+
+/* The default body storage: assert the stored WARC record is byte-verbatim gzip
+   with Content-Encoding preserved and Content-Length = the coded length. */
+static int st_warc_verbatim(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  warc_writer *w;
+  unsigned char *data;
+  size_t data_len = 0;
+  const unsigned char *p, *end;
+  int err = 0, checked = 0;
+  static const char a_plain[] =
+      "Strategy-A verbatim WARC body: the quick brown fox jumps.\n";
+  static const unsigned char a_gz[] = {
+      0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x0b, 0x2e,
+      0x29, 0x4a, 0x2c, 0x49, 0x4d, 0xaf, 0xd4, 0x75, 0x54, 0x28, 0x4b, 0x2d,
+      0x4a, 0x4a, 0x2c, 0xc9, 0xcc, 0x55, 0x08, 0x77, 0x0c, 0x72, 0x56, 0x48,
+      0xca, 0x4f, 0xa9, 0xb4, 0x52, 0x28, 0xc9, 0x48, 0x55, 0x28, 0x2c, 0xcd,
+      0x4c, 0xce, 0x56, 0x48, 0x2a, 0xca, 0x2f, 0xcf, 0x53, 0x48, 0xcb, 0xaf,
+      0x50, 0xc8, 0x2a, 0xcd, 0x2d, 0x28, 0xd6, 0xe3, 0x02, 0x00, 0x5e, 0xb8,
+      0xe7, 0x66, 0x3a, 0x00, 0x00, 0x00};
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-verbatim: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-verbatim.warc.gz");
+
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  /* the body is the coded (gzip) octets, stored verbatim. */
+  warc_write_transaction(
+      w, "http://test.local/z.html", "127.0.0.1",
+      "GET /z.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
+      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
+      "gzip\r\nTransfer-Encoding: chunked\r\nContent-Length: 999\r\n\r\n",
+      (const char *) a_gz, sizeof(a_gz), NULL, 200, 0, 0);
+  warc_close(w);
+
+  data = warc_slurp(path, &data_len);
+  assertf(data != NULL);
+  p = data;
+  end = data + data_len;
+
+  while (p < end) {
+    size_t rlen = 0, hdr_len = 0;
+    long long block_len = 0;
+    unsigned char *rec = warc_next_member(&p, end, &rlen);
+    if (rec == NULL) {
+      if (rlen != 0)
+        err = 1;
+      break;
+    }
+    if (warc_memstr((char *) rec, "msgtype=response", rlen, 16) == NULL) {
+      freet(rec);
+      continue;
+    }
+    if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0) {
+      err = 1;
+      freet(rec);
+      continue;
+    }
+    /* Assert: one Content-Encoding, no Transfer-Encoding, Content-Length =
+       compressed size. */
+    {
+      const char *block = (char *) rec + hdr_len;
+      const char *ce =
+          warc_memstr(block, "Content-Encoding:", (size_t) block_len, 17);
+      const char *hcl =
+          warc_memstr(block, "Content-Length:", (size_t) block_len, 15);
+      long long http_cl = -1;
+      int nce = 0;
+      const char *scan = ce;
+      while (scan != NULL) {
+        size_t rem = (size_t) block_len - (size_t) (scan - block);
+        nce++;
+        scan = warc_memstr(scan + 17, "Content-Encoding:", rem - 17, 17);
+      }
+      if (ce == NULL || strncasecmp(ce + 17, " gzip", 5) != 0 || nce != 1)
+        err = 1;
+      if (warc_memstr(block, "Transfer-Encoding:", (size_t) block_len, 18) !=
+          NULL)
+        err = 1;
+      if (hcl == NULL || sscanf(hcl + 15, "%lld", &http_cl) != 1 ||
+          http_cl != (long long) sizeof(a_gz))
+        err = 1;
+    }
+    /* Stored block bytes equal the gzip input, and inflate to the plaintext. */
+    {
+      const char *bsep = warc_memstr((char *) rec + hdr_len, "\r\n\r\n",
+                                     (size_t) block_len, 4);
+      if (bsep == NULL)
+        err = 1;
+      else {
+        size_t bodyoff = (size_t) (bsep - (char *) rec) + 4;
+        size_t got = rlen - 4 - bodyoff; /* minus the record trailer */
+        if (got != sizeof(a_gz) ||
+            memcmp(rec + bodyoff, a_gz, sizeof(a_gz)) != 0)
+          err = 1;
+        else {
+          const unsigned char *bp = rec + bodyoff;
+          size_t plen = 0;
+          unsigned char *plain = warc_next_member(&bp, bp + got, &plen);
+          if (plain == NULL || plen != sizeof(a_plain) - 1 ||
+              memcmp(plain, a_plain, plen) != 0)
+            err = 1;
+          freet(plain);
+        }
+      }
+    }
+    checked = 1;
+    freet(rec);
+  }
+  freet(data);
+  if (!checked)
+    err = 1;
+  printf("warc-verbatim: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+/* SURT canonicalization vectors (the CDXJ sort key: www-strip, default-port
+   strip, host reversal, non-default port kept, IP/IPv6 verbatim). */
+static int st_warc_surt(httrackp *opt, int argc, char **argv) {
+  static const struct {
+    const char *url, *want;
+  } cases[] = {
+      {"http://www.example.com/", "com,example)/"},
+      {"http://example.com:80/a/b?q=1", "com,example)/a/b?q=1"},
+      {"https://www.EXAMPLE.com/Path", "com,example)/Path"},
+      {"https://example.com:443/", "com,example)/"},
+      {"http://www2.example.com/x", "com,example)/x"},
+      {"http://example.com:8080/p", "com,example:8080)/p"},
+      {"http://user:pass@www.example.com/y", "com,example)/y"},
+      {"http://192.168.0.1/z", "192.168.0.1)/z"},
+      {"http://[2001:db8::1]/w", "[2001:db8::1])/w"},
+      {"http://sub.a.example.co.uk/deep?x=1#frag",
+       "uk,co,example,a,sub)/deep?x=1"},
+  };
+
+  int err = 0;
+  size_t i;
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+    char out[512];
+    if (warc_surt(cases[i].url, out, sizeof(out)) != 0 ||
+        strcmp(out, cases[i].want) != 0) {
+      fprintf(stderr, "warc-surt: %s -> %s (want %s)\n", cases[i].url, out,
+              cases[i].want);
+      err = 1;
+    }
+  }
+  printf("warc-surt: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+/* End-to-end CDXJ: crawl a handful of records with --warc-cdx, then verify the
+   .cdx is sorted, has exactly one line per response/revisit/resource (none for
+   warcinfo/request), and each offset/length points at a gzip member that
+   independently inflates to a record whose WARC-Target-URI matches the line. */
+static int st_warc_cdx(httrackp *opt, int argc, char **argv) {
+  char wpath[HTS_URLMAXSIZE], cpath[HTS_URLMAXSIZE];
+  warc_writer *w;
+  unsigned char *warc = NULL, *cdx = NULL;
+  size_t warc_len = 0, cdx_len = 0;
+  hts_boolean saved_cdx;
+  int err = 0, nlines = 0;
+  const char *lp, *cend;
+  char prev[2048];
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-cdx: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(wpath, sizeof(wpath), argv[0], "warc-cdx.warc.gz");
+  fconcat(cpath, sizeof(cpath), argv[0], "warc-cdx.cdx");
+  saved_cdx = opt->warc_cdx;
+  opt->warc_cdx = 1;
+  w = warc_open(opt, wpath);
+  assertf(w != NULL);
+  warc_write_transaction(w, "http://www.example.com/one", "127.0.0.1",
+                         "GET /one HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+                         "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
+                         "one body\n", 9, NULL, 200, 0, 0);
+  warc_write_resource(w, "ftp://files.example.com/data.bin", "127.0.0.1",
+                      "application/octet-stream", "\x00\x01\x02\x03", 4, NULL,
+                      0);
+  warc_write_transaction(w, "http://alpha.example.com/two", "127.0.0.1",
+                         "GET /two HTTP/1.1\r\nHost: alpha.example.com\r\n\r\n",
+                         "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n",
+                         "two body\n", 9, NULL, 200, 0, 0);
+  /* Same payload as /one at a new URL: identical-payload-digest revisit under
+     OpenSSL, a full response otherwise; either way one index line. */
+  warc_write_transaction(w, "http://zeta.example.com/dup", "127.0.0.1",
+                         "GET /dup HTTP/1.1\r\nHost: zeta.example.com\r\n\r\n",
+                         "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
+                         "one body\n", 9, NULL, 200, 0, 0);
+  warc_close(w);
+  opt->warc_cdx = saved_cdx;
+
+  warc = warc_slurp(wpath, &warc_len);
+  cdx = warc_slurp(cpath, &cdx_len);
+  assertf(warc != NULL);
+  assertf(cdx != NULL);
+
+  prev[0] = '\0';
+  lp = (const char *) cdx;
+  cend = (const char *) cdx + cdx_len;
+  while (lp < cend) {
+    const char *eol = memchr(lp, '\n', (size_t) (cend - lp));
+    size_t llen = eol ? (size_t) (eol - lp) : (size_t) (cend - lp);
+    char line[2048];
+    const char *j, *us, *ue, *o, *l;
+    char url[1024];
+    unsigned long long off = 0, len = 0;
+    const unsigned char *mp, *mend;
+    unsigned char *rec;
+    size_t rlen = 0, urllen;
+    if (llen == 0) {
+      lp = eol ? eol + 1 : cend;
+      continue;
+    }
+    if (llen >= sizeof(line)) {
+      err = 1;
+      break;
+    }
+    memcpy(line, lp, llen);
+    line[llen] = '\0';
+    nlines++;
+    if (prev[0] != '\0' && strcmp(prev, line) > 0)
+      err = 1; /* must be sorted */
+    strlcpybuff(prev, line, sizeof(prev));
+    j = strstr(line, "\"url\": \"");
+    o = strstr(line, "\"offset\": \"");
+    l = strstr(line, "\"length\": \"");
+    if (j == NULL || o == NULL || l == NULL ||
+        sscanf(o + 11, "%llu", &off) != 1 ||
+        sscanf(l + 11, "%llu", &len) != 1) {
+      err = 1;
+      goto nextline;
+    }
+    us = j + 8;
+    ue = strchr(us, '"');
+    if (ue == NULL || (urllen = (size_t) (ue - us)) >= sizeof(url)) {
+      err = 1;
+      goto nextline;
+    }
+    memcpy(url, us, urllen);
+    url[urllen] = '\0';
+    if (len == 0 || off > warc_len || len > warc_len - off) {
+      err = 1;
+      goto nextline;
+    }
+    mp = warc + off;
+    mend = warc + off + len;
+    rec = warc_next_member(&mp, mend, &rlen);
+    if (rec == NULL) {
+      err = 1;
+      goto nextline;
+    }
+    {
+      char needle[1100];
+      snprintf(needle, sizeof(needle), "WARC-Target-URI: %s\r\n", url);
+      if (warc_memstr((char *) rec, needle, rlen, strlen(needle)) == NULL)
+        err = 1;
+    }
+    freet(rec);
+  nextline:
+    lp = eol ? eol + 1 : cend;
+  }
+  freet(warc);
+  freet(cdx);
+  if (nlines != 4)
+    err = 1; /* 3 responses/revisits + 1 resource; no warcinfo/request */
+  printf("warc-cdx: %d index lines: %s\n", nlines, err ? "FAIL" : "OK");
+  return err;
+}
+
+#if HTS_USEOPENSSL
+/* Lowercase-hex SHA-256 of n bytes into out[65]; 1 on success. */
+static int wacz_test_sha256(const void *p, size_t n, char out[65]) {
+  EVP_MD_CTX *c = EVP_MD_CTX_new();
+  unsigned char md[EVP_MAX_MD_SIZE];
+  unsigned int mdlen = 0, i;
+  static const char hx[] = "0123456789abcdef";
+  int ok;
+  if (c == NULL)
+    return 0;
+  ok = EVP_DigestInit_ex(c, EVP_sha256(), NULL) == 1 &&
+       (n == 0 || EVP_DigestUpdate(c, p, n) == 1) &&
+       EVP_DigestFinal_ex(c, md, &mdlen) == 1 && mdlen == 32;
+  EVP_MD_CTX_free(c);
+  if (!ok)
+    return 0;
+  for (i = 0; i < 32; i++) {
+    out[i * 2] = hx[md[i] >> 4];
+    out[i * 2 + 1] = hx[md[i] & 0x0F];
+  }
+  out[64] = '\0';
+  return 1;
+}
+
+/* One unzipped WACZ member: name, raw bytes, and the ZIP compression method. */
+typedef struct {
+  char name[256];
+  unsigned char *data;
+  size_t len;
+  int method;
+} wacz_entry;
+
+/* Package a 2-record WARC as a WACZ, then unzip it in-process and assert the
+   fixed layout, STORE-mode entries, recomputing sha256 digests, the digest
+   chain, and the pages.jsonl header. */
+static int st_warc_wacz(httrackp *opt, int argc, char **argv) {
+  char wpath[HTS_URLMAXSIZE], waczpath[HTS_URLMAXSIZE], cdxpath[HTS_URLMAXSIZE];
+  warc_writer *w;
+  hts_boolean saved_cdx, saved_wacz;
+  wacz_entry ent[16];
+  int nent = 0, err = 0, i;
+  unzFile uf;
+  const wacz_entry *dp = NULL, *dig = NULL, *pages = NULL;
+  int have_archive = 0, have_index = 0, all_store = 1;
+  LLint good_size;
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-wacz: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(wpath, sizeof(wpath), argv[0], "warc-wacz.warc.gz");
+  fconcat(waczpath, sizeof(waczpath), argv[0], "warc-wacz.wacz");
+  fconcat(cdxpath, sizeof(cdxpath), argv[0], "warc-wacz.cdx");
+  saved_cdx = opt->warc_cdx;
+  saved_wacz = opt->warc_wacz;
+  opt->warc_cdx = 1;
+  opt->warc_wacz = 1;
+  w = warc_open(opt, wpath);
+  assertf(w != NULL);
+  warc_write_transaction(w, "http://www.example.com/", "127.0.0.1",
+                         "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+                         "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
+                         "<html>home</html>\n", 18, NULL, 200, 0, 0);
+  warc_write_transaction(
+      w, "http://www.example.com/data.bin", "127.0.0.1",
+      "GET /data.bin HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+      "HTTP/1.1 200 OK\r\n"
+      "Content-Type: application/octet-stream\r\n\r\n",
+      "\x00\x01\x02\x03\x04", 5, NULL, 200, 0, 0);
+  warc_close(w);
+
+  /* Unzip every member in-process. */
+  uf = unzOpen(waczpath);
+  assertf(uf != NULL);
+  if (unzGoToFirstFile(uf) == UNZ_OK) {
+    do {
+      unz_file_info info;
+      wacz_entry *e;
+      if (nent >= (int) (sizeof(ent) / sizeof(ent[0]))) {
+        err = 1;
+        break;
+      }
+      e = &ent[nent];
+      if (unzGetCurrentFileInfo(uf, &info, e->name, sizeof(e->name), NULL, 0,
+                                NULL, 0) != UNZ_OK) {
+        err = 1;
+        break;
+      }
+      e->method = (int) info.compression_method;
+      e->len = (size_t) info.uncompressed_size;
+      e->data = malloct(e->len ? e->len : 1);
+      if (e->data == NULL || unzOpenCurrentFile(uf) != UNZ_OK) {
+        err = 1;
+        break;
+      }
+      if (e->len > 0 &&
+          unzReadCurrentFile(uf, e->data, (unsigned) e->len) != (int) e->len)
+        err = 1;
+      unzCloseCurrentFile(uf);
+      nent++;
+    } while (unzGoToNextFile(uf) == UNZ_OK);
+  }
+  unzClose(uf);
+
+  /* Classify members and assert STORE mode (WACZ spec requirement). */
+  for (i = 0; i < nent; i++) {
+    const wacz_entry *e = &ent[i];
+    if (e->method != 0)
+      all_store = 0;
+    if (strncmp(e->name, "archive/", 8) == 0)
+      have_archive = 1;
+    else if (strcmp(e->name, "indexes/index.cdx") == 0)
+      have_index = 1;
+    else if (strcmp(e->name, "pages/pages.jsonl") == 0)
+      pages = e;
+    else if (strcmp(e->name, "datapackage.json") == 0)
+      dp = e;
+    else if (strcmp(e->name, "datapackage-digest.json") == 0)
+      dig = e;
+  }
+  if (!have_archive || !have_index || pages == NULL || dp == NULL ||
+      dig == NULL || !all_store)
+    err = 1;
+
+  /* pages.jsonl: header line, then >= 1 body row carrying url + ts. */
+  if (pages != NULL) {
+    if (pages->len < 27 ||
+        memcmp(pages->data, "{\"format\": \"json-pages-1.0\"", 27) != 0)
+      err = 1;
+    else {
+      const char *nl = memchr(pages->data, '\n', pages->len);
+      const char *body = nl ? nl + 1 : NULL;
+      size_t blen =
+          body ? pages->len - (size_t) (body - (char *) pages->data) : 0;
+      if (body == NULL || blen == 0 ||
+          warc_memstr(body, "\"url\": ", blen, 7) == NULL ||
+          warc_memstr(body, "\"ts\": ", blen, 6) == NULL)
+        err = 1;
+    }
+  }
+
+  /* Every datapackage resource hash recomputes from the stored member bytes. */
+  if (dp != NULL) {
+    char *json = malloct(dp->len + 1);
+    if (json == NULL) {
+      err = 1;
+    } else {
+      const char *p;
+      memcpy(json, dp->data, dp->len);
+      json[dp->len] = '\0';
+      if (strstr(json, "\"profile\": \"data-package\"") == NULL ||
+          strstr(json, "\"wacz_version\": \"") == NULL)
+        err = 1;
+      p = json;
+      while ((p = strstr(p, "\"path\": \"")) != NULL) {
+        char path[256], want[80], got[65];
+        const char *pe, *h;
+        size_t plen;
+        p += 9;
+        pe = strchr(p, '"');
+        if (pe == NULL || (plen = (size_t) (pe - p)) >= sizeof(path)) {
+          err = 1;
+          break;
+        }
+        memcpy(path, p, plen);
+        path[plen] = '\0';
+        h = strstr(pe, "\"hash\": \"sha256:");
+        if (h == NULL || sscanf(h + 16, "%79[0-9a-f]", want) != 1) {
+          err = 1;
+          break;
+        }
+        for (i = 0; i < nent; i++)
+          if (strcmp(ent[i].name, path) == 0)
+            break;
+        if (i == nent || !wacz_test_sha256(ent[i].data, ent[i].len, got) ||
+            strcmp(got, want) != 0)
+          err = 1;
+        p = pe;
+      }
+      freet(json);
+    }
+  }
+
+  /* datapackage-digest.json chains sha256(datapackage.json). */
+  if (dp != NULL && dig != NULL) {
+    char dphex[65], *djson = malloct(dig->len + 1);
+    const char *h;
+    char want[80];
+    if (djson == NULL || !wacz_test_sha256(dp->data, dp->len, dphex)) {
+      err = 1;
+    } else {
+      memcpy(djson, dig->data, dig->len);
+      djson[dig->len] = '\0';
+      if (strstr(djson, "\"path\": \"datapackage.json\"") == NULL)
+        err = 1;
+      h = strstr(djson, "\"hash\": \"sha256:");
+      if (h == NULL || sscanf(h + 16, "%79[0-9a-f]", want) != 1 ||
+          strcmp(want, dphex) != 0)
+        err = 1;
+    }
+    freet(djson);
+  }
+
+  for (i = 0; i < nent; i++)
+    freet(ent[i].data);
+
+  /* #522-class: a failed re-package must leave the existing .wacz untouched.
+     Drop the .cdx and re-run empty so packaging fails on the missing index. */
+  good_size = fsize(waczpath);
+  if (good_size <= 0)
+    err = 1;
+  (void) UNLINK(cdxpath);
+  w = warc_open(opt, wpath);
+  assertf(w != NULL);
+  warc_close(w);
+  if (fsize(waczpath) != good_size) /* destroyed or rewritten = data loss */
+    err = 1;
+
+  opt->warc_cdx = saved_cdx;
+  opt->warc_wacz = saved_wacz;
+  printf("warc-wacz: %d members (store=%d): %s\n", nent, all_store,
+         err ? "FAIL" : "OK");
+  return err;
+}
+#endif
+
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -3655,6 +4425,22 @@ static const struct selftest_entry {
     {"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
     {"warc", "<dir>", "WARC/1.1 writer: framing, digests, revisit dedup",
      st_warc},
+    {"warc-trunc", "<dir>", "WARC-Truncated on a cap-truncated body",
+     st_warc_trunc},
+    {"warc-ftp", "<dir>", "ftp resource record (no HTTP envelope)",
+     st_warc_ftp},
+    {"warc-rotate", "<dir>", "--warc-max-size segment rotation",
+     st_warc_rotate},
+    {"warc-verbatim", "<dir>", "verbatim compressed response body (default)",
+     st_warc_verbatim},
+    {"warc-surt", "", "SURT canonicalization of the CDXJ sort key",
+     st_warc_surt},
+    {"warc-cdx", "<dir>", "--warc-cdx CDXJ index: sorted, offsets inflate",
+     st_warc_cdx},
+#if HTS_USEOPENSSL
+    {"warc-wacz", "<dir>", "--wacz package: layout, STORE mode, sha256 digests",
+     st_warc_wacz},
+#endif
 };
 
 static void list_selftests(void) {

src/htswarc.h

@@ -45,6 +45,16 @@ extern "C" {
    under the project's output directory at open time. */
 #define WARC_AUTONAME "\001auto"
 
+/* htsblk.warc_truncated / WARC-Truncated reason tokens (ISO 28500 sec 5.13).
+   A cap-truncated body is still archived, tagged with why it was cut short. */
+#define WARC_TRUNC_NONE 0
+#define WARC_TRUNC_LENGTH 1     /* hit a size cap (-M mirror / -m per-file) */
+#define WARC_TRUNC_TIME 2       /* hit the mirror time cap (-E/--max-time) */
+#define WARC_TRUNC_DISCONNECT 3 /* connection dropped mid-body */
+
+/* WARC-Truncated token for a warc_truncated code, or NULL for none. */
+const char *warc_truncated_reason(int code);
+
 typedef struct warc_writer warc_writer;
 
 /* Stash the raw request header block (bstr.buffer) on r for the later WARC
@@ -58,6 +68,13 @@ void warc_stash_response(htsblk *r, const char *resphdr);
 /* Free both stashed header blocks (idempotent, NULL-safe). */
 void warc_free_request(htsblk *r);
 
+/* Adopt the de-chunked compressed spool at tmpfile_path onto
+   r->warc_rawpath/warc_rawsize (strdupt; frees any prior) so the WARC record
+   stores the body verbatim. No-op leaving warc_rawpath NULL when tmpfile_path
+   is empty or the spool is missing/empty (the record then stores the decoded
+   in-memory/on-disk body instead). */
+void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path);
+
 /* Emit the request + response (or revisit) records for one finished
    transaction. Lazily opens the writer into opt->state.warc; a no-op (logged
    once) if the archive cannot be created. */
@@ -76,6 +93,10 @@ warc_writer *warc_open(httrackp *opt, const char *path);
 /* Flush, close and free the writer (NULL-safe). */
 void warc_close(warc_writer *w);
 
+/* SURT-canonicalize url into out[outsz] (the CDXJ sort key). Returns 0 on
+   success, -1 on error or truncation. Exposed for the -#test=warc-surt test. */
+int warc_surt(const char *url, char *out, size_t outsz);
+
 /* Write one transaction's request + response (or revisit) records.
    target_uri:  absolute URL fetched.
    ip:          numeric peer IP, or NULL/"" to omit.
@@ -84,12 +105,24 @@ void warc_close(warc_writer *w);
    body/body_len: decoded in-memory body, or NULL when on disk.
    body_path:   file re-read for the body when body==NULL (may be NULL).
    is_update_unchanged: nonzero for a 304 server-not-modified revisit.
+   truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
+   The body is stored verbatim: Content-Encoding is kept and Content-Length set
+   to body_len, so body/body_len must be the as-received (coded) bytes.
    Returns 0 on success, -1 on error. */
 int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *ip, const char *req_hdr,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
-                           int statuscode, int is_update_unchanged);
+                           int statuscode, int is_update_unchanged,
+                           int truncated);
+
+/* Write one non-HTTP capture as a single WARC 'resource' record: the block is
+   the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.
+   Used for ftp:// transfers. truncated is a WARC_TRUNC_* reason or 0.
+   Returns 0 on success, -1 on error. */
+int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
+                        const char *content_type, const char *body,
+                        size_t body_len, const char *body_path, int truncated);
 
 #ifdef __cplusplus
 }

tests/01_engine-warc-surt.test

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# SURT canonicalization of the CDXJ sort key (--warc-cdx). Pure string work,
+# so it runs under the MSan-instrumented 01_engine glob.
+httrack -O /dev/null -#test=warc-surt | grep -q "warc-surt: OK"

tests/01_zlib-warc-cdx.test

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# --warc-cdx CDXJ index over synthetic transactions: sorted, one line per
+# response/revisit/resource, each offset/length inflates to the right member.
+
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+
+out=$("$httrack_bin" -O /dev/null -#test=warc-cdx "$scratch/")
+echo "$out"
+case "$out" in
+*": OK") ;;
+*) exit 1 ;;
+esac

tests/01_zlib-warc-wacz.test

@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# --wacz packaging over synthetic transactions: the WACZ unzips in-process to
+# the fixed layout, every entry is ZIP STORE, each datapackage sha256 recomputes
+# from the stored bytes, and the digest chains datapackage.json. WACZ needs the
+# OpenSSL SHA-256 digests, so the self-test is absent on non-OpenSSL builds.
+
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+
+if ! "$httrack_bin" -#test 2>&1 | grep -q '^  warc-wacz'; then
+    echo "warc-wacz self-test unavailable (build without OpenSSL); skipping"
+    exit 77
+fi
+
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+
+out=$("$httrack_bin" -O /dev/null -#test=warc-wacz "$scratch/")
+echo "$out"
+case "$out" in
+*": OK") ;;
+*) exit 1 ;;
+esac

tests/01_zlib-warc.test

@@ -4,16 +4,19 @@
 set -euo pipefail
 
 # WARC/1.1 writer self-test: framing, Content-Length, gzip members, round-trip,
-# and identical-payload-digest revisit dedup, over synthetic transactions.
+# revisit dedup, plus v1.1 WARC-Truncated, ftp resource records, and
+# --warc-max-size rotation, over synthetic transactions.
 
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
 trap 'rm -rf "$scratch"' EXIT
 
-out=$("$httrack_bin" -O /dev/null -#test=warc "$scratch/")
-echo "$out"
-case "$out" in
-*": OK") ;;
-*) exit 1 ;;
-esac
+for t in warc warc-trunc warc-ftp warc-rotate warc-verbatim; do
+    out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")
+    echo "$out"
+    case "$out" in
+    *": OK") ;;
+    *) exit 1 ;;
+    esac
+done

tests/74_local-warc-verbatim.test

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# A --warc crawl stores compressed bodies verbatim (the default): the response
+# record keeps Content-Encoding: gzip and its stored bytes inflate back to the
+# served page. The validator's --verbatim mode is the differential gate:
+# inflate(stored) must equal the decoded body the server compressed.
+#
+# Two fixtures cover both adoption branches of back_finalize: page.html (text/html)
+# takes the in-memory branch; data.bin (application/octet-stream) is streamed to
+# disk, so it exercises the is_write direct-to-disk spool adoption.
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+# decoded bodies served (gzip-coded) by route_warcgz_page and route_warcgz_data.
+export WARC_VALIDATE_BODY="warcgz/page.html=3c68746d6c3e3c626f64793e766572626174696d20677a6970207061676520666f72205741524320737472617465677920413c2f626f64793e3c2f68746d6c3e0a warcgz/data.bin=766572626174696d20677a6970206f637465742d73747265616d20626f647920666f722074686520574152432069735f777269746520706174680a"
+export WARC_VALIDATE_VERBATIM=1
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
+    --found 'warcgz/index.html' --found 'warcgz/page.html' --found 'warcgz/data.bin' \
+    httrack 'BASEURL/warcgz/index.html' --warc-file warc-out

tests/74_local-warc-wacz.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# A --wacz crawl packages the WARC archive, its CDXJ index and a generated
+# pages.jsonl into a single WACZ file at crawl end. The stdlib validator is the
+# real gate: STORE-mode entries, the fixed layout, recomputed sha256 digests and
+# the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
+# skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --wacz-validate \
+    --found 'mini304/index.html' --found 'mini304/page.html' \
+    httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz

tests/Makefile.am

@@ -3,7 +3,7 @@
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
 	proxy-https-server.py socks5-server.py proxy-connect-server.py \
-	proxytestlib.py tls-stall-server.py warc-validate.py \
+	proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
 	local-crawl.sh local-server.py testlib.sh server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
@@ -78,9 +78,12 @@ TESTS = \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
 	01_engine-version-macros.test \
+	01_engine-warc-surt.test \
 	01_engine-xfread.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-warc.test \
+	01_zlib-warc-cdx.test \
+	01_zlib-warc-wacz.test \
 	01_zlib-contentcodings.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
@@ -157,6 +160,8 @@ TESTS = \
 	69_local-intl-logdir.test \
 	71_local-crange-repaircache.test \
 	72_watchdog-crawl.test \
-	73_local-warc.test
+	73_local-warc.test \
+	74_local-warc-wacz.test \
+	74_local-warc-verbatim.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -48,6 +48,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 warc_validate=
+wacz_validate=
 html_subdir=
 outdir_intl=
 rerun=
@@ -117,6 +118,8 @@ while test "$pos" -lt "$nargs"; do
     --rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
     # validate the produced .warc.gz (see the validation block near the end)
     --warc-validate) warc_validate=1 ;;
+    # validate the produced .wacz package (stdlib, plus py-wacz/pywb if present)
+    --wacz-validate) wacz_validate=1 ;;
     --no-purge)
         nopurge=1
         audit+=("--no-purge")
@@ -372,9 +375,13 @@ if test -n "$warc_validate"; then
     fresh="${tmpdir}/warc-pass1.gz"
     test -f "$fresh" || fresh="$warc"
     declare -a bodyargs=()
-    if test -n "${WARC_VALIDATE_BODY:-}"; then
-        bodyargs=(--expect-body-hex "$WARC_VALIDATE_BODY")
-    fi
+    # WARC_VALIDATE_BODY holds one or more whitespace-separated SUB=HEX specs.
+    for spec in ${WARC_VALIDATE_BODY:-}; do
+        bodyargs+=(--expect-body-hex "$spec")
+    done
+    # compressed asset: assert the stored (verbatim) body inflates to the served
+    # body and keeps Content-Encoding, instead of expecting a decoded body.
+    test -n "${WARC_VALIDATE_VERBATIM:-}" && bodyargs+=(--verbatim)
     info "validating fresh WARC (response bodies)"
     "$python" "$validator" "$(nativepath "$fresh")" "${bodyargs[@]}" >&2 ||
         die "fresh WARC validation failed"
@@ -398,6 +405,23 @@ if test -n "$warc_validate"; then
     fi
 fi
 
+# --- optional WACZ validation (--wacz) --------------------------------------
+if test -n "$wacz_validate"; then
+    wacz=$(find "$mirrorroot" -maxdepth 2 -name '*.wacz' 2>/dev/null | sort | tail -n1)
+    if test -z "$wacz"; then
+        # No package: only acceptable when the build lacks OpenSSL (SHA-256).
+        if grep -aqi "WACZ requires an OpenSSL" "${logroot}/hts-log.txt"; then
+            info "no .wacz produced (build without OpenSSL); skipping"
+            exit 77
+        fi
+        die "no .wacz file produced under $mirrorroot"
+    fi
+    validator=$(nativepath "${testdir}/wacz-validate.py")
+    info "validating WACZ package"
+    "$python" "$validator" "$(nativepath "$wacz")" >&2 || die "WACZ validation failed"
+    result "OK"
+fi
+
 # No crawl, even a cancelled one, may leave engine temporaries: .delayed (#107,
 # #483), or the .z/.u content-coding temps (#557).
 info "checking for leftover engine temporaries"

tests/local-server.py

@@ -635,6 +635,34 @@ class Handler(SimpleHTTPRequestHandler):
             extra_headers=[("Content-Encoding", "gzip")],
         )
 
+    # A gzip-coded HTML page whose decoded body is known, for the verbatim-WARC
+    # differential (stored compressed bytes must inflate to this).
+    WARCGZ_BODY = b"<html><body>verbatim gzip page for WARC strategy A</body></html>\n"
+
+    # A NON-html gzip-coded asset: HTTrack streams it straight to disk, so the
+    # verbatim spool adoption runs on the is_write (direct-to-disk) branch of
+    # back_finalize, not the in-memory branch route_warcgz_page exercises.
+    WARCGZ_BIN_BODY = b"verbatim gzip octet-stream body for the WARC is_write path\n"
+
+    def route_warcgz_index(self):
+        self.send_html(
+            '\t<a href="page.html">page</a>\n' '\t<a href="data.bin">data</a>\n'
+        )
+
+    def route_warcgz_page(self):
+        self.send_raw(
+            gzip.compress(self.WARCGZ_BODY),
+            "text/html",
+            extra_headers=[("Content-Encoding", "gzip")],
+        )
+
+    def route_warcgz_data(self):
+        self.send_raw(
+            gzip.compress(self.WARCGZ_BIN_BODY),
+            "application/octet-stream",
+            extra_headers=[("Content-Encoding", "gzip")],
+        )
+
     # --- content codings ---------------------------------------------------
     # Canned br/zstd bodies (no brotli/zstd module in the stdlib): both decode
     # to CODEC_BODY. Regenerate with the brotli/zstd CLIs over that string.
@@ -1542,6 +1570,9 @@ class Handler(SimpleHTTPRequestHandler):
         "/gated/index.php": route_gated_index,
         "/gated/secret.php": route_gated_secret,
         "/robots.txt": route_robots,
+        "/warcgz/index.html": route_warcgz_index,
+        "/warcgz/page.html": route_warcgz_page,
+        "/warcgz/data.bin": route_warcgz_data,
         "/codec/index.html": route_codec_index,
         "/codec/br.html": route_codec_br,
         "/codec/zstd.html": route_codec_zstd,

tests/wacz-validate.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# Validate a WACZ package with the stdlib only (no py-wacz needed): every entry
+# is ZIP STORE, the fixed layout is present, each datapackage resource hash and
+# size recomputes, the digest chains datapackage.json, and pages.jsonl carries
+# the json-pages-1.0 header. If py-wacz (`wacz validate`) is importable it runs
+# too; both gates must pass. Exit 0 = valid, nonzero = invalid.
+import sys
+import json
+import zipfile
+import hashlib
+
+
+def fail(msg):
+    print("wacz-validate: FAIL: %s" % msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def main():
+    if len(sys.argv) < 2:
+        fail("usage: wacz-validate.py FILE.wacz")
+    path = sys.argv[1]
+    z = zipfile.ZipFile(path)
+    names = z.namelist()
+
+    for info in z.infolist():
+        if info.compress_type != zipfile.ZIP_STORED:
+            fail("%s is not STORE mode (%d)" % (info.filename, info.compress_type))
+
+    need_arc = any(
+        n.startswith("archive/") and n.endswith((".warc.gz", ".warc")) for n in names
+    )
+    for req, ok in (
+        ("archive/*.warc.gz", need_arc),
+        ("indexes/index.cdx", "indexes/index.cdx" in names),
+        ("pages/pages.jsonl", "pages/pages.jsonl" in names),
+        ("datapackage.json", "datapackage.json" in names),
+        ("datapackage-digest.json", "datapackage-digest.json" in names),
+    ):
+        if not ok:
+            fail("missing %s (entries: %s)" % (req, names))
+
+    lines = [ln for ln in z.read("pages/pages.jsonl").split(b"\n") if ln.strip()]
+    if not lines or json.loads(lines[0]).get("format") != "json-pages-1.0":
+        fail("pages.jsonl header is not json-pages-1.0: %r" % lines[:1])
+    body = [json.loads(ln) for ln in lines[1:]]
+    if not body:
+        fail("pages.jsonl has no page rows after the header")
+    for row in body:
+        if "url" not in row or "ts" not in row:
+            fail("pages.jsonl row missing url/ts: %r" % row)
+
+    dp = json.loads(z.read("datapackage.json"))
+    if dp.get("profile") != "data-package":
+        fail("profile != data-package: %r" % dp.get("profile"))
+    if dp.get("wacz_version") != "1.1.1":
+        fail("wacz_version != 1.1.1: %r" % dp.get("wacz_version"))
+    resources = dp.get("resources", [])
+    if not resources:
+        fail("datapackage has no resources")
+    for r in resources:
+        data = z.read(r["path"])
+        h = "sha256:" + hashlib.sha256(data).hexdigest()
+        if h != r["hash"]:
+            fail("%s hash %s != %s" % (r["path"], h, r["hash"]))
+        if len(data) != r["bytes"]:
+            fail("%s bytes %d != %d" % (r["path"], len(data), r["bytes"]))
+
+    dig = json.loads(z.read("datapackage-digest.json"))
+    want = "sha256:" + hashlib.sha256(z.read("datapackage.json")).hexdigest()
+    if dig.get("path") != "datapackage.json" or dig.get("hash") != want:
+        fail("digest chain broken: %r" % dig)
+
+    print(
+        "wacz-validate: OK (%d entries, %d resources, stdlib)"
+        % (len(names), len(resources))
+    )
+
+    # Optional stricter gate when py-wacz is present.
+    try:
+        import wacz  # noqa: F401
+    except Exception:
+        return
+    try:
+        from wacz.main import main as wacz_main  # type: ignore
+    except Exception:
+        return
+    print("wacz-validate: running py-wacz validate")
+    rc = wacz_main(["validate", "-f", path])
+    if rc not in (0, None):
+        fail("py-wacz validate returned %r" % rc)
+    print("wacz-validate: py-wacz OK")
+
+
+if __name__ == "__main__":
+    main()

tests/warc-validate.py

@@ -11,6 +11,11 @@
 #                             WARC-Payload-Digest matching sha1(body) when present
 #   --no-response-for SUB     the asset containing SUB must be a revisit: no
 #                             response may target it, and a revisit must
+#   --verbatim                compressed asset: --expect-body-hex instead keeps
+#                             Content-Encoding, checks the HTTP Content-Length is
+#                             the stored (compressed) length, asserts the stored
+#                             body inflates to HEX (the served plaintext), and
+#                             requires the payload digest when the file emits any
 import base64
 import hashlib
 import sys
@@ -61,14 +66,53 @@ def check_body(rec, http_hdr, body, sub, want):
             sys.exit("WARC-Payload-Digest mismatch for %s" % sub)
 
 
+def check_body_verbatim(rec, http_hdr, body, sub, want, digests_emitted):
+    """Verbatim: the stored body is the coded octets, Content-Encoding is kept,
+    the HTTP Content-Length equals the stored (compressed) length, inflating the
+    body yields the served plaintext, and the payload digest is over the coded
+    body. The differential: inflate(stored) == the body the server compressed."""
+    if b"Content-Encoding" not in http_hdr:
+        sys.exit("verbatim record for %s dropped Content-Encoding" % sub)
+    if b"Transfer-Encoding" in http_hdr:
+        sys.exit("verbatim record for %s kept Transfer-Encoding" % sub)
+    hcl = field(http_hdr, b"Content-Length")
+    if hcl is None or int(hcl) != len(body):
+        sys.exit("verbatim record for %s: HTTP Content-Length != stored body" % sub)
+    try:
+        decoded = zlib.decompress(body, zlib.MAX_WBITS | 16)
+    except Exception as exc:
+        sys.exit("verbatim record for %s: body did not inflate: %s" % (sub, exc))
+    if decoded != want:
+        sys.exit(
+            "verbatim decoded mismatch for %s: got %d bytes, expected %d"
+            % (sub, len(decoded), len(want))
+        )
+    pd = field(rec[: rec.find(b"\r\n\r\n")], b"WARC-Payload-Digest")
+    # Catch a regression that drops the digest on the verbatim path, but only
+    # when this file emits digests at all (an OpenSSL build; none otherwise).
+    if digests_emitted and pd is None:
+        sys.exit("verbatim record for %s: missing WARC-Payload-Digest" % sub)
+    if pd is not None and pd.startswith(b"sha1:"):
+        want_b32 = base64.b32encode(hashlib.sha1(body).digest()).decode("ascii")
+        if pd[5:].decode("ascii") != want_b32:
+            sys.exit("WARC-Payload-Digest (compressed) mismatch for %s" % sub)
+
+
 def main():
     argv = sys.argv[1:]
     expect_revisit = "--expect-revisit" in argv
+    verbatim = "--verbatim" in argv
     body_specs = [s.split("=", 1) for s in opt_values(argv, "--expect-body-hex")]
     no_resp = opt_values(argv, "--no-response-for")
     path = [a for a in argv if not a.startswith("--") and "=" not in a][0]
     data = open(path, "rb").read()
 
+    # digests are emitted only on an OpenSSL build; detect it once so --verbatim
+    # can require the payload digest exactly when the file carries any.
+    digests_emitted = any(
+        b"WARC-Payload-Digest" in r[: r.find(b"\r\n\r\n")] for r in records(data)
+    )
+
     total = revisits = responses = infos = 0
     body_hits = {sub: False for sub, _ in body_specs}
     revisit_hits = {sub: False for sub in no_resp}
@@ -106,7 +150,13 @@ def main():
             http_hdr, body = block[:bsep], block[bsep + 4 :]
             for sub, hexval in body_specs:
                 if sub.encode() in uri:
-                    check_body(rec, http_hdr, body, sub, bytes.fromhex(hexval))
+                    want = bytes.fromhex(hexval)
+                    if verbatim:
+                        check_body_verbatim(
+                            rec, http_hdr, body, sub, want, digests_emitted
+                        )
+                    else:
+                        check_body(rec, http_hdr, body, sub, want)
                     body_hits[sub] = True
         elif wtype == b"revisit":
             revisits += 1

tests/webhttrack-smoke.sh

@@ -46,9 +46,12 @@ cat >"$stubdir/x-www-browser" <<EOF
 echo "stub browser invoked with: \$1" >&2
 # Also fetch an option page and require a rendered title='' tooltip: proves the
 # option template expands and the \${html:} filter escapes into the attribute.
+# option9 additionally proves the WARC control renders with its expanded label.
 opturl="\${1%/}/server/option2.html"
+warcurl="\${1%/}/server/option9.html"
 if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html &&
-    opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='"; then
+    opt="\$(curl -fsSL --max-time 20 "\$opturl")" && printf '%s' "\$opt" | grep -qaF "title='" &&
+    warc="\$(curl -fsSL --max-time 20 "\$warcurl")" && printf '%s' "\$warc" | grep -qaF 'name="warcfile"' && printf '%s' "\$warc" | grep -qaF WARC; then
     echo PASS >"$marker"
 else
     echo "FAIL: unexpected response from \$1" >"$marker"