Range-check --warc-max-size and regenerate the man page

Replace the unchecked sscanf on the --warc-max-size argument with a
strtoll parse that rejects non-numeric, negative, and overflowing values,
leaving the default 0 (single archive) intact. The man page and its HTML
render were already regenerated for the option's help line in the feature
commit, so this only tightens the parse.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

html/httrack.man.html

@@ -655,7 +655,8 @@ don&rsquo;t wait) (--cached-delayed-type-check)</p></td></tr>
 
 
 <p>write an ISO-28500 WARC/1.1 archive; --warc-file NAME
-sets the output name (--warc)</p></td></tr>
+sets the output name, --warc-max-size N rotates segments
+past N bytes (--warc)</p></td></tr>
 <tr valign="top" align="left">
 <td width="9%"></td>
 <td width="4%">

man/httrack.1

@@ -199,7 +199,7 @@ cached delayed type check, don't wait for remote type during updates, to speedup
 .IP \-%M
 generate a RFC MIME\-encapsulated full\-archive (.mht) (\-\-mime\-html)
 .IP \-%r
-write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name (\-\-warc)
+write an ISO\-28500 WARC/1.1 archive; \-\-warc\-file NAME sets the output name, \-\-warc\-max\-size N rotates segments past N bytes (\-\-warc)
 .IP \-%t
 keep the original file extension, don't rewrite it from the MIME type (%t0 rewrite)
 .IP \-LN

src/htsalias.c

@@ -116,6 +116,8 @@ const char *hts_optalias[][4] = {
    "load extra cookies from a Netscape cookies.txt"},
   {"warc", "-%r", "single", "write an ISO-28500 WARC/1.1 archive of the crawl"},
   {"warc-file", "-%rf", "param1", "write a WARC archive to the given base name"},
+  {"warc-max-size", "-%rs", "param1",
+   "rotate the WARC archive once a segment passes N bytes (0: single file)"},
   {"why", "-%Y", "param1",
    "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",

src/htsback.c

@@ -1062,6 +1062,7 @@ void back_copy_static(const lien_back * src, lien_back * dst) {
   dst->r.headers = NULL;
   dst->r.warc_reqhdr = NULL;
   dst->r.warc_resphdr = NULL;
+  dst->r.warc_truncated = 0;
   dst->r.out = NULL;
   dst->r.location = dst->location_buffer;
   dst->r.fp = NULL;
@@ -1127,6 +1128,7 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.out = NULL;
     (*dst)->r.warc_reqhdr = NULL;
     (*dst)->r.warc_resphdr = NULL;
+    (*dst)->r.warc_truncated = 0;
     (*dst)->r.location = (*dst)->location_buffer;
     (*dst)->r.fp = NULL;
     (*dst)->r.soc = INVALID_SOCKET;
@@ -2519,6 +2521,19 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     for (i = 0; i < (unsigned int) back_max; i++) {
       if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        /* A cap-truncated body is deliberate, not broken: archive what arrived
+           with WARC-Truncated before the abort overwrites the slot's real 2xx
+           status. HTTrack still treats the slot as incomplete afterwards. */
+        if (StringNotEmpty(opt->warc_file) && back[i].r.statuscode > 0 &&
+            back[i].r.warc_resphdr != NULL && back[i].r.size > 0 &&
+            !(back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))) {
+          if (back[i].r.is_write && back[i].r.out != NULL)
+            fflush(back[i].r.out);
+          back[i].r.warc_truncated = (limit == HTS_MIRROR_LIMIT_SIZE)
+                                         ? WARC_TRUNC_LENGTH
+                                         : WARC_TRUNC_TIME;
+          warc_write_backtransaction(opt, &back[i]);
+        }
         if (back[i].r.soc != INVALID_SOCKET) {
           deletehttp(&back[i].r);
         }

src/htscore.c

@@ -3632,6 +3632,7 @@ HTSEXT_API int copy_htsopt(const httrackp * from, httrackp * to) {
 
   if (StringNotEmpty(from->warc_file))
     StringCopyS(to->warc_file, from->warc_file);
+  to->warc_max_size = from->warc_max_size;
 
   if (from->pause_max_ms > 0) {
     to->pause_min_ms = from->pause_min_ms;

src/htscoremain.c

@@ -1806,6 +1806,25 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     return -1;
                   }
                   StringCopy(opt->warc_file, argv[na]);
+                } else if (*(com + 1) == 's') { // --warc-max-size N: rotation
+                  com++;
+                  if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                    HTS_PANIC_PRINTF(
+                        "Option warc-max-size needs a blank space and a size");
+                    htsmain_free();
+                    return -1;
+                  }
+                  na++;
+                  { // reject non-numeric/negative/overflow; keep default 0
+                    // (single file)
+                    char *end;
+                    LLint v;
+                    errno = 0;
+                    v = strtoll(argv[na], &end, 10);
+                    if (isdigit((unsigned char) argv[na][0]) && *end == '\0' &&
+                        errno != ERANGE)
+                      opt->warc_max_size = v;
+                  }
                 } else { // --warc: auto-named archive under the output dir
                   StringCopy(opt->warc_file, WARC_AUTONAME);
                 }

src/htshelp.c

@@ -525,7 +525,7 @@ void help(const char *app, int more) {
     (" %D  cached delayed type check, don't wait for remote type during updates, to speedup them (%D0 wait, * %D1 don't wait)");
   infomsg(" %M  generate a RFC MIME-encapsulated full-archive (.mht)");
   infomsg(" %r  write an ISO-28500 WARC/1.1 archive; --warc-file NAME sets the "
-          "output name");
+          "output name, --warc-max-size N rotates segments past N bytes");
   infomsg(" %t  keep the original file extension, don't rewrite it from the "
           "MIME type (%t0 rewrite)");
   infomsg

src/htslib.c

@@ -6019,6 +6019,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
   StringCopy(opt->warc_file, "");
+  opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
   StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;

src/htsopt.h

@@ -541,6 +541,8 @@ struct httrackp {
                          and exit without crawling */
   String warc_file; /**< WARC output: WARC_AUTONAME for --warc, or the
                          --warc-file basename (appended at the tail: ABI) */
+  LLint warc_max_size; /**< --warc-max-size: rotate the archive past this many
+                            bytes (<=0: single file). Tail: ABI */
 };
 
 /* Running statistics for a mirror. */
@@ -667,6 +669,8 @@ struct htsblk {
   char *warc_reqhdr; /**< stashed raw request header block for WARC (or NULL) */
   char *
       warc_resphdr; /**< stashed raw response header block for WARC (or NULL) */
+  int warc_truncated; /**< WARC-Truncated reason for a cap-truncated body
+                           (WARC_TRUNC_*, 0=none). Tail: ABI */
   /*char digest[32+2];   // md5 digest generated by the engine ("" if none) */
 };
 

src/htsselftest.c

@@ -3381,14 +3381,14 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
       "GET /a.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Encoding: "
       "gzip\r\nTransfer-Encoding: chunked\r\nContent-Length: 999\r\n\r\n",
-      a_body, sizeof(a_body) - 1, NULL, 200, 0);
+      a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
 
   /* 302 redirect: header-only, no body. */
   warc_write_transaction(
       w, "http://test.local/r", "127.0.0.1",
       "GET /r HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 302 Found\r\nLocation: http://test.local/a.html\r\n\r\n", NULL,
-      0, NULL, 302, 0);
+      0, NULL, 302, 0, 0);
 
   /* 200 binary, chunked coding on the wire (already de-chunked here). */
   warc_write_transaction(
@@ -3396,7 +3396,7 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
       "GET /b.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n"
       "Transfer-Encoding: chunked\r\n\r\n",
-      "\x00\x01\x02\x03\x04", 5, NULL, 200, 0);
+      "\x00\x01\x02\x03\x04", 5, NULL, 200, 0, 0);
 
   /* 200 with a body shorter than the declared Content-Length (rewritten). */
   warc_write_transaction(
@@ -3404,20 +3404,20 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
       "GET /trunc HTTP/1.1\r\nHost: test.local\r\n\r\n",
       "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: "
       "100\r\n\r\n",
-      "short", 5, NULL, 200, 0);
+      "short", 5, NULL, 200, 0, 0);
 
   /* Same payload as a.html at a new URL: identical-payload-digest revisit
      (OpenSSL builds only; a plain build writes a second full response). */
   warc_write_transaction(w, "http://test.local/a2.html", "127.0.0.1",
                          "GET /a2.html HTTP/1.1\r\nHost: test.local\r\n\r\n",
                          "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n",
-                         a_body, sizeof(a_body) - 1, NULL, 200, 0);
+                         a_body, sizeof(a_body) - 1, NULL, 200, 0, 0);
 
   /* 304 revisit with an EMPTY response-header block: the block is just the
      2-byte separator, so declared Content-Length must be exactly 2 (F3). */
   warc_write_transaction(w, "http://test.local/nm", "127.0.0.1",
                          "GET /nm HTTP/1.1\r\nHost: test.local\r\n\r\n", "",
-                         NULL, 0, NULL, 304, 1);
+                         NULL, 0, NULL, 304, 1, 0);
 
   warc_close(w);
 
@@ -3534,6 +3534,228 @@ static int st_warc(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* Parse a record's header/block split; sets *hdr_len and *block_len, returns 0
+   when Content-Length matches the actual block bytes, -1 otherwise. */
+static int warc_rec_split(const unsigned char *rec, size_t rlen,
+                          size_t *hdr_len, long long *block_len) {
+  const char *sep = warc_memstr((const char *) rec, "\r\n\r\n", rlen, 4);
+  const char *cl;
+  *block_len = 0;
+  if (sep == NULL)
+    return -1;
+  *hdr_len = (size_t) ((const unsigned char *) sep - rec) + 4;
+  cl = warc_memstr((const char *) rec, "Content-Length:", *hdr_len, 15);
+  if (cl == NULL || sscanf(cl + 15, "%lld", block_len) != 1 ||
+      *hdr_len + (size_t) *block_len + 4 != rlen)
+    return -1;
+  return 0;
+}
+
+/* A cap-truncated body is still archived, tagged WARC-Truncated (v1.1). Assert
+   the sole response carries "WARC-Truncated: length" and every record parses.
+ */
+static int st_warc_trunc(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  warc_writer *w;
+  unsigned char *data;
+  size_t data_len = 0;
+  const unsigned char *p, *end;
+  int err = 0, found_trunc = 0, nresp = 0;
+  static const char body[] = "partial body bytes\n";
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-trunc: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-trunc.warc.gz");
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  warc_write_transaction(
+      w, "http://test.local/big.bin", "127.0.0.1",
+      "GET /big.bin HTTP/1.1\r\nHost: test.local\r\n\r\n",
+      "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n", body,
+      sizeof(body) - 1, NULL, 200, 0, WARC_TRUNC_LENGTH);
+  warc_close(w);
+
+  data = warc_slurp(path, &data_len);
+  assertf(data != NULL);
+  p = data;
+  end = data + data_len;
+  while (p < end) {
+    size_t rlen = 0, hdr_len = 0;
+    long long block_len = 0;
+    unsigned char *rec = warc_next_member(&p, end, &rlen);
+    if (rec == NULL) {
+      if (rlen != 0)
+        err = 1;
+      break;
+    }
+    if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
+      err = 1;
+    else if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) !=
+             NULL) {
+      nresp++;
+      if (warc_memstr((char *) rec, "WARC-Truncated: length", hdr_len, 22) !=
+          NULL)
+        found_trunc = 1;
+    }
+    freet(rec);
+  }
+  freet(data);
+  if (!found_trunc || nresp != 1)
+    err = 1;
+  printf("warc-trunc: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+/* An ftp:// capture is ONE resource record: WARC-Type: resource, the payload's
+   own Content-Type, block == payload, and no request/response pair. */
+static int st_warc_ftp(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  warc_writer *w;
+  unsigned char *data;
+  size_t data_len = 0;
+  const unsigned char *p, *end;
+  int err = 0, nresource = 0, nresp = 0, nreq = 0;
+  static const char body[] = "\x00\x01"
+                             "FTP payload"
+                             "\x02\x03";
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-ftp: needs a writable directory\n");
+    return 1;
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-ftp.warc.gz");
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  warc_write_resource(w, "ftp://ftp.local/file.bin", "127.0.0.1",
+                      "application/octet-stream", body, sizeof(body) - 1, NULL,
+                      0);
+  warc_close(w);
+
+  data = warc_slurp(path, &data_len);
+  assertf(data != NULL);
+  p = data;
+  end = data + data_len;
+  while (p < end) {
+    size_t rlen = 0, hdr_len = 0;
+    long long block_len = 0;
+    unsigned char *rec = warc_next_member(&p, end, &rlen);
+    if (rec == NULL) {
+      if (rlen != 0)
+        err = 1;
+      break;
+    }
+    if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
+      err = 1;
+    if (warc_memstr((char *) rec, "WARC-Type: resource", hdr_len, 19) != NULL) {
+      nresource++;
+      if ((size_t) block_len != sizeof(body) - 1 ||
+          memcmp(rec + hdr_len, body, sizeof(body) - 1) != 0)
+        err = 1; /* block is the raw payload, no HTTP envelope */
+      if (warc_memstr((char *) rec, "WARC-Target-URI: ftp://ftp.local/file.bin",
+                      hdr_len, 41) == NULL ||
+          warc_memstr((char *) rec, "Content-Type: application/octet-stream",
+                      hdr_len, 38) == NULL)
+        err = 1;
+    }
+    if (warc_memstr((char *) rec, "WARC-Type: response", hdr_len, 19) != NULL)
+      nresp++;
+    if (warc_memstr((char *) rec, "WARC-Type: request", hdr_len, 18) != NULL)
+      nreq++;
+    freet(rec);
+  }
+  freet(data);
+  if (nresource != 1 || nresp != 0 || nreq != 0)
+    err = 1;
+  printf("warc-ftp: resource=%d response=%d request=%d: %s\n", nresource, nresp,
+         nreq, err ? "FAIL" : "OK");
+  return err;
+}
+
+/* --warc-max-size rotates into <base>-00000.warc.gz, -00001, ...; each segment
+   is independently valid and begins with its own warcinfo. */
+static int st_warc_rotate(httrackp *opt, int argc, char **argv) {
+  char path[HTS_URLMAXSIZE];
+  char seg[HTS_URLMAXSIZE];
+  warc_writer *w;
+  LLint saved_max;
+  unsigned char body[600];
+  unsigned int rng = 0x12345678u;
+  int err = 0, nseg = 0, i;
+  size_t j;
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-rotate: needs a writable directory\n");
+    return 1;
+  }
+  for (j = 0; j < sizeof(body);
+       j++) { /* incompressible: gzip can't shrink it */
+    rng ^= rng << 13;
+    rng ^= rng >> 17;
+    rng ^= rng << 5;
+    body[j] = (unsigned char) (rng >> 24);
+  }
+  fconcat(path, sizeof(path), argv[0], "warc-rot.warc.gz");
+  saved_max = opt->warc_max_size;
+  opt->warc_max_size =
+      1000; /* a couple records per segment => several segments */
+  w = warc_open(opt, path);
+  assertf(w != NULL);
+  for (i = 0; i < 8; i++) {
+    char uri[64];
+    snprintf(uri, sizeof(uri), "http://test.local/f%d.bin", i);
+    warc_write_transaction(
+        w, uri, "127.0.0.1", "GET / HTTP/1.1\r\nHost: test.local\r\n\r\n",
+        "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n",
+        (const char *) body, sizeof(body), NULL, 200, 0, 0);
+  }
+  warc_close(w);
+  opt->warc_max_size = saved_max;
+
+  for (i = 0;; i++) {
+    char fname[64];
+    unsigned char *data;
+    size_t data_len = 0;
+    const unsigned char *p, *pend;
+    int first = 1;
+    snprintf(fname, sizeof(fname), "warc-rot-%05d.warc.gz", i);
+    fconcat(seg, sizeof(seg), argv[0], fname);
+    data = warc_slurp(seg, &data_len);
+    if (data == NULL)
+      break; /* past the last segment */
+    nseg++;
+    p = data;
+    pend = data + data_len;
+    while (p < pend) {
+      size_t rlen = 0, hdr_len = 0;
+      long long block_len = 0;
+      unsigned char *rec = warc_next_member(&p, pend, &rlen);
+      if (rec == NULL) {
+        if (rlen != 0)
+          err = 1;
+        break;
+      }
+      if (warc_rec_split(rec, rlen, &hdr_len, &block_len) != 0)
+        err = 1;
+      if (first) { /* each segment leads with its own warcinfo */
+        if (warc_memstr((char *) rec, "WARC-Type: warcinfo", hdr_len, 19) ==
+            NULL)
+          err = 1;
+        first = 0;
+      }
+      freet(rec);
+    }
+    freet(data);
+    if (first) /* empty segment */
+      err = 1;
+  }
+  if (nseg < 2)
+    err = 1;
+  printf("warc-rotate: %d segments: %s\n", nseg, err ? "FAIL" : "OK");
+  return err;
+}
+
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -3655,6 +3877,12 @@ static const struct selftest_entry {
     {"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
     {"warc", "<dir>", "WARC/1.1 writer: framing, digests, revisit dedup",
      st_warc},
+    {"warc-trunc", "<dir>", "WARC-Truncated on a cap-truncated body",
+     st_warc_trunc},
+    {"warc-ftp", "<dir>", "ftp resource record (no HTTP envelope)",
+     st_warc_ftp},
+    {"warc-rotate", "<dir>", "--warc-max-size segment rotation",
+     st_warc_rotate},
 };
 
 static void list_selftests(void) {

src/htswarc.c

@@ -63,8 +63,27 @@ struct warc_writer {
   uint64_t rng;     /* PRNG state for the UUID fallback */
   char info_id[64]; /* warcinfo WARC-Record-ID, referenced by every record */
   coucal seen;      /* base32 payload digest -> "uri\001date" (revisit dedup) */
+  /* --warc-max-size rotation: NAME-00000.warc.gz, -00001, ... (wget-style). */
+  uint64_t max_size;   /* rotate once a segment reaches this; 0: single file */
+  char *seg_base;      /* segment path without the .warc[.gz] suffix, or NULL */
+  const char *seg_ext; /* ".warc.gz" or ".warc" */
+  unsigned seg;        /* current segment number */
+  char *info_fields;   /* warcinfo body, re-emitted at each new segment */
 };
 
+const char *warc_truncated_reason(int code) {
+  switch (code) {
+  case WARC_TRUNC_LENGTH:
+    return "length";
+  case WARC_TRUNC_TIME:
+    return "time";
+  case WARC_TRUNC_DISCONNECT:
+    return "disconnect";
+  default:
+    return NULL;
+  }
+}
+
 /* ---- growable byte buffer (overflow-safe, project allocators) ---- */
 
 typedef struct {
@@ -421,21 +440,25 @@ static int normalize_http_headers(const char *resp_hdr, long long set_cl,
   return 0;
 }
 
+/* Close the current segment and open the next; writes its warcinfo. */
+static int warc_rotate(warc_writer *w);
+
 /* Emit one full WARC record. When http_section, the block carries an HTTP
    header block (http_hdr, possibly empty) + a CRLF separator; body follows when
    has_body. block_len is derived here (single source of truth: separator and
    payload are counted exactly as stream_block emits them), so a declared
    Content-Length can never desync from the written bytes. The payload digest
-   (body-only) is passed in when already known. On success the record id is
-   copied to out_id (may be NULL). */
+   (body-only) is passed in when already known. truncated is a WARC-Truncated
+   reason token or NULL. On success the record id is copied to out_id (may be
+   NULL). */
 static int warc_emit(warc_writer *w, const char *type, const char *content_type,
                      const char *target_uri, const char *ip,
                      const char *concurrent_to, const char *refers_uri,
                      const char *refers_date, const char *profile,
-                     const char *payload_digest, int http_section,
-                     const char *http_hdr, size_t http_hdr_len, int has_body,
-                     const char *body, size_t body_len, const char *body_path,
-                     char out_id[64]) {
+                     const char *payload_digest, const char *truncated,
+                     int http_section, const char *http_hdr,
+                     size_t http_hdr_len, int has_body, const char *body,
+                     size_t body_len, const char *body_path, char out_id[64]) {
   wbuf hdr;
   member m;
   char id[64], date[32];
@@ -451,6 +474,14 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
   int have_block_digest = 0;
 #endif
 
+  /* Rotate to the next segment before this record when the current one is full;
+     never split a record, and never rotate a warcinfo (it opens a segment). */
+  if (w->max_size > 0 && w->seg_base != NULL && w->offset >= w->max_size &&
+      strcmp(type, "warcinfo") != 0) {
+    if (warc_rotate(w) != 0)
+      return -1;
+  }
+
   /* F4: overflow-safe block length; http_hdr_len+sep is provably small. */
   if (payload > (size_t) -1 - http_hdr_len - sep)
     return -1;
@@ -516,6 +547,9 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
       wbuf_printf(&hdr, "WARC-Payload-Digest: sha1:%s\r\n", payload_digest) !=
           0)
     goto done;
+  if (truncated != NULL &&
+      wbuf_printf(&hdr, "WARC-Truncated: %s\r\n", truncated) != 0)
+    goto done;
   if (wbuf_puts(&hdr, "\r\n") != 0)
     goto done;
 
@@ -544,6 +578,34 @@ done:
   return rc;
 }
 
+/* ---- segment rotation (--warc-max-size) ---- */
+
+/* Emit the warcinfo that heads a segment; sets w->info_id for its records. */
+static int warc_write_warcinfo_record(warc_writer *w) {
+  w->info_id[0] = '\0'; /* warcinfo itself carries no WARC-Warcinfo-ID */
+  return warc_emit(w, "warcinfo", "application/warc-fields", NULL, NULL, NULL,
+                   NULL, NULL, NULL, NULL, NULL, 0, NULL, 0, 1, w->info_fields,
+                   w->info_fields != NULL ? strlen(w->info_fields) : 0, NULL,
+                   w->info_id);
+}
+
+static int warc_rotate(warc_writer *w) {
+  char namebuf[HTS_URLMAXSIZE * 2];
+  char catbuff[CATBUFF_SIZE];
+  if (w->f != NULL) {
+    fclose(w->f);
+    w->f = NULL;
+  }
+  w->seg++;
+  snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
+           w->seg_ext);
+  w->f = FOPEN(fconv(catbuff, sizeof(catbuff), namebuf), "wb");
+  if (w->f == NULL)
+    return -1;
+  w->offset = 0;
+  return warc_write_warcinfo_record(w);
+}
+
 /* ---- request stash (engine hooks) ---- */
 
 void warc_stash_request(htsblk *r, const char *reqhdr) {
@@ -631,15 +693,9 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
   w->seen = coucal_new(0);
   if (w->seen != NULL)
     coucal_value_is_malloc(w->seen, 1);
+  w->max_size = (opt->warc_max_size > 0) ? (uint64_t) opt->warc_max_size : 0;
 
-  w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
-  if (w->f == NULL) {
-    if (w->seen != NULL)
-      coucal_delete(&w->seen);
-    freet(w);
-    return NULL;
-  }
-
+  /* Build the warcinfo body once; each segment re-emits it. */
   robots = (opt->robots == HTS_ROBOTS_NEVER) ? "ignore" : "obey";
   memset(&info, 0, sizeof(info));
   if (wbuf_printf(&info,
@@ -651,20 +707,57 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
                   HTTRACK_VERSION, robots) != 0 ||
       (StringNotEmpty(opt->path_html) &&
        wbuf_printf(&info, "isPartOf: %s\r\n", StringBuff(opt->path_html)) !=
-           0)) {
-    wbuf_free(&info);
-    warc_close(w);
-    return NULL;
-  }
-  w->info_id[0] = '\0'; /* not yet known: warcinfo omits WARC-Warcinfo-ID */
-  if (warc_emit(w, "warcinfo", "application/warc-fields", NULL, NULL, NULL,
-                NULL, NULL, NULL, NULL, 0, NULL, 0, 1, info.data, info.len,
-                NULL, w->info_id) != 0) {
+           0) ||
+      wbuf_add(&info, "", 1) != 0) { /* NUL-terminate for info_fields */
     wbuf_free(&info);
     warc_close(w);
     return NULL;
   }
+  w->info_fields = strdupt(info.data);
   wbuf_free(&info);
+  if (w->info_fields == NULL) {
+    warc_close(w);
+    return NULL;
+  }
+
+  /* Rotation on: the first segment is <base>-00000<ext> (wget-style); split the
+     resolved path into base + suffix so later segments reuse the base. */
+  if (w->max_size > 0) {
+    size_t l = strlen(path);
+    size_t baselen;
+    if (l >= 8 && strcasecmp(path + l - 8, ".warc.gz") == 0) {
+      baselen = l - 8;
+      w->seg_ext = ".warc.gz";
+    } else if (l >= 5 && strcasecmp(path + l - 5, ".warc") == 0) {
+      baselen = l - 5;
+      w->seg_ext = ".warc";
+    } else {
+      baselen = l;
+      w->seg_ext = w->gz ? ".warc.gz" : ".warc";
+    }
+    w->seg_base = malloct(baselen + 1);
+    if (w->seg_base == NULL) {
+      warc_close(w);
+      return NULL;
+    }
+    memcpy(w->seg_base, path, baselen);
+    w->seg_base[baselen] = '\0';
+    w->seg = 0;
+    snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
+             w->seg_ext);
+    path = namebuf;
+  }
+
+  w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
+  if (w->f == NULL) {
+    warc_close(w);
+    return NULL;
+  }
+
+  if (warc_write_warcinfo_record(w) != 0) {
+    warc_close(w);
+    return NULL;
+  }
   return w;
 }
 
@@ -675,6 +768,8 @@ void warc_close(warc_writer *w) {
     fclose(w->f);
   if (w->seen != NULL)
     coucal_delete(&w->seen);
+  freet(w->seg_base);
+  freet(w->info_fields);
   freet(w);
 }
 
@@ -691,7 +786,8 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *ip, const char *req_hdr,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
-                           int statuscode, int is_update_unchanged) {
+                           int statuscode, int is_update_unchanged,
+                           int truncated) {
   wbuf http;
   char resp_id[64];
   char pdig[33];
@@ -752,11 +848,14 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
     return -1;
   }
 
-  /* Response first: its id links the request via WARC-Concurrent-To. */
+  /* Response first: its id links the request via WARC-Concurrent-To. A revisit
+     is a deliberate dedup, not a truncation, so tag WARC-Truncated only on a
+     full (body-carrying) response. */
   resp_id[0] = '\0';
   if (warc_emit(w, is_revisit ? "revisit" : "response",
                 "application/http;msgtype=response", target_uri, ip, NULL,
-                refers_uri, refers_date, profile, have_pdig ? pdig : NULL, 1,
+                refers_uri, refers_date, profile, have_pdig ? pdig : NULL,
+                emit_body ? warc_truncated_reason(truncated) : NULL, 1,
                 http.data, http.len, emit_body, body, body_len, body_path,
                 resp_id) != 0) {
     wbuf_free(&http);
@@ -767,8 +866,8 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
   if (req_hdr != NULL && req_hdr[0] != '\0') {
     size_t rlen = strlen(req_hdr);
     if (warc_emit(w, "request", "application/http;msgtype=request", target_uri,
-                  NULL, resp_id, NULL, NULL, NULL, NULL, 0, NULL, 0, 1, req_hdr,
-                  rlen, NULL, NULL) != 0)
+                  NULL, resp_id, NULL, NULL, NULL, NULL, NULL, 0, NULL, 0, 1,
+                  req_hdr, rlen, NULL, NULL) != 0)
       return -1;
   }
 
@@ -793,6 +892,26 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
   return rc;
 }
 
+/* ---- one non-HTTP capture ---- */
+
+int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
+                        const char *content_type, const char *body,
+                        size_t body_len, const char *body_path, int truncated) {
+  char pdig[33];
+  int has_body = (body_len > 0 && (body != NULL || body_path != NULL));
+  int have_pdig =
+      has_body ? payload_digest_b32(body, body_len, body_path, pdig) : 0;
+  /* resource: the block is the raw payload, its own MIME is the record's
+     Content-Type, and there is no HTTP request/response envelope. */
+  return warc_emit(w, "resource",
+                   (content_type != NULL && content_type[0] != '\0')
+                       ? content_type
+                       : "application/octet-stream",
+                   target_uri, ip, NULL, NULL, NULL, NULL,
+                   have_pdig ? pdig : NULL, warc_truncated_reason(truncated), 0,
+                   NULL, 0, has_body, body, body_len, body_path, NULL);
+}
+
 /* ---- engine emit hook ---- */
 
 void warc_write_backtransaction(httrackp *opt, lien_back *back) {
@@ -805,6 +924,7 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
   const char *resp_hdr;
   char synth[512];
   int is_unchanged;
+  int is_ftp;
 
   if (opt->state.warc == WARC_DISABLED)
     return;
@@ -823,6 +943,7 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
   if (back->r.statuscode <= 0)
     return;
 
+  is_ftp = strfield(back->url_adr, "ftp://") != 0;
   snprintf(uri, sizeof(uri), "%s%s%s",
            link_has_authority(back->url_adr) ? "" : "http://", back->url_adr,
            back->url_fil);
@@ -847,6 +968,13 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
       body_len = 0;
   }
 
+  /* FTP has no HTTP envelope: one resource record carrying the payload. */
+  if (is_ftp) {
+    warc_write_resource(w, uri, ip, back->r.contenttype, body, body_len,
+                        body_path, back->r.warc_truncated);
+    return;
+  }
+
   is_unchanged = (back->r.notmodified && opt->is_update) ? 1 : 0;
 
   /* Prefer the stashed raw headers; synthesize a minimal status line for the
@@ -862,5 +990,6 @@ void warc_write_backtransaction(httrackp *opt, lien_back *back) {
   }
 
   warc_write_transaction(w, uri, ip, back->r.warc_reqhdr, resp_hdr, body,
-                         body_len, body_path, back->r.statuscode, is_unchanged);
+                         body_len, body_path, back->r.statuscode, is_unchanged,
+                         back->r.warc_truncated);
 }

src/htswarc.h

@@ -45,6 +45,16 @@ extern "C" {
    under the project's output directory at open time. */
 #define WARC_AUTONAME "\001auto"
 
+/* htsblk.warc_truncated / WARC-Truncated reason tokens (ISO 28500 sec 5.13).
+   A cap-truncated body is still archived, tagged with why it was cut short. */
+#define WARC_TRUNC_NONE 0
+#define WARC_TRUNC_LENGTH 1     /* hit a size cap (-M mirror / -m per-file) */
+#define WARC_TRUNC_TIME 2       /* hit the mirror time cap (-E/--max-time) */
+#define WARC_TRUNC_DISCONNECT 3 /* connection dropped mid-body */
+
+/* WARC-Truncated token for a warc_truncated code, or NULL for none. */
+const char *warc_truncated_reason(int code);
+
 typedef struct warc_writer warc_writer;
 
 /* Stash the raw request header block (bstr.buffer) on r for the later WARC
@@ -84,12 +94,22 @@ void warc_close(warc_writer *w);
    body/body_len: decoded in-memory body, or NULL when on disk.
    body_path:   file re-read for the body when body==NULL (may be NULL).
    is_update_unchanged: nonzero for a 304 server-not-modified revisit.
+   truncated: a WARC_TRUNC_* reason to tag a cap-truncated body, else 0.
    Returns 0 on success, -1 on error. */
 int warc_write_transaction(warc_writer *w, const char *target_uri,
                            const char *ip, const char *req_hdr,
                            const char *resp_hdr, const char *body,
                            size_t body_len, const char *body_path,
-                           int statuscode, int is_update_unchanged);
+                           int statuscode, int is_update_unchanged,
+                           int truncated);
+
+/* Write one non-HTTP capture as a single WARC 'resource' record: the block is
+   the raw payload (no HTTP envelope), Content-Type is the payload's own MIME.
+   Used for ftp:// transfers. truncated is a WARC_TRUNC_* reason or 0.
+   Returns 0 on success, -1 on error. */
+int warc_write_resource(warc_writer *w, const char *target_uri, const char *ip,
+                        const char *content_type, const char *body,
+                        size_t body_len, const char *body_path, int truncated);
 
 #ifdef __cplusplus
 }

tests/01_zlib-warc.test

@@ -4,16 +4,19 @@
 set -euo pipefail
 
 # WARC/1.1 writer self-test: framing, Content-Length, gzip members, round-trip,
-# and identical-payload-digest revisit dedup, over synthetic transactions.
+# revisit dedup, plus v1.1 WARC-Truncated, ftp resource records, and
+# --warc-max-size rotation, over synthetic transactions.
 
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
 trap 'rm -rf "$scratch"' EXIT
 
-out=$("$httrack_bin" -O /dev/null -#test=warc "$scratch/")
-echo "$out"
-case "$out" in
-*": OK") ;;
-*) exit 1 ;;
-esac
+for t in warc warc-trunc warc-ftp warc-rotate; do
+    out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")
+    echo "$out"
+    case "$out" in
+    *": OK") ;;
+    *) exit 1 ;;
+    esac
+done