st_cookies: assert the saved jar is non-empty

The mode assertions alone would pass a cookie_save that creates an
empty 0600 file and returns 0; check st_size on both saves (proven by
a negative control that skips the write loop).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/codeql.yml

@@ -43,6 +43,12 @@ jobs:
           languages: c-cpp
           build-mode: manual
           queries: security-extended
+          # fopen's umask-controlled 0666 is intended for mirror/cache/log
+          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
+          config: |
+            query-filters:
+              - exclude:
+                  id: cpp/world-writable-file-creation
 
       # Manual build: CodeQL traces the compiler, so build exactly what ships.
       - name: Build

fuzz/fuzz-cachendx.c

@@ -27,7 +27,7 @@ Please visit our Website: http://www.httrack.com
 
 /* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
    walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the -#C cache-listing scan (htscoremain.c). */
+   Mirrors the loader in cache_readex_new (htscache.c). */
 #include "fuzz.h"
 #include "htscache.h"
 #include "htslib.h"
@@ -42,8 +42,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   a += cache_brstr(a, firstline, sizeof(firstline));
   a += cache_brstr(a, firstline, sizeof(firstline));
 
-  /* body: newline-delimited host/file/position triples; the length-prefixed
+  /* body: newline-delimited host/file/position triples. The coucal insert
-     scan must stay inside the buffer */
+     the real loader ends with is a separate library's concern; this harness
+     targets the length-prefixed scan that must stay inside the buffer. */
   while (a != NULL && a < end) {
     char BIGSTK line[HTS_URLMAXSIZE * 2];
     char linepos[256];

src/htsbauth.c

@@ -307,14 +307,25 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-// écrire cookies.txt
+/* Write cookies.txt; returns 0 on success. The jar holds live session
-// !=0 : erreur
+   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
+#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
+#else
+    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
+                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
+    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
+
+    if (fd != -1 && fp == NULL)
+      close(fd);    /* fdopen failed: don't leak the descriptor */
+    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
+      (void) fchmod(fd, HTS_PROTECT_FILE);
+#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -89,6 +89,11 @@ typedef enum {
    the involved files are absent. */
 void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
+int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
+                    const char *str2, char *outbuff, int len);
+int cache_readdata(cache_back * cache, const char *str1, const char *str2,
+                   char **inbuff, int *len);
+
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);

src/htscache_selftest.c

@@ -78,6 +78,14 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
+  if (cache->dat != NULL) {
+    fclose(cache->dat);
+    cache->dat = NULL;
+  }
+  if (cache->ndx != NULL) {
+    fclose(cache->ndx);
+    cache->ndx = NULL;
+  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -981,15 +989,26 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
 
-  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", TINY);
   hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
 
   /* INTERRUPTED: no lock file, no action */
   reconcile_wipe(opt);
@@ -1039,7 +1058,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
 
-  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-in_progress.lock", 0);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
@@ -1048,13 +1067,9 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
   failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
 
   /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
      its only good generation here) */
@@ -1074,18 +1089,17 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
 
-  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   reconcile_put(opt, "hts-cache/old.lst", MID);
   reconcile_put(opt, "hts-cache/old.txt", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
 
@@ -1099,101 +1113,6 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   return failures;
 }
 
-/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
-   legacy pair, leave the files in place, and not flag an update run. */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  int variant;
-
-  selftest_tag = "cache-legacy";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
-  for (variant = 0; variant < 2; variant++) {
-    cache_back cache;
-    const char *const dat =
-        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
-    const char *const ndx =
-        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
-
-    reconcile_wipe(opt);
-    reconcile_put(opt, dat, 4096);
-    reconcile_put(opt, ndx, 4096);
-    opt->is_update = 0;
-    selftest_open(&cache, opt, /*ro*/ variant == 1);
-    if (cache_readable(&cache)) {
-      printf("cache-legacy: variant %d imported a removed format\n", variant);
-      failures++;
-    }
-    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
-      printf("cache-legacy: variant %d imported index entries\n", variant);
-      failures++;
-    }
-    if (opt->is_update) {
-      printf("cache-legacy: variant %d flagged an update\n", variant);
-      failures++;
-    }
-    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
-        fsize(reconcile_st_path(opt, ndx)) != 4096) {
-      printf("cache-legacy: variant %d touched the legacy files\n", variant);
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  /* a healthy zip must win over stray legacy files, with no refusal */
-  {
-    cache_back cache;
-    const char *const body = "<html>zip wins</html>";
-
-    reconcile_wipe(opt);
-    selftest_open_for_write(&cache, opt);
-    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
-                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
-    selftest_close(&cache);
-    reconcile_put(opt, "hts-cache/old.ndx", 4096);
-    reconcile_put(opt, "hts-cache/old.dat", 4096);
-    selftest_open_for_read(&cache, opt);
-    if (!cache_readable(&cache)) {
-      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
-      failures++;
-    }
-    failures +=
-        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
-                    "utf-8", "", "", "", "", body, strlen(body));
-    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
-        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
-      printf("cache-legacy: zip-wins pass touched the legacy files\n");
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
 /* --- read-side corruption injection --------------------------------------- */
 
 /* canary read back intact after each corruption; victim gets the byte surgery

src/htscache_selftest.h

@@ -60,10 +60,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir);
    under <dir>. Returns the failed-check count. */
 int cache_reconcile_selftest(httrackp *opt, const char *dir);
 
-/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
-   Returns the number of failed checks (0 = pass). */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
-
 /* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
    under <dir> and assert every case degrades to STATUSCODE_INVALID without
    tainting a sibling entry. */

src/htscore.c

@@ -138,6 +138,14 @@ RUN_CALLBACK0(opt, end); \
       freet(cache.use);                                                        \
       cache.use = NULL;                                                        \
     }                                                                          \
+    if (cache.dat) {                                                           \
+      fclose(cache.dat);                                                       \
+      cache.dat = NULL;                                                        \
+    }                                                                          \
+    if (cache.ndx) {                                                           \
+      fclose(cache.ndx);                                                       \
+      cache.ndx = NULL;                                                        \
+    }                                                                          \
     if (cache.zipOutput) {                                                     \
       zipClose(cache.zipOutput,                                                \
                "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
@@ -147,6 +155,10 @@ RUN_CALLBACK0(opt, end); \
       unzClose(cache.zipInput);                                                \
       cache.zipInput = NULL;                                                   \
     }                                                                          \
+    if (cache.olddat) {                                                        \
+      fclose(cache.olddat);                                                    \
+      cache.olddat = NULL;                                                     \
+    }                                                                          \
     if (cache.lst) {                                                           \
       fclose(cache.lst);                                                       \
       cache.lst = opt->state.strc.lst = NULL;                                  \
@@ -476,11 +488,32 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk. */
+/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
+ * on-disk MD5 is unchanged. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  {
+  char digest[32 + 2];
+  off_t fsize_old =
+      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
+  int ok = 0;
+
+  digest[0] = '\0';
+  domd5mem(ht_buff, ht_len, digest, 1);
+  if (fsize_old == (off_t) ht_len) {
+    int mlen = 0;
+    char *mbuff;
+
+    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
+    if (mlen)
+      mbuff[mlen] = '\0';
+    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
+      ok = 1;
+      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
+    }
+    freet(mbuff);
+  }
+  if (!ok) {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -512,7 +545,13 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
+  } else {
+    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
+    filenote(&opt->state.strc, save, NULL);
   }
+  if (cache->ndx)
+    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
+                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */

src/htscore.h

@@ -193,7 +193,7 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-
+  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
   char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
@@ -288,12 +288,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && cache->zipOutput != NULL);
+  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && cache->zipInput != NULL);
+  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
 }
 
 #endif

src/htsglobal.h

@@ -422,9 +422,10 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
+   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
-   modes also grant group/other read; otherwise they stay owner-only. */
+   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
+#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htsselftest.c

@@ -1500,18 +1500,6 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
-  int err;
-
-  if (argc < 1) {
-    fprintf(stderr, "cache-legacy: needs a directory\n");
-    return 1;
-  }
-  err = cache_legacy_refused_selftest(opt, argv[0]);
-  printf("cache-legacy: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 static int st_reconcile(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -1563,6 +1551,26 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     err = 1;
   if (strstr(hdr, "junk") != NULL) // wrong-domain cookie leaked
     err = 1;
+#ifndef _WIN32
+  /* the jar holds live session cookies: cookie_save must keep it 0600 */
+  {
+    const char *jar = "st-cookies-jar.txt";
+    struct stat st;
+
+    (void) UNLINK(jar);
+    assertf(cookie_save(&cookie, jar) == 0);
+    assertf(stat(jar, &st) == 0);
+    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
+    assertf(st.st_size > 0); /* mode-only checks would pass an empty jar */
+    /* a pre-existing world-readable jar must be tightened, not kept */
+    assertf(chmod(jar, 0644) == 0);
+    assertf(cookie_save(&cookie, jar) == 0);
+    assertf(stat(jar, &st) == 0);
+    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
+    assertf(st.st_size > 0);
+    (void) UNLINK(jar);
+  }
+#endif
   printf("cookie-header: %s\n", err ? "FAIL" : "OK");
   if (err)
     printf("  got: %s\n", hdr);
@@ -2306,8 +2314,6 @@ static const struct selftest_entry {
      st_cache_writefail},
     {"reconcile", "<dir>", "cache generation reconcile policy self-test",
      st_reconcile},
-    {"cache-legacy", "<dir>", "pre-3.31 legacy cache refusal self-test",
-     st_cache_legacy},
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},

tests/01_zlib-cache-legacy.test

@@ -1,23 +0,0 @@
-#!/bin/bash
-#
-# The pre-3.31 .dat/.ndx cache import was removed: cache_init must refuse the
-# legacy pair and leave the files untouched (httrack -#test=cache-legacy <dir>).
-
-set -eu
-
-dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
-
-# the refusal errors land on stdout (no log file); pin them and the verdict
-out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)
-cnt=$(printf '%s\n' "$out" | grep -c 'no longer supported' || true)
-test "$cnt" = 2 || {
-    echo "expected 2 refusal messages, got $cnt" >&2
-    exit 1
-}
-out=$(printf '%s\n' "$out" | tail -n1)
-
-test "$out" = "cache-legacy: OK" || {
-    echo "expected 'cache-legacy: OK', got: $out" >&2
-    exit 1
-}

tests/Makefile.am

@@ -68,7 +68,6 @@ TESTS = \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
-	01_zlib-cache-legacy.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
 	01_zlib-savename-cached.test \