htsserver never returns from main(), it blocks on its own exit wait

background_threads counted two unrelated things: the pinger, which only ever
leaves through exit(), and each mirror thread, which does return. main() then
waited on background_threads - 1, so with no pinger it asked for a negative
number of outstanding threads and the counter cannot go there, and with one it
waited on the pinger itself. Either way htsserver hung instead of returning.

Count only what never returns, and let each wait name what it must leave
running: the pinger for main(), the pinger plus itself for back_launch_cmd().
Waiting on background_threads alone would have swapped the hang for an early
return, letting main() tear down while a mirror was still crawling.

Closes #753

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <xroche@gmail.com>

src/htscodec.c

@@ -72,7 +72,7 @@ hts_codec hts_codec_parse(const char *encoding) {
     return HTS_CODEC_IDENTITY;
   if (strfield2(encoding, "gzip") || strfield2(encoding, "x-gzip") ||
       strfield2(encoding, "deflate") || strfield2(encoding, "x-deflate"))
-    return HTS_CODEC_DEFLATE;
+    return HTS_USEZLIB ? HTS_CODEC_DEFLATE : HTS_CODEC_UNSUPPORTED;
   if (strfield2(encoding, "br"))
     return HTS_USEBROTLI ? HTS_CODEC_BROTLI : HTS_CODEC_UNSUPPORTED;
   if (strfield2(encoding, "zstd"))
@@ -98,11 +98,16 @@ hts_codec hts_codec_parse(const char *encoding) {
 const char *hts_acceptencoding(hts_boolean compressible, hts_boolean secure) {
   if (!compressible)
     return "identity";
+#if HTS_USEZLIB
   /* br and zstd over TLS only, as browsers do: a cleartext intermediary that
      rewrites a coding it can not read would corrupt the mirror. */
   if (secure)
     return "gzip, deflate" HTS_AE_BROTLI HTS_AE_ZSTD ", identity;q=0.9";
   return "gzip, deflate, identity;q=0.9";
+#else
+  (void) secure;
+  return "identity";
+#endif
 }
 
 hts_boolean hts_codec_is_archive_ext(hts_codec codec, const char *ext) {
@@ -295,7 +300,11 @@ int hts_codec_unpack(hts_codec codec, const char *filename,
     return -1;
   switch (codec) {
   case HTS_CODEC_DEFLATE:
+#if HTS_USEZLIB
     return hts_zunpack(filename, newfile);
+#else
+    return -1;
+#endif
   case HTS_CODEC_BROTLI:
   case HTS_CODEC_ZSTD:
     break;
@@ -339,8 +348,10 @@ size_t hts_codec_head(hts_codec codec, const void *in, size_t in_len, void *out,
   if (in == NULL || in_len == 0 || out == NULL || out_len == 0)
     return 0;
   switch (codec) {
+#if HTS_USEZLIB
   case HTS_CODEC_DEFLATE:
     return hts_zhead(in, in_len, out, out_len);
+#endif
 #if HTS_USEBROTLI
   case HTS_CODEC_BROTLI:
     return codec_head_brotli(in, in_len, out, out_len);

src/htslib.c

@@ -1131,10 +1131,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
 
       // Compression accepted ?
       if (retour->req.http11) {
-        hts_boolean compressible =
-            (!retour->req.range_used && !retour->req.nocompression);
+        hts_boolean compressible = HTS_FALSE;
         hts_boolean secure = HTS_FALSE;
 
+#if HTS_USEZLIB
+        compressible = (!retour->req.range_used && !retour->req.nocompression);
+#endif
 #if HTS_USEOPENSSL
         secure = retour->ssl ? HTS_TRUE : HTS_FALSE;
 #endif

src/htsname.c

@@ -43,7 +43,9 @@ Please visit our Website: http://www.httrack.com
 #include "htsencoding.h"
 #include "htssniff.h"
 #include "htscodec.h"
+#if HTS_USEZLIB
 #include "htszlib.h"
+#endif
 #include <ctype.h>
 #include <limits.h>
 

src/htsselftest.c

@@ -63,7 +63,9 @@ Please visit our Website: http://www.httrack.com
 #include "htswarc.h"
 #include "htschanges.h"
 #include "htssinglefile.h"
+#if HTS_USEZLIB
 #include "htszlib.h"
+#endif
 #if HTS_USEZSTD
 #include <zstd.h>
 #endif
@@ -3375,6 +3377,7 @@ static int st_status(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+#if HTS_USEZLIB
 /* Deflate src->path at windowBits (16+ gzip, + zlib, - raw); 0 on success. */
 static int ae_write_packed(const char *path, int windowBits,
                            const unsigned char *src, size_t len) {
@@ -3448,6 +3451,7 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   freet(buf);
   return ok ? 0 : 1;
 }
+#endif
 
 /* Write src[0..len) to path as-is; 0 on success. */
 static int ae_write_raw(const char *path, const unsigned char *src,
@@ -3497,6 +3501,7 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
   assertf(strstr(on, "br") == NULL && strstr(on, "zstd") == NULL);
   assertf((strstr(tls, ", br") != NULL) == (HTS_USEBROTLI != 0));
   assertf((strstr(tls, "zstd") != NULL) == (HTS_USEZSTD != 0));
+#if HTS_USEZLIB
   if (argc >= 1) {
     static const int windowBits[] = {16 + MAX_WBITS, MAX_WBITS, -MAX_WBITS};
     const unsigned char small[] =
@@ -3575,6 +3580,10 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     }
     freet(body);
   }
+#else
+  (void) argc;
+  (void) argv;
+#endif
   printf("acceptencoding self-test OK: %s\n", on);
   return 0;
 }
@@ -3919,6 +3928,7 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
     freet(big);
   }
 
+#if HTS_USEZLIB
   /* A highly compressible document decodes without running away: the ratio
      budget cannot bind (deflate tops out near 1032:1), so this pins the
      decompression path itself rather than the 64 MiB ceiling. */
@@ -3969,11 +3979,13 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
     freet(z);
     freet(x);
   }
+#endif
 
   /* An unterminated <loc> at end of buffer must not read past it. */
   assertf(sm_scan("<urlset><loc>http://h.test/a", 100, &idx, &c) == 0);
   assertf(sm_scan("<urlset><lo", 100, &idx, &c) == 0);
 
+#if HTS_USEZLIB
   /* A gzip-framed document is decompressed before scanning. */
   {
     const char *const xml =
@@ -4003,6 +4015,7 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
     assertf(hts_sitemap_scan(z, 4, 100, &idx, sm_take, &c) == -1);
     freet(z);
   }
+#endif
 
   /* robots.txt: only Sitemap: records, comments stripped, case-insensitive,
      and group-independent (no User-agent line needed). */

src/htsweb.c

@@ -101,8 +101,8 @@ static void htsweb_sig_brpipe(int code) {
   /* ignore */
 }
 
-/* Number of background threads */
-static int background_threads = 0;
+/* Threads that never return; no wait may count on them draining. */
+static int nonjoinable_threads = 0;
 
 /* Server/client ping handling */
 static htsmutex pingMutex = HTSMUTEX_INIT;
@@ -224,7 +224,9 @@ int main(int argc, char *argv[]) {
 #ifdef HTS_USESWF
   smallserver_setkey("USESWF", "1");
 #endif
+#ifdef HTS_USEZLIB
   smallserver_setkey("USEZLIB", "1");
+#endif
 #ifdef _WIN32
   smallserver_setkey("WIN32", "1");
 #endif
@@ -297,15 +299,19 @@ int main(int argc, char *argv[]) {
 
   /* pinger */
   if (parentPid > 0) {
-    hts_newthread(client_ping, (void *) (uintptr_t) parentPid);
-    background_threads++; /* Do not wait for this thread! */
+    if (hts_newthread(client_ping, (void *) (uintptr_t) parentPid) == 0) {
+#ifndef _WIN32
+      nonjoinable_threads++; /* client_ping() only ever leaves through exit() */
+#endif
+    }
     smallserver_setpinghandler(pingHandler, NULL);
   }
 
   /* launch */
   ret = help_server(argv[1], defaultPort, bindAddr);
 
-  htsthread_wait_n(background_threads - 1);
+  /* Drain everything a mirror may still have in flight, the pinger aside. */
+  htsthread_wait_n(nonjoinable_threads);
   hts_uninit();
 
 #ifdef _WIN32
@@ -380,7 +386,6 @@ void webhttrack_main(char *cmd) {
   commandRunning = 1;
   DEBUG(fprintf(stderr, "commandRunning=1\n"));
   hts_newthread(back_launch_cmd, (void *) strdup(cmd));
-  background_threads++; /* Do not wait for this thread! */
 }
 
 void webhttrack_lock(void) {
@@ -421,8 +426,8 @@ static int webhttrack_runmain(httrackp * opt, int argc, char **argv) {
   /* Rock'in! */
   ret = hts_main2(argc, argv, opt);
 
-  /* Wait for pending threads to finish */
-  htsthread_wait_n(background_threads);
+  /* Wait for pending threads to finish; the pinger and this thread stay. */
+  htsthread_wait_n(nonjoinable_threads + 1);
 
   return ret;
 }

src/htszlib.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscodec.h"
 #include "htszlib.h"
 
+#if HTS_USEZLIB
 /* zlib */
 /*
 #include <zlib.h>
@@ -273,3 +274,4 @@ const char *hts_get_zerror(int err) {
     break;
   }
 }
+#endif

tests/65_port-siblings.test

@@ -79,6 +79,29 @@ done
 # 65535 is a valid port the old "< 65535" bound refused
 web_accepted 65535
 
+# --- htsserver returns from main() instead of blocking at exit -------------
+# The exit wait must exclude exactly the threads that never return (#753).
+# $tmp has no lang.def, so the server fails right after announcing and main()
+# reaches the wait on its own; rc, not EXITED, carries the verdict.
+
+web_exits() {
+    local rc=0
+
+    : >"$tmp/exit.log"
+    run_with_timeout 30 htsserver "$tmp" "$@" >"$tmp/exit.log" 2>&1 || rc=$?
+    grep -q "^EXITED" "$tmp/exit.log" ||
+        ! echo "FAIL: #753: htsserver ${*:-(no options)} never finished serving" || exit 1
+    # exactly 1, not merely "not 124": a crash or an assertf abort also escapes
+    # the wait, and would otherwise read as a pass
+    test "$rc" -eq 1 ||
+        ! echo "FAIL: #753: htsserver ${*:-(no options)} exited $rc, want 1" || exit 1
+}
+
+# no pinger: the excluded count went negative
+web_exits
+# a pinger, which never returns and so must stay excluded
+web_exits --ppid $$
+
 # --- proxytrack <proxy-addr:port> <ICP-addr:port> --------------------------
 # A bad argument falls through to the usage screen; it had no range check at
 # all, so 65616 quietly listened on port 80. A valid one binds and blocks.