Compare commits

..

1 Commits

Author SHA1 Message Date
Xavier Roche
41292320cc Give the Windows binaries a version resource
libhttrack.dll and httrack.exe have never carried a VERSIONINFO: no product
name, no version, nothing. Right-click either one on Windows and Properties
has no Details worth reading.

That becomes a hard problem now that we are signing them. SignPath enforce
file metadata restrictions: every binary in a release must name the same
product and the same product version, and a binary that names nothing cannot
satisfy the rule. It fails the signing request, on release day, rather than
the build.

So add src/version.rc, and two wrappers that set the parts which differ per
binary. The version is spelled out there as numbers, because a VERSIONINFO
needs 3,49,12,0 and cannot take a string apart, and because htsglobal.h drags
in C declarations that a resource compiler has no business parsing.

That leaves the version written twice, which is how a version resource quietly
starts lying about which release it is. tests/01_engine-version-macros.test
fails the build if version.rc and htsglobal.h ever disagree, on any of the four
values, and on the product name too. I checked that it fires rather than
assuming: drift the numbers and it fails, restore them and it passes.

The product name is HTTrack Website Copier, on all of them. WinHTTrack is the
name of the Windows GUI, not of the command line program or of the engine.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-13 20:18:43 +02:00
29 changed files with 238 additions and 911 deletions

3
.gitattributes vendored Normal file
View File

@@ -0,0 +1,3 @@
# Resource scripts are Windows tooling input and must stay CRLF: the engine has no
# encoding guard, so an autocrlf checkout could otherwise flatten them silently.
*.rc text eol=crlf

View File

@@ -41,7 +41,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential clang autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure
run: |
@@ -50,9 +50,6 @@ jobs:
# committed generated files already let a plain checkout build.
autoreconf -fi
./configure
# a missing decoder would silently drop the coding from Accept-Encoding
grep -q "define HTS_USEBROTLI 1" config.h
grep -q "define HTS_USEZSTD 1" config.h
- name: Build
run: make -j"$(nproc)"
@@ -85,7 +82,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure
run: |
@@ -126,18 +123,14 @@ jobs:
- name: Install build dependencies
run: |
set -euo pipefail
brew install autoconf automake libtool autoconf-archive brotli zstd
brew install autoconf automake libtool autoconf-archive
- name: Configure
run: |
set -euo pipefail
ssl="$(brew --prefix openssl@3)"
brewp="$(brew --prefix)"
autoreconf -fi
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
LDFLAGS="-L${ssl}/lib -L${brewp}/lib"
grep -q "define HTS_USEBROTLI 1" config.h
grep -q "define HTS_USEZSTD 1" config.h
./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib"
- name: Build
run: make -j"$(sysctl -n hw.ncpu)"
@@ -173,16 +166,14 @@ jobs:
- name: Install build dependencies
run: |
set -euo pipefail
brew install autoconf automake libtool autoconf-archive brotli zstd
brew install autoconf automake libtool autoconf-archive
- name: Build and install into a temp prefix
run: |
set -euo pipefail
ssl="$(brew --prefix openssl@3)"
brewp="$(brew --prefix)"
autoreconf -fi
./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
--prefix="$RUNNER_TEMP/inst"
make -j"$(sysctl -n hw.ncpu)"
make install
@@ -210,8 +201,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential gcc-multilib autoconf automake libtool \
autoconf-archive zlib1g-dev:i386 libssl-dev:i386 \
libbrotli-dev:i386 libzstd-dev:i386
autoconf-archive zlib1g-dev:i386 libssl-dev:i386
- name: Configure
run: |
@@ -252,7 +242,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure (sanitized)
run: |
@@ -354,7 +344,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential clang autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: Configure (fuzzers, static)
run: |
@@ -434,7 +424,7 @@ jobs:
apt-get install -y --no-install-recommends \
ca-certificates git \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev \
zlib1g-dev libssl-dev \
debhelper devscripts lintian fakeroot
- uses: actions/checkout@v6
@@ -478,7 +468,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
build-essential autoconf automake libtool autoconf-archive \
zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
zlib1g-dev libssl-dev
- name: distcheck
run: |

View File

@@ -119,23 +119,6 @@ jobs:
throw "CRT mismatch across the boundary: libhttrack.dll=$a httrack.exe=$b"
}
# The engine self-tests exercise the codecs the DLL was linked with: a
# missing brotli/zstd decoder shows up here, not just at link time.
- name: Run the content-coding self-tests
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$exe = "src\${{ matrix.platform }}\${{ matrix.configuration }}\httrack.exe"
$tmp = New-Item -ItemType Directory -Path (Join-Path $env:RUNNER_TEMP "st")
foreach ($t in @("acceptencoding", "contentcodings")) {
$out = & $exe -O NUL "-#test=$t" $tmp.FullName run
if ($LASTEXITCODE -ne 0) { throw "$t failed" }
if (-not ($out | Select-String -SimpleMatch "$t self-test OK")) {
throw "$t did not report OK: $out"
}
Write-Host ($out -join "`n")
}
- name: Upload MSBuild logs
if: always()
uses: actions/upload-artifact@v4

View File

@@ -1 +0,0 @@
/home/roche/git/httrack-works/CLAUDE.httrack.local.md

View File

@@ -173,10 +173,6 @@ AC_CHECK_HEADERS([execinfo.h])
### zlib
CHECK_ZLIB()
### brotli and zstd content codings (optional)
CHECK_BROTLI()
CHECK_ZSTD()
### OpenSSL is explicitly enabled/disabled ?
AC_MSG_CHECKING(whether to enable https support)
AC_ARG_ENABLE([https],

2
debian/control vendored
View File

@@ -2,7 +2,7 @@ Source: httrack
Section: web
Maintainer: Xavier Roche <roche@httrack.com>
Standards-Version: 4.7.4
Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev, libbrotli-dev, libzstd-dev
Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
Rules-Requires-Root: no
Homepage: http://www.httrack.com
Vcs-Git: https://github.com/xroche/httrack.git

View File

@@ -1,66 +0,0 @@
dnl @synopsis CHECK_BROTLI()
dnl @synopsis CHECK_ZSTD()
dnl
dnl Look for libbrotlidec and libzstd, the decoders for the "br" and "zstd"
dnl HTTP content codings. Both are optional: a missing library only drops the
dnl coding from Accept-Encoding. --with-brotli=DIR / --with-zstd=DIR point at a
dnl prefix; --without-brotli / --without-zstd disable the coding; the default
dnl is to use the library when it is found.
dnl
dnl Define HTS_USEBROTLI / HTS_USEZSTD to 1 or 0, and substitute BROTLI_LIBS /
dnl ZSTD_LIBS.
dnl CHECK_CODEC(name, NAME, header, library, symbol, companion-libs)
AC_DEFUN([CHECK_CODEC], [
AC_ARG_WITH([$1],
[AS_HELP_STRING([--with-$1@<:@=DIR@:>@],[Enable the $1 content coding @<:@default=auto@:>@])],
[codec_want=$withval], [codec_want=auto])
$2_LIBS=""
codec_have=no
if test "$codec_want" != "no"; then
codec_old_cppflags="$CPPFLAGS"
codec_old_ldflags="$LDFLAGS"
codec_dir=no
if test "$codec_want" != "yes" -a "$codec_want" != "auto"; then
codec_dir=yes
# An explicit prefix is authoritative: if the header is not under
# it, error rather than silently pick a system copy.
if test ! -f "$codec_want/include/$3"; then
AC_MSG_ERROR([$1 requested at $codec_want but $codec_want/include/$3 is missing])
fi
CPPFLAGS="$CPPFLAGS -I$codec_want/include"
LDFLAGS="$LDFLAGS -L$codec_want/lib"
fi
AC_CHECK_HEADER([$3], [codec_h=yes], [codec_h=no])
AC_CHECK_LIB([$4], [$5], [codec_lib=yes], [codec_lib=no], [$6])
CPPFLAGS="$codec_old_cppflags"
LDFLAGS="$codec_old_ldflags"
if test "$codec_h" = "yes" -a "$codec_lib" = "yes"; then
codec_have=yes
$2_LIBS="-l$4 $6"
if test "$codec_dir" = "yes"; then
$2_LIBS="-L$codec_want/lib -l$4 $6"
# The rest of configure still needs the header path.
CPPFLAGS="$CPPFLAGS -I$codec_want/include"
fi
elif test "$codec_want" != "auto"; then
AC_MSG_ERROR([$1 requested but lib$4/$3 not found])
fi
fi
AC_MSG_CHECKING([whether to enable the $1 content coding])
AC_MSG_RESULT([$codec_have])
if test "$codec_have" = "yes"; then
AC_DEFINE([HTS_USE$2], [1], [Define to 1 to decode the $1 content coding])
else
AC_DEFINE([HTS_USE$2], [0], [Define to 1 to decode the $1 content coding])
fi
AC_SUBST([$2_LIBS])
AC_SUBST([$2_ENABLED], [$codec_have])
])
dnl brotlidec needs brotlicommon (the static dictionary) for a fully-static link.
AC_DEFUN([CHECK_BROTLI],
[CHECK_CODEC([brotli], [BROTLI], [brotli/decode.h], [brotlidec], [BrotliDecoderDecompressStream], [-lbrotlicommon])])
AC_DEFUN([CHECK_ZSTD],
[CHECK_CODEC([zstd], [ZSTD], [zstd.h], [zstd], [ZSTD_decompressStream], [])])

View File

@@ -61,7 +61,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
htshelp.c htslib.c htscoremain.c \
htsname.c htsrobots.c htstools.c htswizard.c \
htsalias.c htsthread.c htsindex.c htsbauth.c \
htsmd5.c htscodec.c htszlib.c htswrap.c htsconcat.c \
htsmd5.c htszlib.c htswrap.c htsconcat.c \
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
md5.c \
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
@@ -72,17 +72,20 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
htshelp.h htsindex.h htslib.h htsmd5.h \
htsmodules.h htsname.h htsnet.h htssniff.h \
htsopt.h htsrobots.h htsthread.h \
htstools.h htswizard.h htswrap.h htscodec.h htszlib.h \
htstools.h htswizard.h htswrap.h htszlib.h \
htsstrings.h htsarrays.h httrack-library.h \
htscharset.h punycode.h htsencoding.h \
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
md5.h coucal/murmurhash3.h \
minizip/crypt.h minizip/ioapi.h minizip/mztools.h minizip/unzip.h minizip/zip.h
libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(BROTLI_LIBS) $(ZSTD_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $(SOCKET_LIBS) $(ICONV_LIBS)
libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $(SOCKET_LIBS) $(ICONV_LIBS)
libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
EXTRA_DIST = httrack.h webhttrack \
version.rc \
libhttrack.rc \
httrack.rc \
coucal/murmurhash3.h.diff \
coucal/murmurhash3.h.orig \
minizip/iowin32.c \

View File

@@ -44,7 +44,6 @@ Please visit our Website: http://www.httrack.com
#include "htsback.h"
#include "htsftp.h"
#include "htscodec.h"
#if HTS_USEZLIB
#include "htszlib.h"
#else
@@ -628,7 +627,8 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
if (!back[p].testmode) { // not test mode
const char *state = "unknown";
/* Undo the content coding */
/* décompression */
#if HTS_USEZLIB
if (back[p].r.compressed) {
if (back[p].r.size > 0) {
// stats
@@ -674,15 +674,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
// décompression
if (back[p].tmpfile != NULL) {
if (back[p].url_sav[0]) {
const hts_codec codec =
hts_codec_parse(back[p].r.contentencoding);
LLint size;
file_notify(opt, back[p].url_adr, back[p].url_fil,
back[p].url_sav, 1, 1, back[p].r.notmodified);
filecreateempty(&opt->state.strc, back[p].url_sav); // filenote & co
if ((size = hts_codec_unpack(codec, back[p].tmpfile,
back[p].url_sav)) >= 0) {
if ((size = hts_zunpack(back[p].tmpfile, back[p].url_sav)) >= 0) {
back[p].r.size = back[p].r.totalsize = size;
// fichier -> mémoire
if (!back[p].r.is_write) {
@@ -697,16 +694,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
}
} else {
back[p].r.statuscode = STATUSCODE_INVALID;
snprintf(back[p].r.msg, sizeof(back[p].r.msg),
codec == HTS_CODEC_UNSUPPORTED
? "Unsupported Content-Encoding (%s)"
: "Error when decompressing (%s)",
back[p].r.contentencoding);
/* Drop the undecoded body and its placeholder so the writer
can't commit the coded bytes as the page. */
if (!back[p].r.is_write)
deleteaddr(&back[p].r);
UNLINK(back[p].url_sav);
strcpybuff(back[p].r.msg, "Error when decompressing");
}
}
/* ensure that no remaining temporary file exists */
@@ -720,6 +708,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
// unflag
}
}
#endif
/* Body fully received: keep the freshly written url_sav, drop the
backup of the previous copy. */
back_finalize_backup(opt, &back[p], HTS_TRUE);
@@ -818,7 +807,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
else
strcatbuff(flags, "-");
if (back[p].r.compressed)
strcatbuff(flags, "Z"); // content coding
strcatbuff(flags, "Z"); // gzip
else
strcatbuff(flags, "-");
/* Err I had to split these.. */
@@ -827,7 +816,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
fprintf(cache->txt, LLintP, (LLint) back[p].r.totalsize);
fprintf(cache->txt, "\t%s\t", flags);
}
#if HTS_USEZLIB
back[p].r.compressed = 0;
#endif
if (back[p].r.statuscode == HTTP_OK) {
if (back[p].r.size >= 0) {
@@ -2950,13 +2941,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
int last_errno = 0;
back[i].r.is_write = 1; // écrire
/* a .gz/.br/.zst saved under its own coding stays
packed on disk */
if (back[i].r.compressed &&
!hts_codec_is_archive_ext(
hts_codec_parse(back[i].r.contentencoding),
get_ext(catbuff, sizeof(catbuff),
back[i].url_sav))) {
/* .gz are *NOT* depacked!! */
strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
&& strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
) {
if (create_back_tmpfile(opt, &back[i], "z") ==
0) {
assertf(back[i].tmpfile != NULL);

View File

@@ -1,366 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: HTTP content codings (gzip/deflate, brotli, zstd) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
/* Internal engine bytecode */
#define HTS_INTERNAL_BYTECODE
#include "htsbase.h"
#include "htscore.h"
#include "htscodec.h"
#include "htszlib.h"
#if HTS_USEBROTLI
#include <brotli/decode.h>
#endif
#if HTS_USEZSTD
#include <zstd.h>
/* 8 MB, the window an HTTP zstd decoder must honor (RFC 9659); libzstd
defaults to 128 MB. */
#define HTS_ZSTD_WINDOWLOG_MAX 23
#endif
/* Decoded-size budget, bounding a bomb: brotli and zstd reach a million to one.
Deflate cannot pass 1032x, so it only meets this at the 2 GiB ceiling. */
#define HTS_CODEC_MAX_RATIO 4096
#define HTS_CODEC_MIN_MAXOUT (1024 * 1024)
LLint hts_codec_maxout(LLint in_size) {
LLint maxout;
if (in_size <= 0 || in_size > INT_MAX / HTS_CODEC_MAX_RATIO)
maxout = INT_MAX;
else
maxout = in_size * HTS_CODEC_MAX_RATIO;
if (maxout < HTS_CODEC_MIN_MAXOUT)
maxout = HTS_CODEC_MIN_MAXOUT;
return maxout;
}
hts_codec hts_codec_parse(const char *encoding) {
if (encoding == NULL || encoding[0] == '\0' ||
strfield2(encoding, "identity"))
return HTS_CODEC_IDENTITY;
if (strfield2(encoding, "gzip") || strfield2(encoding, "x-gzip") ||
strfield2(encoding, "deflate") || strfield2(encoding, "x-deflate"))
return HTS_USEZLIB ? HTS_CODEC_DEFLATE : HTS_CODEC_UNSUPPORTED;
if (strfield2(encoding, "br"))
return HTS_USEBROTLI ? HTS_CODEC_BROTLI : HTS_CODEC_UNSUPPORTED;
if (strfield2(encoding, "zstd"))
return HTS_USEZSTD ? HTS_CODEC_ZSTD : HTS_CODEC_UNSUPPORTED;
if (strfield2(encoding, "compress") || strfield2(encoding, "x-compress"))
return HTS_CODEC_UNSUPPORTED; /* LZW: never advertised, no decoder here */
/* Not a coding at all: broken servers put charsets and the like here, and
the plain body they sent must survive it. */
return HTS_CODEC_IDENTITY;
}
#if HTS_USEBROTLI
#define HTS_AE_BROTLI ", br"
#else
#define HTS_AE_BROTLI ""
#endif
#if HTS_USEZSTD
#define HTS_AE_ZSTD ", zstd"
#else
#define HTS_AE_ZSTD ""
#endif
const char *hts_acceptencoding(hts_boolean compressible, hts_boolean secure) {
if (!compressible)
return "identity";
#if HTS_USEZLIB
/* br and zstd over TLS only, as browsers do: a cleartext intermediary that
rewrites a coding it can not read would corrupt the mirror. */
if (secure)
return "gzip, deflate" HTS_AE_BROTLI HTS_AE_ZSTD ", identity;q=0.9";
return "gzip, deflate, identity;q=0.9";
#else
(void) secure;
return "identity";
#endif
}
hts_boolean hts_codec_is_archive_ext(hts_codec codec, const char *ext) {
if (ext == NULL || ext[0] == '\0')
return HTS_FALSE;
switch (codec) {
case HTS_CODEC_DEFLATE:
return strfield2(ext, "gz") || strfield2(ext, "tgz") ? HTS_TRUE : HTS_FALSE;
case HTS_CODEC_BROTLI:
return strfield2(ext, "br") ? HTS_TRUE : HTS_FALSE;
case HTS_CODEC_ZSTD:
return strfield2(ext, "zst") || strfield2(ext, "tzst") ? HTS_TRUE
: HTS_FALSE;
default:
return HTS_FALSE;
}
}
#if HTS_USEBROTLI || HTS_USEZSTD
/* Append produced bytes to out under the decoded-size budget, advancing *total.
HTS_FALSE on a short write or once the budget is exceeded (a bomb); the
over-budget bytes are never written. */
static hts_boolean codec_sink(FILE *out, const void *buf, size_t produced,
LLint *total, LLint maxout) {
if (produced == 0)
return HTS_TRUE;
*total += (LLint) produced;
if (*total > maxout)
return HTS_FALSE;
return fwrite(buf, 1, produced, out) == produced ? HTS_TRUE : HTS_FALSE;
}
#endif
#if HTS_USEBROTLI
static int codec_unpack_brotli(FILE *in, FILE *out, LLint maxout) {
BrotliDecoderState *const state =
BrotliDecoderCreateInstance(NULL, NULL, NULL);
hts_boolean ok = HTS_TRUE, done = HTS_FALSE;
LLint total = 0;
if (state == NULL)
return -1;
while (ok && !done) {
unsigned char BIGSTK inbuf[8192];
size_t avail_in = fread(inbuf, 1, sizeof(inbuf), in);
const uint8_t *next_in = inbuf;
if (avail_in == 0)
break; /* EOF; a stream still hungry for input is truncated */
for (;;) {
unsigned char BIGSTK outbuf[8192];
uint8_t *next_out = outbuf;
size_t avail_out = sizeof(outbuf);
size_t produced;
BrotliDecoderResult res;
res = BrotliDecoderDecompressStream(state, &avail_in, &next_in,
&avail_out, &next_out, NULL);
produced = sizeof(outbuf) - avail_out;
if (!codec_sink(out, outbuf, produced, &total, maxout)) {
ok = HTS_FALSE;
break;
}
if (res == BROTLI_DECODER_RESULT_ERROR) {
ok = HTS_FALSE;
break;
}
if (res == BROTLI_DECODER_RESULT_SUCCESS) {
done = HTS_TRUE;
break;
}
if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT)
break;
}
if (ferror(in))
ok = HTS_FALSE;
}
BrotliDecoderDestroyInstance(state);
return (ok && done && !ferror(in)) ? (int) total : -1;
}
static size_t codec_head_brotli(const void *in, size_t in_len, void *out,
size_t out_len) {
BrotliDecoderState *const state =
BrotliDecoderCreateInstance(NULL, NULL, NULL);
const uint8_t *next_in = (const uint8_t *) in;
uint8_t *next_out = (uint8_t *) out;
size_t avail_in = in_len, avail_out = out_len;
BrotliDecoderResult res;
if (state == NULL)
return 0;
res = BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out,
&next_out, NULL);
BrotliDecoderDestroyInstance(state);
return (res != BROTLI_DECODER_RESULT_ERROR) ? out_len - avail_out : 0;
}
#endif
#if HTS_USEZSTD
static int codec_unpack_zstd(FILE *in, FILE *out, LLint maxout) {
ZSTD_DStream *const zds = ZSTD_createDStream();
hts_boolean ok = HTS_TRUE, eof = HTS_FALSE;
size_t zret = 1; /* 0 once a frame is fully flushed */
LLint total = 0;
if (zds == NULL)
return -1;
if (ZSTD_isError(ZSTD_DCtx_setParameter(zds, ZSTD_d_windowLogMax,
HTS_ZSTD_WINDOWLOG_MAX))) {
ZSTD_freeDStream(zds);
return -1;
}
while (ok && !eof) {
unsigned char BIGSTK inbuf[8192];
ZSTD_inBuffer input;
input.src = inbuf;
input.size = fread(inbuf, 1, sizeof(inbuf), in);
input.pos = 0;
if (input.size == 0) {
eof = HTS_TRUE;
break;
}
while (input.pos < input.size) {
unsigned char BIGSTK outbuf[8192];
ZSTD_outBuffer output;
output.dst = outbuf;
output.size = sizeof(outbuf);
output.pos = 0;
zret = ZSTD_decompressStream(zds, &output, &input);
if (ZSTD_isError(zret)) {
ok = HTS_FALSE;
break;
}
if (!codec_sink(out, outbuf, output.pos, &total, maxout)) {
ok = HTS_FALSE;
break;
}
}
if (ferror(in))
ok = HTS_FALSE;
}
ZSTD_freeDStream(zds);
/* zret != 0 at EOF: the last frame never completed */
return (ok && zret == 0 && !ferror(in)) ? (int) total : -1;
}
static size_t codec_head_zstd(const void *in, size_t in_len, void *out,
size_t out_len) {
ZSTD_DStream *const zds = ZSTD_createDStream();
ZSTD_inBuffer input;
ZSTD_outBuffer output;
size_t zret;
if (zds == NULL)
return 0;
if (ZSTD_isError(ZSTD_DCtx_setParameter(zds, ZSTD_d_windowLogMax,
HTS_ZSTD_WINDOWLOG_MAX))) {
ZSTD_freeDStream(zds);
return 0;
}
input.src = in;
input.size = in_len;
input.pos = 0;
output.dst = out;
output.size = out_len;
output.pos = 0;
zret = ZSTD_decompressStream(zds, &output, &input);
ZSTD_freeDStream(zds);
return ZSTD_isError(zret) ? 0 : output.pos;
}
#endif
LLint hts_codec_coded_size(FILE *in) {
long size;
if (fseek(in, 0, SEEK_END) != 0)
return -1;
size = ftell(in);
if (size < 0 || fseek(in, 0, SEEK_SET) != 0)
return -1;
return (LLint) size;
}
int hts_codec_unpack(hts_codec codec, const char *filename,
const char *newfile) {
if (filename == NULL || newfile == NULL || !filename[0] || !newfile[0])
return -1;
switch (codec) {
case HTS_CODEC_DEFLATE:
#if HTS_USEZLIB
return hts_zunpack(filename, newfile);
#else
return -1;
#endif
case HTS_CODEC_BROTLI:
case HTS_CODEC_ZSTD:
break;
default: /* IDENTITY never reaches the unpacker; UNSUPPORTED must fail */
return -1;
}
#if HTS_USEBROTLI || HTS_USEZSTD
{
char catbuff[CATBUFF_SIZE];
FILE *out, *in = FOPEN(fconv(catbuff, sizeof(catbuff), filename), "rb");
LLint maxout;
int ret = -1;
if (in == NULL)
return -1;
maxout = hts_codec_maxout(hts_codec_coded_size(in));
out = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
if (out == NULL) {
fclose(in);
return -1;
}
#if HTS_USEBROTLI
if (codec == HTS_CODEC_BROTLI)
ret = codec_unpack_brotli(in, out, maxout);
#endif
#if HTS_USEZSTD
if (codec == HTS_CODEC_ZSTD)
ret = codec_unpack_zstd(in, out, maxout);
#endif
fclose(out);
fclose(in);
return ret;
}
#else
return -1;
#endif
}
size_t hts_codec_head(hts_codec codec, const void *in, size_t in_len, void *out,
size_t out_len) {
if (in == NULL || in_len == 0 || out == NULL || out_len == 0)
return 0;
switch (codec) {
#if HTS_USEZLIB
case HTS_CODEC_DEFLATE:
return hts_zhead(in, in_len, out, out_len);
#endif
#if HTS_USEBROTLI
case HTS_CODEC_BROTLI:
return codec_head_brotli(in, in_len, out, out_len);
#endif
#if HTS_USEZSTD
case HTS_CODEC_ZSTD:
return codec_head_zstd(in, in_len, out, out_len);
#endif
default:
return 0;
}
}

View File

@@ -1,79 +0,0 @@
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2026 Xavier Roche and other contributors
SPDX-License-Identifier: GPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.
Please visit our Website: http://www.httrack.com
*/
/* ------------------------------------------------------------ */
/* File: HTTP content codings (gzip/deflate, brotli, zstd) */
/* Author: Xavier Roche */
/* ------------------------------------------------------------ */
#ifndef HTS_DEFCODEC
#define HTS_DEFCODEC
#include "htsglobal.h"
/* Content codings we may meet in a Content-Encoding header. */
typedef enum {
HTS_CODEC_IDENTITY = 0, /**< no coding, or a token we treat as none */
HTS_CODEC_DEFLATE, /**< gzip, x-gzip, deflate, x-deflate */
HTS_CODEC_BROTLI, /**< br */
HTS_CODEC_ZSTD, /**< zstd */
HTS_CODEC_UNSUPPORTED /**< a real coding we can not decode */
} hts_codec;
#ifdef HTS_INTERNAL_BYTECODE
/* Codec for a Content-Encoding token. A known coding we can not decode is
UNSUPPORTED; an unrecognized token is IDENTITY (servers do emit junk). */
extern hts_codec hts_codec_parse(const char *encoding);
/* Accept-Encoding value to advertise; secure (TLS) also offers br and zstd. */
extern const char *hts_acceptencoding(hts_boolean compressible,
hts_boolean secure);
/* HTS_TRUE if ext is the codec's own archive extension: a foo.gz served as
Content-Encoding: gzip stays packed on disk. */
extern hts_boolean hts_codec_is_archive_ext(hts_codec codec, const char *ext);
/* Decode filename into newfile. Returns the decoded size, or -1 on a truncated
or corrupt stream, an unsupported coding, a bomb over the budget, or I/O. */
extern int hts_codec_unpack(hts_codec codec, const char *filename,
const char *newfile);
/* Decode up to out_len leading bytes of a coded body for the MIME sniffer;
0 when nothing could be decoded. */
extern size_t hts_codec_head(hts_codec codec, const void *in, size_t in_len,
void *out, size_t out_len);
/* Ceiling on the decoded size of a body of in_size coded bytes. */
extern LLint hts_codec_maxout(LLint in_size);
/* Coded size of an open stream, left rewound; -1 if unknown. */
extern LLint hts_codec_coded_size(FILE *in);
#endif
#endif

View File

@@ -140,15 +140,6 @@ Please visit our Website: http://www.httrack.com
#define HTS_USEZLIB 1
#endif
// brotli and zstd content codings; off unless the build opted in (configure,
// or the Visual Studio projects, which link the vcpkg libraries)
#ifndef HTS_USEBROTLI
#define HTS_USEBROTLI 0
#endif
#ifndef HTS_USEZSTD
#define HTS_USEZSTD 0
#endif
#ifndef HTS_INET6
#define HTS_INET6 0
#endif

View File

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
#include "htsmodules.h"
#include "htscharset.h"
#include "htsencoding.h"
#include "htscodec.h"
#ifdef _WIN32
#include <direct.h>
@@ -1280,16 +1279,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
// Compression accepted ?
if (retour->req.http11) {
hts_boolean compressible = HTS_FALSE;
hts_boolean secure = HTS_FALSE;
#if HTS_USEZLIB
compressible = (!retour->req.range_used && !retour->req.nocompression);
#endif
#if HTS_USEOPENSSL
secure = retour->ssl ? HTS_TRUE : HTS_FALSE;
#endif
print_buffer(&bstr, "Accept-Encoding: %s" H_CRLF,
hts_acceptencoding(compressible, secure));
hts_acceptencoding(compressible));
}
/* Authentification */
@@ -1674,10 +1668,22 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
strcpybuff(retour->contentencoding, tempo);
else
retour->contentencoding[0] = '\0';
/* A coding to undo (or one we can not undo, which must fail the fetch
rather than save the coded bytes as the page) */
if (hts_codec_parse(retour->contentencoding) != HTS_CODEC_IDENTITY)
retour->compressed = 1;
#if HTS_USEZLIB
/* Check known encodings */
if (retour->contentencoding[0]) {
if ((strfield2(retour->contentencoding, "gzip"))
|| (strfield2(retour->contentencoding, "x-gzip"))
/*
|| (strfield2(retour->contentencoding, "compress"))
|| (strfield2(retour->contentencoding, "x-compress"))
*/
|| (strfield2(retour->contentencoding, "deflate"))
|| (strfield2(retour->contentencoding, "x-deflate"))
) {
retour->compressed = 1;
}
}
#endif
}
} else if ((p = strfield(rcvd, "Location:")) != 0) {
if (retour) {
@@ -4359,6 +4365,11 @@ HTSEXT_API void get_httptype(httrackp *opt, char *s, const char *fil,
(void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, flag);
}
/* Advertised Accept-Encoding; gzip and deflate both decode via hts_zunpack */
const char *hts_acceptencoding(hts_boolean compressible) {
return compressible ? "gzip, deflate, identity;q=0.9" : "identity";
}
// get type of fil (php)
// s: buffer (text/html) or NULL
// return: 1 if known by user

View File

@@ -283,6 +283,9 @@ int ishttperror(int err);
int get_userhttptype(httrackp * opt, char *s, const char *fil);
int give_mimext(char *s, size_t ssize, const char *st);
/* Advertised Accept-Encoding value (no header name/CRLF); see htslib.c. */
const char *hts_acceptencoding(hts_boolean compressible);
int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename);
int may_unknown2(httrackp * opt, const char *mime, const char *filename);

View File

@@ -42,7 +42,6 @@ Please visit our Website: http://www.httrack.com
#include "htscharset.h"
#include "htsencoding.h"
#include "htssniff.h"
#include "htscodec.h"
#if HTS_USEZLIB
#include "htszlib.h"
#endif
@@ -152,6 +151,29 @@ typedef struct sniff_src {
const char *prev_save; /* previous run's save name (cache X-Save) */
} sniff_src;
#if HTS_USEZLIB
/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
size_t out_len) {
z_stream zs;
size_t n = 0;
int err;
memset(&zs, 0, sizeof(zs));
if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
return 0;
zs.next_in = (const Bytef *) in;
zs.avail_in = (uInt) in_len;
zs.next_out = (Bytef *) out;
zs.avail_out = (uInt) out_len;
err = inflate(&zs, Z_SYNC_FLUSH);
if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
n = out_len - zs.avail_out;
inflateEnd(&zs);
return n;
}
#endif
static size_t sniff_read_head(const char *path, void *buf, size_t len) {
char catbuff[CATBUFF_SIZE];
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
@@ -183,12 +205,16 @@ static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
n = sniff_read_head(slot->tmpfile, buf, len);
}
if (n > 0 && r->compressed) {
#if HTS_USEZLIB
unsigned char raw[HTS_SNIFF_LEN];
if (n > sizeof(raw))
n = sizeof(raw);
memcpy(raw, buf, n);
n = hts_codec_head(hts_codec_parse(r->contentencoding), raw, n, buf, len);
n = sniff_inflate_head(raw, n, buf, len);
#else
n = 0;
#endif
}
return n;
}

View File

@@ -54,13 +54,9 @@ Please visit our Website: http://www.httrack.com
#include "htsftp.h"
#include "htsmd5.h"
#include "htssniff.h"
#include "htscodec.h"
#if HTS_USEZLIB
#include "htszlib.h"
#endif
#if HTS_USEZSTD
#include <zstd.h>
#endif
#include "coucal/coucal.h"
#include <ctype.h>
@@ -2309,7 +2305,6 @@ static int ae_write_collision(const char *path, const unsigned char *src,
freet(buf);
return ok ? 0 : 1;
}
#endif
/* Write src[0..len) to path as-is; 0 on success. */
static int ae_write_raw(const char *path, const unsigned char *src,
@@ -2343,22 +2338,17 @@ static int ae_check_decoded(const char *path, const unsigned char *expect,
fclose(f);
return (off == len) ? 0 : 1;
}
#endif
/* Accept-Encoding (#450): advertise gzip+deflate; both decode (hts_zunpack) */
static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
const char *off = hts_acceptencoding(HTS_FALSE, HTS_TRUE);
const char *on = hts_acceptencoding(HTS_TRUE, HTS_FALSE);
const char *tls = hts_acceptencoding(HTS_TRUE, HTS_TRUE);
const char *off = hts_acceptencoding(HTS_FALSE);
const char *on = hts_acceptencoding(HTS_TRUE);
(void) opt;
assertf(strcmp(off, "identity") == 0);
assertf(strstr(on, "gzip") != NULL);
assertf(strstr(on, "deflate") != NULL); /* fails on the old gzip-only list */
/* br and zstd ride on TLS only, so a cleartext proxy can not be handed a
coding it may try to rewrite */
assertf(strstr(on, "br") == NULL && strstr(on, "zstd") == NULL);
assertf((strstr(tls, ", br") != NULL) == (HTS_USEBROTLI != 0));
assertf((strstr(tls, "zstd") != NULL) == (HTS_USEZSTD != 0));
#if HTS_USEZLIB
if (argc >= 1) {
static const int windowBits[] = {16 + MAX_WBITS, MAX_WBITS, -MAX_WBITS};
@@ -2446,115 +2436,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
return 0;
}
#if HTS_USEBROTLI
/* No brotli encoder is linked, so the coded bytes are canned: brotli quality 9
over cc_text, and over 4 MiB of 'A' (14 bytes, ~300000x). */
static const unsigned char cc_br_text[] = {
0x1b, 0x43, 0x00, 0x00, 0x44, 0xdd, 0x96, 0xea, 0xe8, 0x22, 0xdd, 0x90,
0xa4, 0x1b, 0x8a, 0xf7, 0x47, 0x0e, 0xc2, 0xc5, 0x3d, 0x09, 0x1b, 0x70,
0xe0, 0x1e, 0x60, 0xa0, 0x8b, 0xcc, 0xbe, 0xcb, 0xb0, 0x31, 0x76, 0x9e,
0xcf, 0x6e, 0x41, 0xb5, 0xe8, 0x2e, 0x56, 0x78, 0x08, 0x1b, 0xfa, 0x08,
0x8a, 0x50, 0x83, 0x4e, 0x62, 0x7f, 0xbf, 0x05, 0xf2, 0x22, 0x8f, 0xdf,
0x28, 0xdc, 0x9f, 0xa9, 0x90, 0x50, 0x37, 0x62, 0x56, 0x4f, 0xa8};
static const unsigned char cc_br_bomb[] = {0x9b, 0xff, 0xff, 0x3f, 0x00,
0x24, 0x82, 0xe2, 0xb1, 0x40,
0x72, 0xef, 0x7f, 0x00};
#endif
static const unsigned char cc_text[] =
"content codings: HTTrack decodes gzip, brotli and zstd bodies alike.";
/* Content codings: br and zstd decode, junk tokens stay identity, a coding we
can not undo fails the fetch, and a bomb never lands on disk. */
static int st_contentcodings(httrackp *opt, int argc, char **argv) {
const size_t tlen = sizeof(cc_text) - 1;
char inpath[HTS_URLMAXSIZE], outpath[HTS_URLMAXSIZE];
(void) opt;
assertf(hts_codec_parse("gzip") == HTS_CODEC_DEFLATE);
assertf(hts_codec_parse("x-deflate") == HTS_CODEC_DEFLATE);
assertf(hts_codec_parse("") == HTS_CODEC_IDENTITY);
assertf(hts_codec_parse("identity") == HTS_CODEC_IDENTITY);
/* servers do label plain bodies with junk; the page must survive that */
assertf(hts_codec_parse("utf-8") == HTS_CODEC_IDENTITY);
/* a real coding with no decoder here: fail, never save the coded bytes */
assertf(hts_codec_parse("compress") == HTS_CODEC_UNSUPPORTED);
assertf(hts_codec_parse("br") ==
(HTS_USEBROTLI ? HTS_CODEC_BROTLI : HTS_CODEC_UNSUPPORTED));
assertf(hts_codec_parse("zstd") ==
(HTS_USEZSTD ? HTS_CODEC_ZSTD : HTS_CODEC_UNSUPPORTED));
assertf(hts_codec_is_archive_ext(HTS_CODEC_DEFLATE, "tgz"));
assertf(!hts_codec_is_archive_ext(HTS_CODEC_DEFLATE, "html"));
assertf(hts_codec_is_archive_ext(HTS_CODEC_BROTLI, "br"));
assertf(hts_codec_is_archive_ext(HTS_CODEC_ZSTD, "zst"));
/* decoded-size budget: 4096x, floor 1 MiB, ceiling INT_MAX */
assertf(hts_codec_maxout(1) == 1024 * 1024);
assertf(hts_codec_maxout(1024) == 4096 * 1024);
assertf(hts_codec_maxout(1024 * 1024) == INT_MAX);
if (argc >= 1) {
snprintf(inpath, sizeof(inpath), "%s/cc-in", argv[0]);
snprintf(outpath, sizeof(outpath), "%s/cc-out", argv[0]);
#if HTS_USEBROTLI
{
unsigned char head[16];
assertf(ae_write_raw(inpath, cc_br_text, sizeof(cc_br_text)) == 0);
assertf(hts_codec_unpack(HTS_CODEC_BROTLI, inpath, outpath) ==
(int) tlen);
assertf(ae_check_decoded(outpath, cc_text, tlen) == 0);
assertf(hts_codec_head(HTS_CODEC_BROTLI, cc_br_text, sizeof(cc_br_text),
head, sizeof(head)) == sizeof(head));
assertf(memcmp(head, cc_text, sizeof(head)) == 0);
/* truncated: must fail, not fall back to a verbatim copy */
assertf(ae_write_raw(inpath, cc_br_text, sizeof(cc_br_text) - 4) == 0);
assertf(hts_codec_unpack(HTS_CODEC_BROTLI, inpath, outpath) < 0);
/* cc_br_bomb is a valid stream that expands to 4 MiB; the budget, not a
corrupt frame, is what must reject it. */
assertf((LLint) (4 * 1024 * 1024) >
hts_codec_maxout((LLint) sizeof(cc_br_bomb)));
assertf(ae_write_raw(inpath, cc_br_bomb, sizeof(cc_br_bomb)) == 0);
assertf(hts_codec_unpack(HTS_CODEC_BROTLI, inpath, outpath) < 0);
}
#endif
#if HTS_USEZSTD
{
const size_t bomblen = 4 * 1024 * 1024;
const size_t bound = ZSTD_compressBound(bomblen);
unsigned char *bomb = malloct(bomblen);
unsigned char *zbuf = malloct(bound);
unsigned char head[16];
size_t zlen;
assertf(bomb != NULL && zbuf != NULL);
zlen = ZSTD_compress(zbuf, bound, cc_text, tlen, 6);
assertf(!ZSTD_isError(zlen));
assertf(ae_write_raw(inpath, zbuf, zlen) == 0);
assertf(hts_codec_unpack(HTS_CODEC_ZSTD, inpath, outpath) == (int) tlen);
assertf(ae_check_decoded(outpath, cc_text, tlen) == 0);
assertf(hts_codec_head(HTS_CODEC_ZSTD, zbuf, zlen, head, sizeof(head)) ==
sizeof(head));
assertf(memcmp(head, cc_text, sizeof(head)) == 0);
assertf(ae_write_raw(inpath, zbuf, zlen - 4) == 0);
assertf(hts_codec_unpack(HTS_CODEC_ZSTD, inpath, outpath) < 0);
memset(bomb, 'A', bomblen);
zlen = ZSTD_compress(zbuf, bound, bomb, bomblen, 6);
assertf(!ZSTD_isError(zlen));
/* the fixture must really be past the budget, whatever the ratio is */
assertf((LLint) bomblen > hts_codec_maxout((LLint) zlen));
assertf(ae_write_raw(inpath, zbuf, zlen) == 0);
assertf(hts_codec_unpack(HTS_CODEC_ZSTD, inpath, outpath) < 0);
freet(bomb);
freet(zbuf);
}
#endif
/* a coding we can not undo yields no file at all */
assertf(hts_codec_unpack(HTS_CODEC_UNSUPPORTED, inpath, outpath) < 0);
}
printf("contentcodings self-test OK\n");
return 0;
}
/* Each call parses `txt` under a fresh host, then checkrobots() for `path`. */
static int rb_decide(robots_wizard *r, const char *txt, const char *path) {
static int n = 0;
@@ -2796,9 +2677,6 @@ static const struct selftest_entry {
{"status", "", "HTTP status code -> reason phrase self-test", st_status},
{"acceptencoding", "[dir]",
"Accept-Encoding advertises gzip+deflate, both decode", st_acceptencoding},
{"contentcodings", "[dir]",
"brotli and zstd bodies decode; bombs and unknown codings are refused",
st_contentcodings},
{"robots", "", "robots.txt RFC 9309 Allow/Disallow precedence self-test",
st_robots},
#ifndef _WIN32

View File

@@ -37,7 +37,6 @@ Please visit our Website: http://www.httrack.com
/* specific definitions */
#include "htsbase.h"
#include "htscore.h"
#include "htscodec.h"
#include "htszlib.h"
#if HTS_USEZLIB
@@ -47,8 +46,13 @@ Please visit our Website: http://www.httrack.com
#include "htszlib.h"
*/
/*
Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
A body provably in no deflate framing is copied verbatim (identity).
Return value: size of the new file, or -1 if an error occurred
*/
/* Note: utf-8 */
int hts_zunpack(const char *filename, const char *newfile) {
int hts_zunpack(char *filename, char *newfile) {
int ret = -1;
if (filename != NULL && newfile != NULL && filename[0] && newfile[0]) {
@@ -57,7 +61,6 @@ int hts_zunpack(const char *filename, const char *newfile) {
if (in != NULL) {
unsigned char BIGSTK inbuf[8192];
const LLint maxout = hts_codec_maxout(hts_codec_coded_size(in));
size_t navail = fread(inbuf, 1, sizeof(inbuf), in);
/* gzip/zlib headers -> +32 windowBits; else raw deflate (RFC1951) */
const hts_boolean wrapped =
@@ -70,10 +73,9 @@ int hts_zunpack(const char *filename, const char *newfile) {
deflate framing at all. env_error: local I/O or memory failure. */
hts_boolean not_deflate = HTS_FALSE;
hts_boolean env_error = HTS_FALSE;
hts_boolean bomb = HTS_FALSE;
/* deflate is ambiguous; on failure retry with the other windowBits */
for (attempt = 0; attempt < 2 && ret < 0 && !bomb; attempt++) {
for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
const int windowBits =
(attempt == 0 ? wrapped : !wrapped) ? (32 + MAX_WBITS) : -MAX_WBITS;
FILE *fpout;
@@ -100,7 +102,7 @@ int hts_zunpack(const char *filename, const char *newfile) {
}
{
hts_boolean ok = HTS_TRUE;
LLint size = 0;
int size = 0;
int zerr = Z_OK;
/* chunked inflate; first chunk in inbuf, single member */
@@ -124,33 +126,27 @@ int hts_zunpack(const char *filename, const char *newfile) {
break;
}
produced = sizeof(outbuf) - strm.avail_out;
size += (LLint) produced;
if (size > maxout) { /* decompression bomb: no retry, no copy */
bomb = HTS_TRUE;
ok = HTS_FALSE;
break;
}
if (produced > 0 &&
fwrite(outbuf, 1, produced, fpout) != produced) {
env_error = HTS_TRUE;
ok = HTS_FALSE;
break;
}
size += (int) produced;
} while (strm.avail_out == 0);
if (!ok || zerr == Z_STREAM_END)
break;
navail = fread(inbuf, 1, sizeof(inbuf), in);
} while (navail > 0);
if (ok && zerr == Z_STREAM_END)
ret = (int) size;
ret = size;
}
inflateEnd(&strm);
fclose(fpout);
}
/* keep a mislabeled identity body verbatim only when provably not
deflate; truncation or a local failure must keep failing (#47) */
if (ret < 0 && !bomb && !wrapped && not_deflate && !env_error &&
!ferror(in)) {
if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
FILE *const fpout =
FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
@@ -176,25 +172,6 @@ int hts_zunpack(const char *filename, const char *newfile) {
return ret;
}
size_t hts_zhead(const void *in, size_t in_len, void *out, size_t out_len) {
z_stream zs;
size_t n = 0;
int err;
memset(&zs, 0, sizeof(zs));
if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
return 0;
zs.next_in = (const Bytef *) in;
zs.avail_in = (uInt) in_len;
zs.next_out = (Bytef *) out;
zs.avail_out = (uInt) out_len;
err = inflate(&zs, Z_SYNC_FLUSH);
if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
n = out_len - zs.avail_out;
inflateEnd(&zs);
return n;
}
int hts_extract_meta(const char *path) {
char catbuff[CATBUFF_SIZE];
unzFile zFile = unzOpen(fconcat(catbuff, sizeof(catbuff), path, "hts-cache/new.zip"));

View File

@@ -44,13 +44,7 @@ Please visit our Website: http://www.httrack.com
/* Library internal definictions */
#ifdef HTS_INTERNAL_BYTECODE
/* Inflate filename into newfile (gzip, zlib or raw deflate); decoded size, or
-1 on a corrupt/truncated stream, an over-budget body, or an I/O failure. A
body provably in no deflate framing is copied verbatim. */
extern int hts_zunpack(const char *filename, const char *newfile);
/* Inflate the head of a gzip/zlib stream, out_len max; 0 if undecodable */
extern size_t hts_zhead(const void *in, size_t in_len, void *out,
size_t out_len);
extern int hts_zunpack(char *filename, char *newfile);
extern int hts_extract_meta(const char *path);
extern const char *hts_get_zerror(int err);
#endif

5
src/httrack.rc Normal file
View File

@@ -0,0 +1,5 @@
// Version resource for httrack.exe, the command line program. See version.rc.
#define VER_FILE_DESCRIPTION "HTTrack Website Copier (command line)"
#define VER_ORIGINAL_FILENAME "httrack.exe"
#define VER_FILETYPE VFT_APP
#include "version.rc"

View File

@@ -108,6 +108,9 @@
<Project>{e76ad871-54c1-45e8-a657-6117adeffb46}</Project>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="httrack.rc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
</Project>

5
src/libhttrack.rc Normal file
View File

@@ -0,0 +1,5 @@
// Version resource for libhttrack.dll. See version.rc.
#define VER_FILE_DESCRIPTION "HTTrack Website Copier engine"
#define VER_ORIGINAL_FILENAME "libhttrack.dll"
#define VER_FILETYPE VFT_DLL
#include "version.rc"

View File

@@ -59,17 +59,15 @@
<ItemDefinitionGroup>
<ClCompile>
<!-- LIBHTTRACK_EXPORTS turns HTSEXT_API into __declspec(dllexport); ZLIB_DLL
imports zlib from its DLL. HTS_USEBROTLI/HTS_USEZSTD enable the br and
zstd content codings. Windows 7 floor, matching WinHTTrack. -->
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;HTS_USEBROTLI=1;HTS_USEZSTD=1;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
imports zlib from its DLL. Windows 7 floor, matching WinHTTrack. -->
<PreprocessorDefinitions>WIN32;_WINDOWS;_MBCS;_USRDLL;LIBHTTRACK_EXPORTS;ZLIB_DLL;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>$(MSBuildThisFileDirectory);$(MSBuildThisFileDirectory)coucal;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<WarningLevel>Level3</WarningLevel>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<CompileAs>CompileAsC</CompileAs>
</ClCompile>
<Link>
<!-- vcpkg auto-links libssl/libcrypto/zlib/brotli/zstd; only the OS import
lib is explicit. -->
<!-- vcpkg auto-links libssl/libcrypto/zlib; only the OS import lib is explicit. -->
<AdditionalDependencies>Ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
@@ -109,7 +107,6 @@
<ClCompile Include="htscache_selftest.c" />
<ClCompile Include="htscatchurl.c" />
<ClCompile Include="htscharset.c" />
<ClCompile Include="htscodec.c" />
<ClCompile Include="htsconcat.c" />
<ClCompile Include="htscore.c" />
<ClCompile Include="htscoremain.c" />
@@ -141,6 +138,9 @@
<ClCompile Include="minizip\zip.c" />
<ClCompile Include="punycode.c" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="libhttrack.rc" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
</Project>

View File

@@ -1,12 +1,9 @@
{
"$schema":
"https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
"$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
"name": "libhttrack",
"version-string": "3.49",
"dependencies": [
"brotli",
"openssl",
"zlib",
"zstd"
"zlib"
]
}

50
src/version.rc Normal file
View File

@@ -0,0 +1,50 @@
// Version resource for libhttrack.dll and httrack.exe. Signing enforces that every
// binary in a release names the same product and version, so both need one.
// Spelled out here because a VERSIONINFO cannot take a version string apart;
// tests/01_engine-version-macros.test fails if this drifts from htsglobal.h.
// The per-binary parts come from libhttrack.rc / httrack.rc.
#include <winver.h>
#ifndef VER_FILE_DESCRIPTION
#define VER_FILE_DESCRIPTION "HTTrack Website Copier"
#endif
#ifndef VER_ORIGINAL_FILENAME
#define VER_ORIGINAL_FILENAME ""
#endif
#ifndef VER_FILETYPE
#define VER_FILETYPE VFT_APP
#endif
VS_VERSION_INFO VERSIONINFO
FILEVERSION 3, 49, 12, 0
PRODUCTVERSION 3, 49, 12, 0
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
#else
FILEFLAGS 0x0L
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VER_FILETYPE
FILESUBTYPE VFT2_UNKNOWN
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0" // U.S. English, Unicode
BEGIN
VALUE "CompanyName", "Xavier Roche"
VALUE "FileDescription", VER_FILE_DESCRIPTION
VALUE "FileVersion", "3.49.12"
VALUE "InternalName", VER_ORIGINAL_FILENAME
VALUE "LegalCopyright", "Copyright (C) 1998-2026 Xavier Roche and other contributors. GNU GPL v3 or later."
VALUE "OriginalFilename", VER_ORIGINAL_FILENAME
VALUE "ProductName", "HTTrack Website Copier"
VALUE "ProductVersion", "3.49-12"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END

View File

@@ -0,0 +1,64 @@
#!/bin/bash
#
# version.rc repeats the version that htsglobal.h declares. Signing enforces that
# every binary in a release reports the same one, so a drift fails the signing
# request on release day rather than the build. Assert the two agree.
set -euo pipefail
src="${top_srcdir:-..}/src"
h="$src/htsglobal.h"
rc="$src/version.rc"
for f in "$h" "$rc"; do
[ -f "$f" ] || {
echo "cannot find $f"
exit 1
}
done
# 3.49-12 (display) and 3.49.12 (dotted).
version=$(sed -n 's/^#define HTTRACK_VERSION[[:space:]][[:space:]]*"\([^"]*\)".*/\1/p' "$h")
versionid=$(sed -n 's/^#define HTTRACK_VERSIONID[[:space:]][[:space:]]*"\([^"]*\)".*/\1/p' "$h")
if [ -z "$version" ] || [ -z "$versionid" ]; then
echo "could not read the version from $h"
exit 1
fi
# The same version, as version.rc states it.
fileversion=$(sed -n 's/^[[:space:]]*FILEVERSION[[:space:]][[:space:]]*\(.*\)$/\1/p' "$rc" | tr -d ' \r')
productversion=$(sed -n 's/^[[:space:]]*PRODUCTVERSION[[:space:]][[:space:]]*\(.*\)$/\1/p' "$rc" | tr -d ' \r')
rc_fileversion=$(sed -n 's/.*VALUE "FileVersion",[[:space:]]*"\([^"]*\)".*/\1/p' "$rc")
rc_productversion=$(sed -n 's/.*VALUE "ProductVersion",[[:space:]]*"\([^"]*\)".*/\1/p' "$rc")
rc_productname=$(sed -n 's/.*VALUE "ProductName",[[:space:]]*"\([^"]*\)".*/\1/p' "$rc")
# 3.49.12 -> 3,49,12,0
expected_numeric="$(echo "$versionid" | tr '.' ','),0"
fail=0
check() { # what expected actual
if [ "$2" != "$3" ]; then
echo "version.rc $1 is \"$3\", but htsglobal.h says it should be \"$2\""
fail=1
fi
}
check FILEVERSION "$expected_numeric" "$fileversion"
check PRODUCTVERSION "$expected_numeric" "$productversion"
check FileVersion "$versionid" "$rc_fileversion"
check ProductVersion "$version" "$rc_productversion"
# Signing pins the product name too.
check ProductName "HTTrack Website Copier" "$rc_productname"
[ "$fail" -eq 0 ] || exit 1
# And the shipped binary must agree with the header it was built from.
out=$(httrack --version)
case "$out" in
*"$version"*) ;;
*)
echo "httrack --version says \"$out\", which does not mention $version"
exit 1
;;
esac
echo "version resource agrees with htsglobal.h: $version ($expected_numeric)"

View File

@@ -1,11 +0,0 @@
#!/bin/bash
#
set -euo pipefail
# brotli/zstd decode, unknown codings, and the decoded-size budget.
dir=$(mktemp -d)
trap 'rm -rf "$dir"' EXIT
httrack -O /dev/null -#test=contentcodings "$dir" run |
grep -q "contentcodings self-test OK"

View File

@@ -1,38 +0,0 @@
#!/bin/bash
#
# br and zstd on the wire: both decode, a junk coding leaves the body alone, an
# undecodable one fails the fetch, and br/zstd are advertised over TLS only.
set -euo pipefail
: "${top_srcdir:=..}"
# Both decoders are needed for the wire fixtures; the codec-off matrix is
# covered by the fenced 01_zlib-contentcodings self-test.
if test "${BROTLI_ENABLED:-yes}" != "yes" || test "${ZSTD_ENABLED:-yes}" != "yes"; then
echo "brotli/zstd not both compiled, skipping"
exit 77
fi
# bad.html (in memory) and bin.dat (direct-to-disk) both carry an undecodable
# coding: the fetch fails and neither leaves coded bytes on disk.
bash "$top_srcdir/tests/local-crawl.sh" --errors 2 --files 5 \
--file-matches 'codec/br.html' 'coded body' \
--file-matches 'codec/zstd.html' 'coded body' \
--file-matches 'codec/junk.html' 'junk coding' \
--file-matches 'codec/ae.html' 'AE=gzip, deflate,' \
--file-not-matches 'codec/ae.html' ' br' \
--file-not-matches 'codec/ae.html' 'zstd' \
--not-found 'codec/bad.html' \
--not-found 'codec/bin.dat' \
--log-found 'Unsupported Content-Encoding' \
httrack 'BASEURL/codec/index.html'
if test "${HTTPS_SUPPORT:-}" == "no"; then
echo "no https support compiled, skipping the TLS leg"
exit 0
fi
bash "$top_srcdir/tests/local-crawl.sh" --tls --errors 2 --files 5 \
--file-matches 'codec/ae.html' 'AE=gzip, deflate, br, zstd,' \
httrack 'BASEURL/codec/index.html'

View File

@@ -15,8 +15,6 @@ TESTS_ENVIRONMENT += PATH=$(top_builddir)/src$(PATH_SEPARATOR)$$PATH
### TESTS_ENVIRONMENT += $(SHLIBPATH_VAR)="$(top_builddir)/src/$(LT_CV_OBJDIR)$${$(SHLIBPATH_VAR):+$(PATH_SEPARATOR)}$$$(SHLIBPATH_VAR)"
TESTS_ENVIRONMENT += ONLINE_UNIT_TESTS=$(ONLINE_UNIT_TESTS)
TESTS_ENVIRONMENT += HTTPS_SUPPORT=$(HTTPS_SUPPORT)
TESTS_ENVIRONMENT += BROTLI_ENABLED=$(BROTLI_ENABLED)
TESTS_ENVIRONMENT += ZSTD_ENABLED=$(ZSTD_ENABLED)
TESTS_ENVIRONMENT += V6_SUPPORT=$(V6_SUPPORT)
TESTS_ENVIRONMENT += top_srcdir=$(top_srcdir)
@@ -68,9 +66,9 @@ TESTS = \
01_engine-urlhack.test \
01_engine-unescape-bounds.test \
01_engine-useragent.test \
01_engine-version-macros.test \
01_engine-xfread.test \
01_zlib-acceptencoding.test \
01_zlib-contentcodings.test \
01_zlib-cache.test \
01_zlib-cache-corrupt.test \
01_zlib-cache-legacy.test \
@@ -122,7 +120,6 @@ TESTS = \
46_local-update-304-resume.test \
47_local-crange-overflow.test \
48_local-crange-memresume.test \
49_local-cookiewall.test \
50_local-contentcodings.test
49_local-cookiewall.test
CLEANFILES = check-network_sh.cache

View File

@@ -14,7 +14,6 @@ stdlib only (http.server + ssl) -- no new build or runtime dependency.
"""
import argparse
import base64
import gzip
import hashlib
import os
@@ -634,75 +633,6 @@ class Handler(SimpleHTTPRequestHandler):
extra_headers=[("Content-Encoding", "gzip")],
)
# --- content codings ---------------------------------------------------
# Canned br/zstd bodies (no brotli/zstd module in the stdlib): both decode
# to CODEC_BODY. Regenerate with the brotli/zstd CLIs over that string.
CODEC_BODY = (
b"<html><head><title>codec</title></head>"
b"<body><p>coded body</p></body></html>"
)
CODEC_BR = base64.b64decode(
"G0sAAAQccqSBBfJlUvOccsDeitqC9CbHwENWiptQj5aExP0mBjjVgy2DF17olLzLo2T2Eg=="
)
CODEC_ZSTD = base64.b64decode(
"KLUv/SBMFQIAFAM8aHRtbD48aGVhZD48dGl0bGU+Y29kZWM8LzwvYm9keT48"
"cGQgPC9wPjwvL2h0bWw+BQA7p8QMDNQ1PgcWhjkG"
)
def route_codec_index(self):
self.send_html(
'\t<a href="br.html">br</a>\n'
'\t<a href="zstd.html">zstd</a>\n'
'\t<a href="junk.html">junk</a>\n'
'\t<a href="bad.html">bad</a>\n'
'\t<a href="bin.dat">bin</a>\n'
'\t<a href="ae.html">ae</a>\n'
)
def route_codec_br(self):
self.send_raw(
self.CODEC_BR, "text/html", extra_headers=[("Content-Encoding", "br")]
)
def route_codec_zstd(self):
self.send_raw(
self.CODEC_ZSTD, "text/html", extra_headers=[("Content-Encoding", "zstd")]
)
# Junk token on a plain body: the page must survive (broken servers do this)
def route_codec_junk(self):
self.send_raw(
b"<html><body><p>junk coding</p></body></html>",
"text/html",
extra_headers=[("Content-Encoding", "utf-8")],
)
# A real coding we have no decoder for: the fetch must fail rather than
# save the coded bytes as the page.
def route_codec_bad(self):
self.send_raw(
b"<html><body><p>never decoded</p></body></html>",
"text/html",
extra_headers=[("Content-Encoding", "compress")],
)
# Same, on a non-HTML body: this takes the direct-to-disk (is_write) branch,
# a different discard path than the in-memory bad.html above.
def route_codec_bin(self):
self.send_raw(
b"\x00\x01\x02 CODED-BINARY-MUST-NOT-LAND \xff\xfe" * 8,
"application/octet-stream",
extra_headers=[("Content-Encoding", "compress")],
)
# Echo what httrack advertised, so a crawl can assert the header.
def route_codec_ae(self):
self.send_raw(
b"<html><body><p>AE=%s</p></body></html>"
% self.headers.get("Accept-Encoding", "").encode(),
"text/html",
)
# --- MIME-type exclusion abort (issue #58) -----------------------------
# A -mime:application/pdf filter must abort the transfer once the header
# arrives, not download the whole body and discard it.
@@ -1385,13 +1315,6 @@ class Handler(SimpleHTTPRequestHandler):
"/gated/index.php": route_gated_index,
"/gated/secret.php": route_gated_secret,
"/robots.txt": route_robots,
"/codec/index.html": route_codec_index,
"/codec/br.html": route_codec_br,
"/codec/zstd.html": route_codec_zstd,
"/codec/junk.html": route_codec_junk,
"/codec/bad.html": route_codec_bad,
"/codec/bin.dat": route_codec_bin,
"/codec/ae.html": route_codec_ae,
"/types/index.html": route_types_index,
"/types/control.php": route_types,
"/types/photo.png": route_types,