mirror of
https://github.com/xroche/httrack.git
synced 2026-07-07 17:36:25 +03:00
Compare commits
1 Commits
remove-dea
...
fix/empty-
| Author | SHA1 | Date | |
|---|---|---|---|
|
02180549f6 |
128
.github/workflows/ci.yml
vendored
128
.github/workflows/ci.yml
vendored
@@ -61,50 +61,6 @@ jobs:
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# Reproduce the Debian buildds: they build in a minimal chroot with no
|
||||
# python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
|
||||
# runners ship python3, so every other job hides this path; here we remove it
|
||||
# before `make check`. This is the guard that would have caught the 3.49.10-1
|
||||
# FTBFS (28_local-pause failed instead of skipping when python3 was absent).
|
||||
buildd-no-python3:
|
||||
name: build (no python3, Debian buildd)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev
|
||||
|
||||
- name: Configure
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test without python3
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Hide every python3* so `command -v python3` fails like it does in the
|
||||
# buildd chroot; masking with /bin/false would still resolve.
|
||||
sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
|
||||
-exec mv {} {}.hidden \;
|
||||
! command -v python3
|
||||
make check
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# Portability: build and test on macOS (Darwin/clang) on a native runner --
|
||||
# no VM. The tree has no __APPLE__ branches, so Darwin exercises the
|
||||
# generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,
|
||||
@@ -232,52 +188,6 @@ jobs:
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# MemorySanitizer catches reads of uninitialized memory (#143's stack-garbage
|
||||
# size filter) that ASan/UBSan miss. It flags any byte an uninstrumented lib
|
||||
# wrote, so the job stays in our own code: offline self-tests only, no openssl
|
||||
# (--disable-https), no zlib cache tests, static (the runtime is not in .so's).
|
||||
msan:
|
||||
name: msan (MemorySanitizer, clang)
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential clang autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev
|
||||
|
||||
- name: Configure (MSan, static, no https)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
autoreconf -fi
|
||||
./configure CC=clang \
|
||||
CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
|
||||
LDFLAGS="-fsanitize=memory" \
|
||||
--disable-https --disable-shared --enable-static
|
||||
|
||||
- name: Build
|
||||
run: make -j"$(nproc)"
|
||||
|
||||
- name: Test (offline self-tests under MSan)
|
||||
env:
|
||||
MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
|
||||
# skipped here (uninstrumented libz floods MSan with false positives).
|
||||
tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
|
||||
make check TESTS="$tests"
|
||||
|
||||
- name: Print the test log on failure
|
||||
if: failure()
|
||||
run: cat tests/test-suite.log 2>/dev/null || true
|
||||
|
||||
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
|
||||
# the configuration users on minimal systems build, and one libssl is not even
|
||||
# installed here so configure cannot silently re-enable it. The matrix above
|
||||
@@ -322,42 +232,30 @@ jobs:
|
||||
deb:
|
||||
name: deb package (lintian)
|
||||
runs-on: ubuntu-24.04
|
||||
# Build and gate inside Debian sid, the upload target. A Debian dpkg-deb
|
||||
# produces archive-legal xz members (an Ubuntu host defaults to zstd, which
|
||||
# the archive's lintian rejects), and sid's lintian carries the same
|
||||
# data-driven checks (embedded-lib fingerprints and the like) the buildds and
|
||||
# UDD apply -- so issues surface here instead of after upload.
|
||||
container: debian:sid
|
||||
steps:
|
||||
- name: Install packaging toolchain
|
||||
run: |
|
||||
set -euo pipefail
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates git \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev \
|
||||
debhelper devscripts lintian fakeroot
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Install packaging toolchain
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
build-essential autoconf automake libtool autoconf-archive \
|
||||
zlib1g-dev libssl-dev \
|
||||
debhelper devscripts lintian fakeroot
|
||||
|
||||
# --unsigned: CI has no GPG key (also skips the release sig/checksums).
|
||||
# mkdeb builds every package then runs the lintian gate (--fail-on=error,
|
||||
# warning); debuild runs the packaged test pass.
|
||||
# debuild builds every package, then lintian gates on errors.
|
||||
#
|
||||
# DEB_BUILD_OPTIONS trims work CI does not need (release builds via
|
||||
# mkdeb.sh are untouched): noautodbgsym drops the -dbgsym packages whose
|
||||
# LTO payloads are slow to compress and that CI never ships; parallel uses
|
||||
# every core.
|
||||
- name: Build and lint Debian packages
|
||||
# every core. We let debuild run its test pass -- the only one now that
|
||||
# mkdeb no longer runs its own -- so CI exercises the packaged tests.
|
||||
- name: Build Debian packages
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The workspace volume is owned by the host runner uid, but the
|
||||
# container runs as root, so mkdeb's git calls (superproject and the
|
||||
# coucal submodule) trip "dubious ownership"; mark them all safe.
|
||||
git config --global --add safe.directory "*"
|
||||
export DEB_BUILD_OPTIONS="noautodbgsym parallel=$(nproc)"
|
||||
bash tools/mkdeb.sh --unsigned --no-release-artifacts
|
||||
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -39,6 +39,3 @@ Makefile
|
||||
|
||||
# Editor / autotools backup files.
|
||||
*~
|
||||
|
||||
# Python bytecode (tests/local-server.py).
|
||||
__pycache__/
|
||||
|
||||
@@ -33,9 +33,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
|
||||
- Be terse. Comment the why, in English; translate French comments you touch.
|
||||
- Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague
|
||||
attributions). Ref: Wikipedia "Signs of AI writing". Claude Code: `/humanizer`.
|
||||
- Behavior change → add a test. Fast path: a hidden `httrack -#test=NAME` engine
|
||||
self-test (registry in `htsselftest.c`; `-#test` lists them) driven by a
|
||||
`tests/NN_*.test`, over a slow crawl.
|
||||
- Behavior change → add a test. Fast path: a hidden `httrack -#N` debug
|
||||
subcommand (`htscoremain.c`) driven by a `tests/NN_*.test`, over a slow crawl.
|
||||
|
||||
## Review your change adversarially (strongly suggested)
|
||||
Before pushing, and when reviewing others, don't skim for bugs:
|
||||
|
||||
@@ -39,10 +39,6 @@ Welcome, and nothing to disclose. Two rules:
|
||||
|
||||
The sign-off covers AI-assisted code too.
|
||||
|
||||
## Translations
|
||||
|
||||
Interface strings live in [`lang/`](lang/). See [lang/README.md](lang/README.md) for the file format and how to add or update a language.
|
||||
|
||||
## Bugs
|
||||
|
||||
Open an issue with the version, OS, command used, and expected vs actual result.
|
||||
|
||||
23
configure.ac
23
configure.ac
@@ -1,6 +1,6 @@
|
||||
AC_PREREQ([2.71])
|
||||
|
||||
AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_INIT([httrack], [3.49.8], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
|
||||
AC_COPYRIGHT([
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2015 Xavier Roche and other contributors
|
||||
@@ -29,11 +29,9 @@ AC_CONFIG_SRCDIR(src/httrack.c)
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
AC_CONFIG_HEADERS(config.h)
|
||||
AM_INIT_AUTOMAKE([subdir-objects])
|
||||
# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
|
||||
# installed headers (no struct layout or exported signature changed vs
|
||||
# 3.49.10), so it stays soname .so.3; bump revision.
|
||||
# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
|
||||
VERSION_INFO="3:3:0"
|
||||
# 4:0:0: htsblk gained the contenttype_given field, an incompatible ABI break,
|
||||
# so bump current and reset revision/age.
|
||||
VERSION_INFO="4:0:0"
|
||||
AM_MAINTAINER_MODE
|
||||
AC_USE_SYSTEM_EXTENSIONS
|
||||
|
||||
@@ -64,16 +62,6 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
|
||||
# Export version info
|
||||
AC_SUBST(VERSION_INFO)
|
||||
|
||||
# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
|
||||
# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
|
||||
HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
|
||||
case "$host_os" in
|
||||
darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
|
||||
*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
|
||||
esac
|
||||
AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
|
||||
[Versioned libhtsjava runtime name, derived from VERSION_INFO])
|
||||
|
||||
### Default CFLAGS
|
||||
DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
|
||||
-Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
|
||||
@@ -226,12 +214,9 @@ AC_SUBST(OPENSSL_LIBS)
|
||||
fi
|
||||
|
||||
### Support IPv6
|
||||
V6_SUPPORT=no
|
||||
AC_CHECK_LIB(c, getaddrinfo, [V6_FLAG="-DINET6"
|
||||
V6_SUPPORT=yes
|
||||
AC_DEFINE(HTS_INET6, 1, [Check for IPv6])], AC_MSG_WARN([*** IPv6 not found IPv6 compatibility disabled]))
|
||||
AC_SUBST(V6_FLAG)
|
||||
AC_SUBST(V6_SUPPORT)
|
||||
|
||||
### Check for LFS
|
||||
AC_CHECK_LIB(c, fopen64, [LFS_FLAG="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE"
|
||||
|
||||
49
debian/changelog
vendored
49
debian/changelog
vendored
@@ -1,47 +1,12 @@
|
||||
httrack (3.49.11-1) unstable; urgency=medium
|
||||
httrack (3.49.8-3) unstable; urgency=medium
|
||||
|
||||
* New upstream release: crawl correctness and security fixes (network-facing
|
||||
buffer overflows, file-type detection, redirect handling) and modernized
|
||||
web defaults; full list in history.txt.
|
||||
* Add DEP-12 upstream metadata (#466).
|
||||
* Bump debhelper compat to 14 (#466).
|
||||
* Drop the redundant Priority field and update the NMU lintian override to
|
||||
the current tag names (#466).
|
||||
* Rename libhttrack3 to libhttrack4 to follow the SONAME bump to
|
||||
libhttrack.so.4: htsblk gained a contenttype_given field, an
|
||||
incompatible ABI change (VERSION_INFO 3 -> 4). The .files wildcard
|
||||
now tracks .so.4* so the runtime libraries land in the right
|
||||
package. New binary package, via NEW.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 05 Jul 2026 00:03:18 +0200
|
||||
|
||||
httrack (3.49.10-2) unstable; urgency=medium
|
||||
|
||||
* Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
|
||||
absent (the local-server tests need python3, which the buildds lack). Add
|
||||
patches/skip-local-pause-test-without-python3.patch to guard the test on
|
||||
python3 up front, like its siblings, so it skips cleanly.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 20:18:46 +0200
|
||||
|
||||
httrack (3.49.10-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: new download-pacing and URL-handling options plus a
|
||||
batch of crawl and robustness fixes (full list in history.txt).
|
||||
* Rewrite debian/copyright in machine-readable DEP-5 format, crediting the
|
||||
bundled minizip, md5 and coucal sources (#415).
|
||||
* Lead the webhttrack browser dependency with chromium so httrack is not
|
||||
dragged into the firefox-esr autoremoval cascade (#436).
|
||||
* Override the embedded-library lint for the bundled minizip (#419).
|
||||
* Bump Standards-Version to 4.7.4 (no changes required).
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 28 Jun 2026 14:01:53 +0200
|
||||
|
||||
httrack (3.49.9-1) unstable; urgency=medium
|
||||
|
||||
* New upstream release: Content-Type and file-type detection fixes (trust a
|
||||
declared Content-Type over a binary URL extension, honor --assume under the
|
||||
delayed type check, keep a known extension against a bogus or empty
|
||||
Content-Type, and avoid an uninitialised read on an empty Content-Type), and
|
||||
restored C++ source-compatibility of the installed headers so reverse
|
||||
dependencies (httraqt) build again.
|
||||
|
||||
-- Xavier Roche <xavier@debian.org> Sun, 21 Jun 2026 17:59:38 +0200
|
||||
-- Xavier Roche <xavier@debian.org> Sat, 20 Jun 2026 19:46:16 +0200
|
||||
|
||||
httrack (3.49.8-2) unstable; urgency=medium
|
||||
|
||||
|
||||
13
debian/control
vendored
13
debian/control
vendored
@@ -1,8 +1,9 @@
|
||||
Source: httrack
|
||||
Section: web
|
||||
Priority: optional
|
||||
Maintainer: Xavier Roche <roche@httrack.com>
|
||||
Standards-Version: 4.7.4
|
||||
Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
|
||||
Standards-Version: 4.7.0
|
||||
Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
|
||||
Rules-Requires-Root: no
|
||||
Homepage: http://www.httrack.com
|
||||
Vcs-Git: https://github.com/xroche/httrack.git
|
||||
@@ -29,7 +30,7 @@ Description: Copy websites to your computer (Offline browser)
|
||||
Package: webhttrack
|
||||
Architecture: any
|
||||
Multi-Arch: foreign
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, chromium | firefox-esr | www-browser
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, firefox-esr | chromium | www-browser
|
||||
Replaces: webhttrack-common (<< 3.43.9-2)
|
||||
Breaks: webhttrack-common (<< 3.43.9-2)
|
||||
Suggests: httrack, httrack-doc
|
||||
@@ -57,13 +58,13 @@ Description: webhttrack common files
|
||||
This package is the common files of webhttrack, website copier and
|
||||
mirroring utility
|
||||
|
||||
Package: libhttrack3
|
||||
Package: libhttrack4
|
||||
Architecture: any
|
||||
Multi-Arch: same
|
||||
Section: libs
|
||||
Depends: ${misc:Depends}, ${shlibs:Depends}
|
||||
Replaces: libhttrack2, httrack (<< 3.49.8-2~)
|
||||
Breaks: libhttrack2, httrack (<< 3.49.8-2~)
|
||||
Replaces: libhttrack3, httrack (<< 3.49.8-3~)
|
||||
Breaks: libhttrack3, httrack (<< 3.49.8-3~)
|
||||
Description: Httrack website copier library
|
||||
This package is the library part of httrack, website copier and mirroring
|
||||
utility
|
||||
|
||||
118
debian/copyright
vendored
118
debian/copyright
vendored
@@ -1,109 +1,21 @@
|
||||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Name: httrack
|
||||
Upstream-Contact: Xavier Roche <roche@httrack.com>
|
||||
Source: https://www.httrack.com/
|
||||
This package was debianized by Xavier Roche <roche@httrack.com> on
|
||||
Fri, 27 Sep 2002 16:42:26 +0200
|
||||
|
||||
Files: *
|
||||
Copyright: 1998-2026 Xavier Roche and other contributors
|
||||
License: GPL-3+
|
||||
Comment:
|
||||
The engine includes contributions from Yann Philippot (src/htsjava.c,
|
||||
src/htsjava.h). htsbasenet.h links against the system OpenSSL library
|
||||
(originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
|
||||
The current Debian maintainer is Xavier Roche <xavier@debian.org>
|
||||
|
||||
Files: src/minizip/*
|
||||
Copyright: 1998-2010 Gilles Vollant
|
||||
2007-2008 Even Rouault
|
||||
2009-2010 Mathias Svensson
|
||||
1990-2000 Info-ZIP
|
||||
License: Zlib
|
||||
Comment:
|
||||
The decryption code in src/minizip/crypt.h and src/minizip/unzip.c derives
|
||||
from the Info-ZIP distribution, distributed under the same terms.
|
||||
Upstream author: Xavier Roche <roche@httrack.com>
|
||||
|
||||
Files: src/md5.c
|
||||
Copyright: 1993 Colin Plumb
|
||||
License: public-domain-md5
|
||||
This code implements the MD5 message-digest algorithm, due to Ron Rivest.
|
||||
It was written by Colin Plumb in 1993, no copyright is claimed. This code
|
||||
is in the public domain; do with it what you wish.
|
||||
Copyright: 1998-2014 Xavier Roche and other contributors
|
||||
|
||||
Files: src/coucal/*
|
||||
Copyright: 2013-2014 Xavier Roche
|
||||
License: BSD-3-clause
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
Files: src/coucal/murmurhash3.h*
|
||||
Copyright: Austin Appleby
|
||||
License: public-domain-murmurhash3
|
||||
MurmurHash3 was written by Austin Appleby, and is placed in the public
|
||||
domain. The author hereby disclaims copyright to this source code.
|
||||
On Debian systems, the complete text of the GNU General Public
|
||||
License version 3 can be found in /usr/share/common-licenses/GPL-3 file.
|
||||
|
||||
Files: html/server/div/com.httrack.WebHTTrack.metainfo.xml
|
||||
Copyright: 1998-2026 Xavier Roche and other contributors
|
||||
License: FSFAP
|
||||
Copying and distribution of this file, with or without modification, are
|
||||
permitted in any medium without royalty provided the copyright notice and
|
||||
this notice are preserved. This file is offered as-is, without any warranty.
|
||||
|
||||
Files: debian/*
|
||||
Copyright: 2002-2026 Xavier Roche <xavier@debian.org>
|
||||
License: GPL-3+
|
||||
|
||||
License: GPL-3+
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
.
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
.
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
.
|
||||
On Debian systems, the complete text of the GNU General Public License
|
||||
version 3 can be found in /usr/share/common-licenses/GPL-3.
|
||||
|
||||
License: Zlib
|
||||
This software is provided 'as-is', without any express or implied warranty.
|
||||
In no event will the authors be held liable for any damages arising from the
|
||||
use of this software.
|
||||
.
|
||||
Permission is granted to anyone to use this software for any purpose,
|
||||
including commercial applications, and to alter it and redistribute it
|
||||
freely, subject to the following restrictions:
|
||||
.
|
||||
1. The origin of this software must not be misrepresented; you must not claim
|
||||
that you wrote the original software. If you use this software in a product,
|
||||
an acknowledgment in the product documentation would be appreciated but is
|
||||
not required.
|
||||
2. Altered source versions must be plainly marked as such, and must not be
|
||||
misrepresented as being the original software.
|
||||
3. This notice may not be removed or altered from any source distribution.
|
||||
|
||||
License: BSD-3-clause
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
.
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
||||
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
3
debian/libhttrack3.files
vendored
3
debian/libhttrack3.files
vendored
@@ -1,3 +0,0 @@
|
||||
usr/lib/*/libhttrack.so.3*
|
||||
usr/lib/*/libhtsjava.so.3*
|
||||
usr/share/httrack/templates
|
||||
8
debian/libhttrack3.lintian-overrides
vendored
8
debian/libhttrack3.lintian-overrides
vendored
@@ -1,8 +0,0 @@
|
||||
# The shared libraries ship without a versioned symbols control file (ABI is
|
||||
# tracked via the SONAME plus a >= upstream-version dependency, see debian/rules).
|
||||
libhttrack3: no-symbols-control-file usr/lib/*
|
||||
|
||||
# Bundled, locally patched minizip (src/minizip): it adds a zipFlush() API the
|
||||
# system libminizip lacks (htscache.c flushes the cache .zip so an interrupted
|
||||
# crawl leaves a valid archive), plus Android/old-zlib portability fixes.
|
||||
libhttrack3: embedded-library *libminizip*
|
||||
3
debian/libhttrack4.files
vendored
Normal file
3
debian/libhttrack4.files
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
usr/lib/*/libhttrack.so.4*
|
||||
usr/lib/*/libhtsjava.so.4*
|
||||
usr/share/httrack/templates
|
||||
3
debian/libhttrack4.lintian-overrides
vendored
Normal file
3
debian/libhttrack4.lintian-overrides
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# The shared libraries ship without a versioned symbols control file (ABI is
|
||||
# tracked via the SONAME plus a >= upstream-version dependency, see debian/rules).
|
||||
libhttrack4: no-symbols-control-file usr/lib/*
|
||||
3
debian/proxytrack.lintian-overrides
vendored
3
debian/proxytrack.lintian-overrides
vendored
@@ -1,3 +0,0 @@
|
||||
# Statically linked against httrack's bundled, patched minizip (see src/minizip
|
||||
# and libhttrack3's override): the zipFlush() API is absent from the system one.
|
||||
proxytrack: embedded-library *libminizip*
|
||||
2
debian/rules
vendored
2
debian/rules
vendored
@@ -135,7 +135,7 @@ binary-arch: build install
|
||||
dh_makeshlibs -a -X/usr/lib/$(DEB_HOST_MULTIARCH)/httrack/libtest --version-info
|
||||
dh_installdeb -a
|
||||
# we depend on the current version (ABI may change)
|
||||
dh_shlibdeps -a -ldebian/libhttrack3/usr/lib/$(DEB_HOST_MULTIARCH)
|
||||
dh_shlibdeps -a -ldebian/libhttrack4/usr/lib/$(DEB_HOST_MULTIARCH)
|
||||
dh_gencontrol -a
|
||||
dh_md5sums -a
|
||||
dh_builddeb -a
|
||||
|
||||
4
debian/source/lintian-overrides
vendored
4
debian/source/lintian-overrides
vendored
@@ -1,6 +1,4 @@
|
||||
# Maintainer uploads sign the changelog as xavier@debian.org while the control
|
||||
# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
|
||||
httrack source: no-nmu-in-changelog
|
||||
httrack source: changelog-should-mention-nmu
|
||||
httrack source: source-nmu-has-incorrect-version-number
|
||||
|
||||
# The bundled HTML pages are the genuine upstream documentation taken from
|
||||
|
||||
6
debian/upstream/metadata
vendored
6
debian/upstream/metadata
vendored
@@ -1,6 +0,0 @@
|
||||
---
|
||||
Repository: https://github.com/xroche/httrack.git
|
||||
Repository-Browse: https://github.com/xroche/httrack
|
||||
Bug-Database: https://github.com/xroche/httrack/issues
|
||||
Bug-Submit: https://github.com/xroche/httrack/issues/new
|
||||
Contact: Xavier Roche <roche@httrack.com>
|
||||
41
history.txt
41
history.txt
@@ -4,47 +4,6 @@ HTTrack Website Copier release history:
|
||||
|
||||
This file lists all changes and fixes that have been made for HTTrack
|
||||
|
||||
3.49-11
|
||||
+ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
|
||||
+ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
|
||||
+ New: follow <source> and <track> media elements as embedded links (#451)
|
||||
+ New: added modern web MIME types to the type/extension table (#448)
|
||||
+ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
|
||||
+ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
|
||||
+ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
|
||||
+ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
|
||||
+ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
|
||||
+ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
|
||||
+ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
|
||||
+ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
|
||||
+ Changed: decode the full WHATWG set of HTML named character references (#443)
|
||||
+ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
|
||||
+ Changed: multiple internal hardening, build, test and CI improvements
|
||||
|
||||
3.49-10
|
||||
+ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
|
||||
+ New: --pause to space out file downloads by a random delay (#185)
|
||||
+ New: --strip-query to drop selected query keys from the dedup naming (#112)
|
||||
+ Changed: split the -%u URL hacks into independent --keep-www-prefix, --keep-double-slashes and --keep-query-order toggles (#271)
|
||||
+ Fixed: follow a redirect Location after dropping its #fragment, instead of requesting the fragment and polluting the saved name (#204)
|
||||
+ Fixed: escaped brackets inside a *[...] filter character class (#148)
|
||||
+ Fixed: honor the server's Content-Range when resuming a partial download, instead of appending overlapping bytes (#198)
|
||||
+ Fixed: abort the download as soon as the response type is excluded by -mime:, instead of fetching then discarding the body (#58)
|
||||
+ Fixed: keep size-based filter rules neutral until the file size is known (#143)
|
||||
+ Fixed: stop the mirror with a clean fatal error on a cache write failure, instead of crashing (#174, #219)
|
||||
+ Fixed: stop the 412/416 partial re-get loop on --continue and --update (#206)
|
||||
+ Fixed: keep an unrecognized URL tail instead of mangling it to .html (#115)
|
||||
+ Fixed: honor --tolerant (-%B) on a broken Content-Length, and fix an out-of-bounds read it exposed (#32, #41)
|
||||
+ Fixed: fall back to the next resolved address when a connection fails or stalls, instead of hanging on a dead IPv6 address
|
||||
+ Fixed: report why a -%L URL list could not be loaded (#49)
|
||||
+ Changed: multiple internal hardening, build and CI improvements
|
||||
|
||||
.49-9
|
||||
+ Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
|
||||
+ Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
|
||||
+ Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)
|
||||
+ Changed: multiple internal build, packaging and test-harness improvements
|
||||
|
||||
3.49-8
|
||||
+ New: tunnel HTTPS downloads through the configured HTTP proxy via CONNECT (#85)
|
||||
+ New: parse every candidate URL in <img> and <source> srcset lists (#326)
|
||||
|
||||
@@ -247,7 +247,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
|
||||
<td>the \ character</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td nowrap><tt>*[\[,\]]</tt></td>
|
||||
<td nowrap><tt>*[\[\]]</tt></td>
|
||||
<td>the [ or ] character</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
@@ -295,7 +295,7 @@ Max Depth
|
||||
Maximum external depth:
|
||||
Maximum external depth:
|
||||
Filters (refuse/accept links) :
|
||||
Filters (refuse/accept links):
|
||||
Filters (refuse/accept links) :
|
||||
Paths
|
||||
Paths
|
||||
Save prefs
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
# Translating HTTrack
|
||||
|
||||
Interface strings live here, one `.txt` file per language. `English.txt` is the reference: every other file maps each English string to its translation.
|
||||
|
||||
## File format
|
||||
|
||||
Plain text, entries in consecutive pairs of lines:
|
||||
|
||||
```
|
||||
<English string>
|
||||
<translation>
|
||||
```
|
||||
|
||||
The first line of a pair is the lookup key and must stay identical to the one in `English.txt`; translate only the second line. Missing entries fall back to the English text at runtime, so a partial translation works.
|
||||
|
||||
Preserve any `\r\n`, `\t` and `printf` placeholders (`%s`, `%d`, ...) in the translation.
|
||||
|
||||
A few `LANGUAGE_*` entries at the top describe the file itself:
|
||||
|
||||
| Key | Meaning |
|
||||
| --- | --- |
|
||||
| `LANGUAGE_NAME` | Name shown in the language picker, in its own language (`Deutsch`, not `German`) |
|
||||
| `LANGUAGE_ISO` | ISO 639 code, with region if needed (`de`, `pt_BR`) |
|
||||
| `LANGUAGE_CHARSET` | Encoding the file is saved in (`ISO-8859-1`, `windows-1251`, `UTF-8`, ...) |
|
||||
| `LANGUAGE_AUTHOR` | Your name and contact |
|
||||
| `LANGUAGE_WINDOWSID` | Windows locale name used by WinHTTrack (`German (Standard)`) |
|
||||
|
||||
Save the file in exactly its declared `LANGUAGE_CHARSET`; an editor that rewrites it as UTF-8 will corrupt the non-ASCII bytes.
|
||||
|
||||
## Adding or updating a language
|
||||
|
||||
1. Copy `English.txt` to `<Language>.txt`, or edit the existing file.
|
||||
2. Translate each second line; leave the English keys untouched.
|
||||
3. Fill in the `LANGUAGE_*` header for a new file.
|
||||
4. Open a pull request, or attach the file to a GitHub issue.
|
||||
|
||||
When new strings land in `English.txt` they show up untranslated (as English) until a translator fills them in.
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "27 June 2026" "httrack website copier"
|
||||
.TH httrack 1 "13 June 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -24,7 +24,6 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-EN, \-\-max\-time[=N]\fR ]
|
||||
[ \fB\-AN, \-\-max\-rate[=N]\fR ]
|
||||
[ \fB\-%cN, \-\-connection\-per\-second[=N]\fR ]
|
||||
[ \fB\-%G, \-\-pause\fR ]
|
||||
[ \fB\-GN, \-\-max\-pause[=N]\fR ]
|
||||
[ \fB\-cN, \-\-sockets[=N]\fR ]
|
||||
[ \fB\-TN, \-\-timeout[=N]\fR ]
|
||||
@@ -44,13 +43,11 @@ httrack \- offline browser : copy websites to a local directory
|
||||
[ \fB\-x, \-\-replace\-external\fR ]
|
||||
[ \fB\-%x, \-\-disable\-passwords\fR ]
|
||||
[ \fB\-%q, \-\-include\-query\-string\fR ]
|
||||
[ \fB\-%g, \-\-strip\-query\fR ]
|
||||
[ \fB\-o, \-\-generate\-errors\fR ]
|
||||
[ \fB\-X, \-\-purge\-old[=N]\fR ]
|
||||
[ \fB\-%p, \-\-preserve\fR ]
|
||||
[ \fB\-%T, \-\-utf8\-conversion\fR ]
|
||||
[ \fB\-bN, \-\-cookies[=N]\fR ]
|
||||
[ \fB\-%K, \-\-cookies\-file\fR ]
|
||||
[ \fB\-u, \-\-check\-type[=N]\fR ]
|
||||
[ \fB\-j, \-\-parse\-java[=N]\fR ]
|
||||
[ \fB\-sN, \-\-robots[=N]\fR ]
|
||||
@@ -156,8 +153,6 @@ maximum mirror time in seconds (60=1 minute, 3600=1 hour) (\-\-max\-time[=N])
|
||||
maximum transfer rate in bytes/seconds (1000=1KB/s max) (\-\-max\-rate[=N])
|
||||
.IP \-%cN
|
||||
maximum number of connections/seconds (*%c10) (\-\-connection\-per\-second[=N])
|
||||
.IP \-%G
|
||||
random pause of MIN[:MAX] seconds between files (e.g. %G5:10) (\-\-pause <param>)
|
||||
.IP \-GN
|
||||
pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\-pause[=N])
|
||||
.SS Flow control:
|
||||
@@ -203,8 +198,6 @@ replace external html links by error pages (\-\-replace\-external)
|
||||
do not include any password for external password protected websites (%x0 include) (\-\-disable\-passwords)
|
||||
.IP \-%q
|
||||
*include query string for local files (useless, for information purpose only) (%q0 don't include) (\-\-include\-query\-string)
|
||||
.IP \-%g
|
||||
strip query keys for dedup ([host/pattern=]key1,key2,...) (\-\-strip\-query <param>)
|
||||
.IP \-o
|
||||
*generate output html file in case of error (404..) (o0 don't generate) (\-\-generate\-errors)
|
||||
.IP \-X
|
||||
@@ -216,8 +209,6 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
|
||||
.SS Spider options:
|
||||
.IP \-bN
|
||||
accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
|
||||
.IP \-%K
|
||||
load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
|
||||
.IP \-u
|
||||
check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
|
||||
.IP \-j
|
||||
@@ -234,8 +225,6 @@ tolerant requests (accept bogus responses on some servers, but not standard!) (\
|
||||
update hacks: various hacks to limit re\-transfers when updating (identical size, bogus response..) (\-\-updatehack)
|
||||
.IP \-%u
|
||||
url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..) (\-\-urlhack)
|
||||
.br
|
||||
opt out of one url\-hack part: \-\-keep\-www\-prefix (www.foo.com<>foo.com), \-\-keep\-double\-slashes (//), \-\-keep\-query\-order (?b&a)
|
||||
.IP \-%A
|
||||
assume that a type (cgi,asp..) is always linked with a mime type (\-%A php3,cgi=text/html;dat,bin=application/x\-zip) (\-\-assume <param>)
|
||||
.br
|
||||
@@ -324,8 +313,12 @@ debug HTTP headers in logfile (\-\-debug\-headers)
|
||||
.SS Guru options: (do NOT use if possible)
|
||||
.IP \-#X
|
||||
*use optimized engine (limited memory boundary checks) (\-\-fast\-engine)
|
||||
.IP \-#test
|
||||
list engine self\-tests (run one with \-#test=NAME [args])
|
||||
.IP \-#0
|
||||
filter test (\-#0 '*.gif' 'www.bar.com/foo.gif') (\-\-debug\-testfilters <param>)
|
||||
.IP \-#1
|
||||
simplify test (\-#1 ./foo/bar/../foobar)
|
||||
.IP \-#2
|
||||
type test (\-#2 /foo/bar.php)
|
||||
.IP \-#C
|
||||
cache list (\-#C '*.com/spider*.gif' (\-\-debug\-cache <param>)
|
||||
.IP \-#R
|
||||
|
||||
@@ -56,21 +56,21 @@ whttrackrundir = $(bindir)
|
||||
whttrackrun_SCRIPTS = webhttrack
|
||||
|
||||
libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htscache_selftest.c htsdns_selftest.c htsselftest.c \
|
||||
htscache_selftest.c \
|
||||
htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
|
||||
htshelp.c htslib.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
|
||||
htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h \
|
||||
htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htscatchurl.h \
|
||||
htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h \
|
||||
htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
|
||||
htshelp.h htsindex.h htslib.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsmodules.h htsname.h htsnet.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
|
||||
@@ -60,9 +60,6 @@ Please visit our Website: http://www.httrack.com
|
||||
param1 : this option must be alone, and needs one distinct parameter (-P <path>)
|
||||
param0 : this option must be alone, but the parameter should be put together (+*.gif)
|
||||
*/
|
||||
/* clang-format off: hand-aligned table; clang-format reflows the whole
|
||||
initializer (2->4 space) on any edit, churning every untouched row. */
|
||||
/* clang-format off */
|
||||
const char *hts_optalias[][4] = {
|
||||
/* {"","","",""}, */
|
||||
{"path", "-O", "param1", "output path"},
|
||||
@@ -110,12 +107,6 @@ const char *hts_optalias[][4] = {
|
||||
{"disable-passwords", "-%x", "single", ""}, {"disable-password", "-%x",
|
||||
"single", ""},
|
||||
{"include-query-string", "-%q", "single", ""},
|
||||
{"strip-query", "-%g", "param1",
|
||||
"strip [host/pattern=]key1,key2,... from URLs"},
|
||||
{"cookies-file", "-%K", "param1",
|
||||
"load extra cookies from a Netscape cookies.txt"},
|
||||
{"pause", "-%G", "param1",
|
||||
"random pause of MIN[:MAX] seconds between files"},
|
||||
{"generate-errors", "-o", "single", ""},
|
||||
{"do-not-generate-errors", "-o0", "single", ""},
|
||||
{"purge-old", "-X", "param", ""},
|
||||
@@ -132,9 +123,6 @@ const char *hts_optalias[][4] = {
|
||||
{"tolerant", "-%B", "single", ""},
|
||||
{"updatehack", "-%s", "single", ""}, {"sizehack", "-%s", "single", ""},
|
||||
{"urlhack", "-%u", "single", ""},
|
||||
{"keep-www-prefix", "-%j", "single", ""},
|
||||
{"keep-double-slashes", "-%o", "single", ""},
|
||||
{"keep-query-order", "-%y", "single", ""},
|
||||
{"user-agent", "-F", "param1", "user-agent identity"},
|
||||
{"referer", "-%R", "param1", "default referer URL"},
|
||||
{"from", "-%E", "param1", "from email address"},
|
||||
@@ -253,7 +241,6 @@ const char *hts_optalias[][4] = {
|
||||
|
||||
{"", "", "", ""}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
/*
|
||||
Check for alias in command-line
|
||||
@@ -283,6 +270,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
|
||||
char *position;
|
||||
int need_param = 1;
|
||||
|
||||
//int return_param=0;
|
||||
int pos;
|
||||
|
||||
command[0] = param[0] = addcommand[0] = '\0';
|
||||
@@ -360,6 +348,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
|
||||
strlcatbuff(return_argv[0], "0", return_argv_size);
|
||||
else if (strcmp(param, "on") == 0) {
|
||||
// on is the default
|
||||
// strcatbuff(return_argv[0],"1");
|
||||
} else
|
||||
strlcatbuff(return_argv[0], param, return_argv_size);
|
||||
}
|
||||
|
||||
520
src/htsback.c
520
src/htsback.c
@@ -43,22 +43,21 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htsback.h"
|
||||
|
||||
//#ifdef _WIN32
|
||||
#include "htsftp.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#else
|
||||
#error HTS_USEZLIB not defined
|
||||
#endif
|
||||
//#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef __cplusplus
|
||||
// DOS
|
||||
#include <process.h> /* _beginthread, _endthread */
|
||||
#endif
|
||||
#include <io.h> /* _chsize_s */
|
||||
#define HTS_FTRUNCATE(fp, sz) _chsize_s(_fileno(fp), (sz))
|
||||
#else
|
||||
#define HTS_FTRUNCATE(fp, sz) ftruncate(fileno(fp), (sz))
|
||||
#endif
|
||||
|
||||
#define VT_CLREOL "\33[K"
|
||||
@@ -74,8 +73,6 @@ struct_back *back_new(httrackp *opt, int back_max) {
|
||||
|
||||
sback->count = back_max;
|
||||
sback->lnk = (lien_back *) calloct((back_max + 1), sizeof(lien_back));
|
||||
sback->connect_fallback = (hts_connect_fallback *) calloct(
|
||||
(back_max + 1), sizeof(hts_connect_fallback));
|
||||
sback->ready = coucal_new(0);
|
||||
hts_set_hash_handler(sback->ready, opt);
|
||||
coucal_set_name(sback->ready, "back_new");
|
||||
@@ -86,7 +83,6 @@ struct_back *back_new(httrackp *opt, int back_max) {
|
||||
sback->lnk[i].r.location = sback->lnk[i].location_buffer;
|
||||
sback->lnk[i].status = STATUS_FREE;
|
||||
sback->lnk[i].r.soc = INVALID_SOCKET;
|
||||
sback->connect_fallback[i].addr_count = -1; // not yet probed
|
||||
}
|
||||
return sback;
|
||||
}
|
||||
@@ -97,7 +93,6 @@ void back_free(struct_back ** sback) {
|
||||
freet((*sback)->lnk);
|
||||
(*sback)->lnk = NULL;
|
||||
}
|
||||
freet((*sback)->connect_fallback);
|
||||
if ((*sback)->ready != NULL) {
|
||||
coucal_delete(&(*sback)->ready);
|
||||
(*sback)->ready_size_bytes = 0;
|
||||
@@ -107,72 +102,6 @@ void back_free(struct_back ** sback) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Per-candidate connect deadline cap (seconds): a connecting slot with another
|
||||
address to try waits at most this long before falling back, instead of the
|
||||
full (default 120s) slot timeout. Caps the dead-IPv6 stall while staying well
|
||||
above a normal handshake. The last candidate still gets the full timeout. */
|
||||
#define HTS_CONNECT_FALLBACK_TIMEOUT 10
|
||||
|
||||
int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
|
||||
int timeout) {
|
||||
int deadline;
|
||||
|
||||
if (addr_index + 1 >= addr_count) // last (or only) candidate: no fallback
|
||||
return 0;
|
||||
if (timeout <= 0) // no timeout management: never force it
|
||||
return 0;
|
||||
deadline = (timeout < HTS_CONNECT_FALLBACK_TIMEOUT)
|
||||
? timeout
|
||||
: HTS_CONNECT_FALLBACK_TIMEOUT;
|
||||
return elapsed >= deadline;
|
||||
}
|
||||
|
||||
/* Pending-connect result for a non-blocking socket reported ready by select():
|
||||
0 = connected, >0 = the connect errno (refused, unreachable, ...), -1 if the
|
||||
probe itself failed. A failed connect is reported writable too, so this is
|
||||
how success is told from failure without blocking. */
|
||||
static int connect_socket_error(T_SOC soc) {
|
||||
int soerr = 0;
|
||||
socklen_t len = (socklen_t) sizeof(soerr);
|
||||
|
||||
if (getsockopt(soc, SOL_SOCKET, SO_ERROR, (char *) &soerr, &len) != 0)
|
||||
return -1;
|
||||
return soerr;
|
||||
}
|
||||
|
||||
/* Retry a stuck/failed connecting slot against its next resolved address.
|
||||
Closes the current socket and starts a non-blocking connect to the next
|
||||
candidate, leaving the slot in STATUS_CONNECTING. Returns 1 if a new connect
|
||||
was started, 0 if no fallback address remains (caller fails the slot). */
|
||||
static int back_connect_next(httrackp *opt, struct_back *sback, int i) {
|
||||
hts_connect_fallback *const cf = &sback->connect_fallback[i];
|
||||
lien_back *const back = sback->lnk;
|
||||
const int next = cf->addr_index + 1;
|
||||
T_SOC soc;
|
||||
|
||||
if (next >= cf->addr_count)
|
||||
return 0;
|
||||
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
}
|
||||
soc = newhttp_addr(opt, back[i].url_adr, &back[i].r, -1, 0, next, NULL);
|
||||
if (soc == INVALID_SOCKET)
|
||||
return 0;
|
||||
|
||||
back[i].r.soc = soc;
|
||||
cf->addr_index = next;
|
||||
cf->connect_start = time_local();
|
||||
if (back[i].timeout > 0)
|
||||
back[i].timeout_refresh = cf->connect_start;
|
||||
back[i].status = STATUS_CONNECTING;
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"connect failed, trying next address (%d/%d) for %s", next + 1,
|
||||
cf->addr_count, back[i].url_adr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback) {
|
||||
if (sback != NULL) {
|
||||
int i;
|
||||
@@ -570,12 +499,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
&& back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
|
||||
if (back[p].status == STATUS_READY) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"incomplete transfer (expected " LLintP
|
||||
" bytes, got " LLintP
|
||||
"): file not cached, will be retried on the next update"
|
||||
" (use -%%B to cache anyway): %s%s",
|
||||
back[p].r.totalsize, back[p].r.size, back[p].url_adr,
|
||||
back[p].url_fil);
|
||||
"file not stored in cache due to bogus state (broken size, expected "
|
||||
LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
|
||||
back[p].r.size, back[p].url_adr, back[p].url_fil);
|
||||
} else {
|
||||
hts_log_print(opt, LOG_INFO,
|
||||
"incomplete file not yet stored in cache (expected "
|
||||
@@ -595,6 +521,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
#if HTS_USEZLIB
|
||||
if (back[p].r.compressed) {
|
||||
if (back[p].r.size > 0) {
|
||||
//if ( (back[p].r.adr) && (back[p].r.size>0) ) {
|
||||
// stats
|
||||
back[p].compressed_size = back[p].r.size;
|
||||
// en mémoire -> passage sur disque
|
||||
@@ -879,12 +806,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
back[p].url_fil, NULL);
|
||||
} else {
|
||||
/* Partial file, but marked as "ok" ? */
|
||||
hts_log_print(
|
||||
opt, LOG_WARNING,
|
||||
"file with unresolved type not cached (%s (%d), size " LLintP
|
||||
"): %s%s",
|
||||
back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
|
||||
back[p].url_adr, back[p].url_fil);
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file not stored in cache due to bogus state (incomplete type with %s (%d), size "
|
||||
LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
|
||||
(LLint) back[p].r.size, back[p].url_adr,
|
||||
back[p].url_fil);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -924,6 +850,7 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
/* clear everything but connection: switch, close, and reswitch */
|
||||
back_connxfr(src, &tmp);
|
||||
back_delete(opt, cache, sback, p);
|
||||
//deletehttp(src);
|
||||
back_connxfr(&tmp, src);
|
||||
src->req.flush_garbage = 1; /* ignore CRLF garbage */
|
||||
return 1;
|
||||
@@ -1358,49 +1285,7 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Move a still-writing .delayed placeholder to its final name (#483). */
|
||||
hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
|
||||
const char *newname) {
|
||||
hts_boolean renamed;
|
||||
|
||||
if (!back->r.is_write || back->tmpfile != NULL ||
|
||||
!IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
|
||||
return HTS_TRUE; /* nothing bound to the placeholder name */
|
||||
if (back->r.out != NULL) {
|
||||
fclose(back->r.out);
|
||||
back->r.out = NULL;
|
||||
}
|
||||
renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
if (renamed && (back->status == STATUS_READY ||
|
||||
(back->r.out = FOPEN(newname, "ab")) != NULL)) {
|
||||
filenote(&opt->state.strc, newname, NULL);
|
||||
hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
|
||||
newname);
|
||||
return HTS_TRUE;
|
||||
}
|
||||
/* partial lost: drop only what we own (Windows rename won't overwrite) */
|
||||
hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
|
||||
back->url_sav, newname);
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(back->r.msg, "Write error on disk");
|
||||
back->r.is_write = 0;
|
||||
(void) UNLINK(renamed ? newname : back->url_sav);
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
// effacer entrée
|
||||
/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
|
||||
void back_delayed_discard(httrackp *opt, lien_back *back) {
|
||||
if (back->r.out != NULL) {
|
||||
fclose(back->r.out);
|
||||
back->r.out = NULL;
|
||||
}
|
||||
back->r.is_write = 0;
|
||||
if (opt != NULL)
|
||||
url_savename_refname_remove(opt, back->url_adr, back->url_fil);
|
||||
(void) UNLINK(back->url_sav);
|
||||
}
|
||||
|
||||
int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p) {
|
||||
lien_back *const back = sback->lnk;
|
||||
@@ -1408,12 +1293,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
|
||||
assertf(p >= 0 && p < back_max);
|
||||
if (p >= 0 && p < sback->count) { // on sait jamais..
|
||||
/* mid-write cancel: drop a .delayed placeholder; real-named partials
|
||||
survive for resume (--continue) */
|
||||
if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
|
||||
(back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
|
||||
back_delayed_discard(opt, &back[p]);
|
||||
}
|
||||
// Vérificateur d'intégrité
|
||||
#if DEBUG_CHECKINT
|
||||
_CHECKINT(&back[p], "Appel back_delete")
|
||||
@@ -1433,6 +1312,7 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
back[p].url_adr, back[p].url_fil, back[p].url_sav);
|
||||
}
|
||||
if (cache != NULL) {
|
||||
//hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
|
||||
back_finalize(opt, cache, sback, p);
|
||||
}
|
||||
}
|
||||
@@ -1573,6 +1453,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
if (back[p].r.soc != INVALID_SOCKET) { /* we never know */
|
||||
deletehttp(&back[p].r);
|
||||
}
|
||||
//memset(&(back[p].r), 0, sizeof(htsblk));
|
||||
hts_init_htsblk(&back[p].r);
|
||||
back[p].r.location = back[p].location_buffer;
|
||||
|
||||
@@ -1580,6 +1461,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
strcpybuff(back[p].url_adr, adr);
|
||||
strcpybuff(back[p].url_fil, fil);
|
||||
strcpybuff(back[p].url_sav, save);
|
||||
//back[p].links_index = links_index;
|
||||
// copier referer si besoin
|
||||
strcpybuff(back[p].referer_adr, "");
|
||||
strcpybuff(back[p].referer_fil, "");
|
||||
@@ -1818,7 +1700,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
back[p].url_adr, back[p].url_fil);
|
||||
}
|
||||
back[p].r.notmodified = 1; // fichier non modifié
|
||||
back[p].status = STATUS_READY; // OK prêt
|
||||
back[p].status = STATUS_READY; // OK prêt
|
||||
//file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified); // not modified
|
||||
back_set_finished(sback, p);
|
||||
|
||||
// finalize transfer
|
||||
@@ -1833,6 +1716,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
} else { // erreur
|
||||
// effacer r
|
||||
hts_init_htsblk(&back[p].r);
|
||||
//memset(&(back[p].r), 0, sizeof(htsblk));
|
||||
back[p].r.location = back[p].location_buffer;
|
||||
// et continuer (chercher le fichier)
|
||||
}
|
||||
@@ -2027,10 +1911,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
// ouvrir liaison, envoyer requète
|
||||
// ne pas traiter ou recevoir l'en tête immédiatement
|
||||
hts_init_htsblk(&back[p].r);
|
||||
//memset(&(back[p].r), 0, sizeof(htsblk));
|
||||
back[p].r.location = back[p].location_buffer;
|
||||
// fresh connect: address list not yet probed, start at the first
|
||||
sback->connect_fallback[p].addr_index = 0;
|
||||
sback->connect_fallback[p].addr_count = -1;
|
||||
// recopier proxy
|
||||
if ((back[p].r.req.proxy.active = opt->proxy.active)) {
|
||||
if (StringBuff(opt->proxy.bindhost) != NULL)
|
||||
@@ -2077,6 +1959,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
#if HTS_USEOPENSSL
|
||||
else if (strfield(back[p].url_adr, "https://")) { // let's rock
|
||||
back[p].r.ssl = 1;
|
||||
// back[p].r.ssl_soc = NULL;
|
||||
back[p].r.ssl_con = NULL;
|
||||
}
|
||||
#endif
|
||||
@@ -2142,9 +2025,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
back[p].rateout = -1; // pas de gestion (default)
|
||||
}
|
||||
|
||||
// Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
|
||||
// cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
|
||||
// page!)
|
||||
// Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
|
||||
// rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
|
||||
//if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
|
||||
|
||||
#if CNXDEBUG
|
||||
printf("Xfopen ok, poll..\n");
|
||||
@@ -2162,6 +2045,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
if (soc == INVALID_SOCKET) { // erreur socket
|
||||
back[p].status = STATUS_READY; // FINI
|
||||
back_set_finished(sback, p);
|
||||
//if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
|
||||
back[p].r.soc = INVALID_SOCKET;
|
||||
} else {
|
||||
if (!back[p].r.is_file)
|
||||
@@ -2182,6 +2066,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
|
||||
// note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
|
||||
// le lien est considéré comme traité
|
||||
//if (back[p].soc<0) // erreur
|
||||
// return -1;
|
||||
|
||||
return 0;
|
||||
} else {
|
||||
@@ -2254,6 +2140,9 @@ void back_solve(httrackp * opt, lien_back * back) {
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
|
||||
}
|
||||
//if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
|
||||
// hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
|
||||
//}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2272,13 +2161,12 @@ int host_wait(httrackp * opt, lien_back * back) {
|
||||
|
||||
static int slot_can_be_cleaned(const lien_back * back) {
|
||||
return (back->status == STATUS_READY) // ready
|
||||
/* Check autoclean */
|
||||
&& (!back->locked) // not held by hts_wait_delayed (name pending)
|
||||
&& (!back->testmode) // not test mode
|
||||
&& (strnotempty(back->url_sav)) // filename exists
|
||||
&& (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
|
||||
&& (back->r.size >= 0) // size>=0
|
||||
;
|
||||
/* Check autoclean */
|
||||
&& (!back->testmode) // not test mode
|
||||
&& (strnotempty(back->url_sav)) // filename exists
|
||||
&& (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
|
||||
&& (back->r.size >= 0) // size>=0
|
||||
;
|
||||
}
|
||||
|
||||
static int slot_can_be_finalized(httrackp * opt, const lien_back * back) {
|
||||
@@ -2302,6 +2190,11 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
|
||||
(void) back_flush_output(opt, cache, sback, i); // flush output buffers
|
||||
usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
//if (back[i].links_index >= 0) {
|
||||
// assertf(back[i].links_index < opt->hash->max_lien);
|
||||
// opt->hash->liens[back[i].links_index]->pass2 = -1;
|
||||
// // *back[i].pass2_ptr=-1; // Done!
|
||||
//}
|
||||
/* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
|
||||
{
|
||||
int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME ); // lecture type 0 (sav)
|
||||
@@ -2449,34 +2342,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back_clean(opt, cache, sback);
|
||||
#endif
|
||||
|
||||
/* Time limit exceeded past grace: abort in-flight transfers so no wait loop
|
||||
starves (#481). FTP slots stay, their thread owns the socket. */
|
||||
if (!back_checkmirror(opt)) {
|
||||
int aborted = 0;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < (unsigned int) back_max; i++) {
|
||||
if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
/* drop a .delayed placeholder; real partials survive for resume */
|
||||
if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
|
||||
back_delayed_discard(opt, &back[i]);
|
||||
back[i].r.statuscode = STATUSCODE_TIMEOUT;
|
||||
strcpybuff(back[i].r.msg, "Mirror Time Out");
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
aborted++;
|
||||
}
|
||||
}
|
||||
if (aborted > 0)
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"time limit reached, %d transfer(s) aborted", aborted);
|
||||
return;
|
||||
}
|
||||
|
||||
// recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
|
||||
do_wait = 0;
|
||||
gestion_timeout = 0;
|
||||
@@ -2485,6 +2350,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
busy_state = busy_recv = 0;
|
||||
|
||||
#if 0
|
||||
check_rate(stat_timestart, opt->maxrate); // vérifier taux de transfert
|
||||
#endif
|
||||
// inscrire les sockets actuelles, et rechercher l'ID la plus élevée
|
||||
FD_ZERO(&fds);
|
||||
FD_ZERO(&fds_c);
|
||||
@@ -2494,31 +2362,28 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
nfds = INVALID_SOCKET;
|
||||
|
||||
max_c = 1;
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
// for(i=0;i<back_max;i++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
// en cas de gestion du connect préemptif
|
||||
#if HTS_XCONN
|
||||
if (back[i].status == STATUS_CONNECTING) { // connexion
|
||||
// a connecting slot always carries a live socket; guard anyway so a
|
||||
// stray INVALID_SOCKET can never reach FD_SET (mirrors the recv branch)
|
||||
if (back[i].r.soc != INVALID_SOCKET) {
|
||||
do_wait = 1;
|
||||
do_wait = 1;
|
||||
|
||||
// noter socket write
|
||||
FD_SET(back[i].r.soc, &fds_c);
|
||||
// noter socket write
|
||||
FD_SET(back[i].r.soc, &fds_c);
|
||||
|
||||
// noter socket erreur
|
||||
FD_SET(back[i].r.soc, &fds_e);
|
||||
// noter socket erreur
|
||||
FD_SET(back[i].r.soc, &fds_e);
|
||||
|
||||
// calculer max
|
||||
if (max_c) {
|
||||
max_c = 0;
|
||||
nfds = back[i].r.soc;
|
||||
} else if (back[i].r.soc > nfds) {
|
||||
// ID socket la plus élevée
|
||||
nfds = back[i].r.soc;
|
||||
}
|
||||
// calculer max
|
||||
if (max_c) {
|
||||
max_c = 0;
|
||||
nfds = back[i].r.soc;
|
||||
} else if (back[i].r.soc > nfds) {
|
||||
// ID socket la plus élevée
|
||||
nfds = back[i].r.soc;
|
||||
}
|
||||
|
||||
} else
|
||||
@@ -2531,7 +2396,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// poll pour la lecture sur les sockets
|
||||
if ((back[i].status > 0) && (back[i].status < 100)) { // en réception http
|
||||
|
||||
#if BDEBUG == 1
|
||||
#if BDEBUG==1
|
||||
//printf("....socket in progress: %d\n",back[i].r.soc);
|
||||
#endif
|
||||
// non local et non ftp
|
||||
if (!back[i].r.is_file) {
|
||||
@@ -2617,7 +2483,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
busy_recv = 0;
|
||||
|
||||
// recevoir les données arrivées
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
// for(i=0;i<back_max;i++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
if (back[i].status > 0) {
|
||||
@@ -2650,20 +2517,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
// ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
|
||||
if (back[i].status == STATUS_CONNECTING) { // attendre connect
|
||||
hts_connect_fallback *const cf = &sback->connect_fallback[i];
|
||||
int dispo = 0;
|
||||
|
||||
// probe the resolved address list once per fresh connect (cache hit:
|
||||
// the host was resolved when this connect was opened)
|
||||
if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
|
||||
!back[i].r.is_file) {
|
||||
SOCaddr scratch[HTS_MAXADDRNUM];
|
||||
|
||||
cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
|
||||
HTS_MAXADDRNUM, NULL);
|
||||
cf->connect_start = time_local();
|
||||
}
|
||||
|
||||
// vérifier l'existance de timeout-check
|
||||
if (!gestion_timeout)
|
||||
if (back[i].timeout > 0)
|
||||
@@ -2671,20 +2526,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
// connecté?
|
||||
dispo = FD_ISSET(back[i].r.soc, &fds_c);
|
||||
if (dispo) { // socket ready: connect() finished (ok or failed)
|
||||
// a refused/failed connect is reported writable too; probe SO_ERROR
|
||||
// and, on failure, fall back to the next address (or fail the slot)
|
||||
if (connect_socket_error(back[i].r.soc) != 0) {
|
||||
if (!back_connect_next(opt, sback, i)) {
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_CONNERROR;
|
||||
strcpybuff(back[i].r.msg, "Connect Error");
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
}
|
||||
continue; // reconnected (stay connecting) or failed
|
||||
}
|
||||
if (dispo) { // ok connected!!
|
||||
busy_state = 1;
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
@@ -2757,6 +2599,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].rateout_time = back[i].ka_time_start;
|
||||
}
|
||||
// envoyer header
|
||||
//if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0) // vrai get
|
||||
HTS_STAT.stat_nrequests++;
|
||||
if (!back[i].head_request)
|
||||
http_sendhead(opt, opt->cookie, 0, back[i].send_too,
|
||||
@@ -2823,6 +2666,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
#if HTS_XGETHOST
|
||||
else if (back[i].status == STATUS_WAIT_DNS) { // attendre gethostbyname
|
||||
#if DEBUGDNS
|
||||
//printf("status 101 for %s\n",back[i].url_adr);
|
||||
#endif
|
||||
|
||||
if (!gestion_timeout)
|
||||
@@ -2942,10 +2786,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// range size hack old location
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
// Shortcut: store the file directly on disk when possible,
|
||||
// sparing memory
|
||||
if (back[i].status &&
|
||||
!back[i].locked) { // name still pending when locked
|
||||
// Court-circuit:
|
||||
// Peut-on stocker le fichier directement sur disque?
|
||||
// Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci!
|
||||
if (back[i].status) {
|
||||
if (back[i].r.is_write == 0) { // mode mémoire
|
||||
if (back[i].r.adr == NULL) { // rien n'a été écrit
|
||||
if (!back[i].testmode) { // pas mode test
|
||||
@@ -3095,7 +2939,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// Si réception chunk, tester si on est pas à la fin!
|
||||
if (back[i].status == 1) {
|
||||
if (back[i].is_chunk) { // attendre prochain chunk
|
||||
if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
|
||||
if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
|
||||
//printf("chunk end at %d\n",back[i].r.size);
|
||||
back[i].status = STATUS_CHUNK_CR; /* fetch ending CRLF */
|
||||
if (back[i].chunk_adr != NULL) {
|
||||
freet(back[i].chunk_adr);
|
||||
@@ -3158,9 +3003,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
|
||||
if (back[i].r.totalsize >= 0) { // tester totalsize
|
||||
if (back[i].r.totalsize >= 0) { // tester totalsize
|
||||
//if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) { // tester totalsize
|
||||
if (back[i].r.totalsize != back[i].r.size) { // pas la même!
|
||||
if (!opt->tolerant) {
|
||||
//#if HTS_CL_IS_FATAL
|
||||
deleteaddr(&back[i].r);
|
||||
if (back[i].r.size < back[i].r.totalsize)
|
||||
back[i].r.statuscode = STATUSCODE_CONNERROR; // recatch
|
||||
@@ -3169,13 +3016,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
" expected)", (LLint) back[i].r.size,
|
||||
(LLint) back[i].r.totalsize);
|
||||
} else {
|
||||
//#else
|
||||
// Un warning suffira..
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Incorrect length (" LLintP "!=" LLintP
|
||||
" expected) for %s%s",
|
||||
(LLint) back[i].r.size,
|
||||
" expected) for %s%s", (LLint) back[i].r.size,
|
||||
(LLint) back[i].r.totalsize, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
//#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3302,8 +3150,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].r.size);
|
||||
#endif
|
||||
/* End */
|
||||
back[i].status = STATUS_READY; // fin
|
||||
//if (back[i].status==STATUS_CHUNK_CR) {
|
||||
back[i].status = STATUS_READY; // fin
|
||||
back_set_finished(sback, i);
|
||||
//}
|
||||
|
||||
// finalize transfer if not temporary
|
||||
if (!IS_DELAYED_EXT(back[i].url_sav)) {
|
||||
@@ -3770,6 +3620,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
|
||||
// finalize
|
||||
//file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified); // not modified
|
||||
if (back[i].r.statuscode > 0) {
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
@@ -3779,32 +3630,38 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
#endif
|
||||
|
||||
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
|
||||
} else { // erreur
|
||||
back[i].status = STATUS_READY; // terminé
|
||||
back_set_finished(sback, i);
|
||||
//printf("erreur cache\n");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
// MIME type excluded by a -mime: filter: abort, don't fetch
|
||||
// the body (#58)
|
||||
else if (HTTP_IS_OK(back[i].r.statuscode) &&
|
||||
!back[i].testmode &&
|
||||
strnotempty(back[i].r.contenttype) &&
|
||||
hts_acceptmime(opt, 0, back[i].url_adr,
|
||||
back[i].url_fil,
|
||||
back[i].r.contenttype) == 1) {
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].status = STATUS_READY;
|
||||
/********** NO - must complete the body! ********** */
|
||||
#if 0
|
||||
} else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
|
||||
|| (back[i].r.statuscode == 412)
|
||||
|| (back[i].r.statuscode == 416)
|
||||
) { // Ne pas prendre le html, erreurs connues et gérées
|
||||
#if HTS_DEBUG_CLOSESOCK
|
||||
DEBUG_W
|
||||
("back_wait(301,302,303,307,412,416..): deletehttp\n");
|
||||
#endif
|
||||
// Couper connexion
|
||||
/*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
|
||||
back_maydeletehttp(opt, cache, sback, i);
|
||||
|
||||
back[i].status = STATUS_READY; // terminé
|
||||
back_set_finished(sback, i);
|
||||
back[i].r.statuscode = STATUSCODE_EXCLUDED;
|
||||
strcpybuff(back[i].r.msg, "Excluded by MIME type filter");
|
||||
hts_log_print(
|
||||
opt, LOG_NOTICE,
|
||||
"File excluded by MIME type filter (%s): %s%s",
|
||||
back[i].r.contenttype, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
} else { // il faut aller le chercher
|
||||
// finalize
|
||||
if (back[i].r.statuscode > 0) {
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
#endif
|
||||
/********** **************************** ********** */
|
||||
} else { // il faut aller le chercher
|
||||
|
||||
// effacer buffer (requète)
|
||||
if (!noFreebuff) {
|
||||
@@ -3815,70 +3672,35 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// xxc SI CHUNK VERIFIER QUE CA MARCHE??
|
||||
if (back[i].r.statuscode == 206) { // on nous envoie un morceau (la fin) coz une partie sur disque!
|
||||
off_t sz = fsize_utf8(back[i].url_sav);
|
||||
/* RFC 7233: resume at the server's Content-Range start,
|
||||
not the offset we requested; a server may resume
|
||||
earlier and appending the overlap duplicates bytes
|
||||
(#198). */
|
||||
const LLint resume = back[i].r.crange_start;
|
||||
const hts_boolean range_ok =
|
||||
back[i].r.crange > 0 && resume >= 0 &&
|
||||
resume <= (LLint) sz &&
|
||||
back[i].r.crange_end + 1 == back[i].r.crange &&
|
||||
(back[i].r.totalsize < 0 ||
|
||||
back[i].r.totalsize ==
|
||||
back[i].r.crange_end - resume + 1);
|
||||
|
||||
#if HDEBUG
|
||||
printf("partial content: " LLintP " on disk..\n",
|
||||
(LLint) sz);
|
||||
#endif
|
||||
if (sz >= 0 && range_ok) {
|
||||
if (sz >= 0) {
|
||||
if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_sav)) { // pas HTML
|
||||
if (opt->getmode & HTS_GETMODE_NONHTML) {
|
||||
filenote(&opt->state.strc, back[i].url_sav, NULL); // noter fichier comme connu
|
||||
file_notify(opt, back[i].url_adr, back[i].url_fil,
|
||||
back[i].url_sav, 0, 1,
|
||||
back[i].r.notmodified);
|
||||
back[i].r.out =
|
||||
FOPEN(fconv(catbuff, sizeof(catbuff),
|
||||
back[i].url_sav),
|
||||
"r+b"); // resume in place
|
||||
back[i].r.out = FOPEN(fconv(catbuff, sizeof(catbuff), back[i].url_sav), "ab"); // append
|
||||
if (back[i].r.out && opt->cache != 0) {
|
||||
back[i].r.is_write = 1;
|
||||
back[i].r.size = resume; // bytes already on disk
|
||||
back[i].r.statuscode = HTTP_OK; // force 'OK'
|
||||
back[i].r.is_write = 1; // écrire
|
||||
back[i].r.size = sz; // déja écrit
|
||||
back[i].r.statuscode = HTTP_OK; // Forcer 'OK'
|
||||
if (back[i].r.totalsize >= 0)
|
||||
back[i].r.totalsize += resume; // -> full size
|
||||
// drop bytes past the resume point; a silent
|
||||
// failure could leave a stale tail, so on error
|
||||
// drop the partial and refetch the whole file
|
||||
if (HTS_FTRUNCATE(back[i].r.out,
|
||||
(off_t) resume) != 0) {
|
||||
fclose(back[i].r.out);
|
||||
back[i].r.out = NULL;
|
||||
url_savename_refname_remove(
|
||||
opt, back[i].url_adr, back[i].url_fil);
|
||||
UNLINK(back[i].url_sav);
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
strcpybuff(back[i].r.msg,
|
||||
"Can not truncate partial file, "
|
||||
"restarting");
|
||||
} else {
|
||||
fseeko(back[i].r.out, (off_t) resume, SEEK_SET);
|
||||
/* create a temporary reference file in case of
|
||||
* broken mirror */
|
||||
if (back_serialize_ref(opt, &back[i]) != 0) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Could not create temporary "
|
||||
"reference file for %s%s",
|
||||
back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
}
|
||||
#if HDEBUG
|
||||
printf("continue interrupted file\n");
|
||||
#endif
|
||||
back[i].r.totalsize += sz; // plus en fait
|
||||
fseek(back[i].r.out, 0, SEEK_END); // à la fin
|
||||
/* create a temporary reference file in case of broken mirror */
|
||||
if (back_serialize_ref(opt, &back[i]) != 0) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Could not create temporary reference file for %s%s",
|
||||
back[i].url_adr, back[i].url_fil);
|
||||
}
|
||||
#if HDEBUG
|
||||
printf("continue interrupted file\n");
|
||||
#endif
|
||||
} else { // On est dans la m**
|
||||
back[i].status = STATUS_READY; // terminé (voir plus loin)
|
||||
back_set_finished(sback, i);
|
||||
@@ -3890,18 +3712,17 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
FILE *fp =
|
||||
FOPEN(fconv(catbuff, sizeof(catbuff), back[i].url_sav), "rb");
|
||||
if (fp) {
|
||||
LLint alloc_mem = resume + 1;
|
||||
LLint alloc_mem = sz + 1;
|
||||
|
||||
if (back[i].r.totalsize >= 0)
|
||||
alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
|
||||
if (deleteaddr(&back[i].r)
|
||||
&& (back[i].r.adr =
|
||||
(char *) malloct((size_t) alloc_mem))) {
|
||||
back[i].r.size = resume;
|
||||
back[i].r.size = sz;
|
||||
if (back[i].r.totalsize >= 0)
|
||||
back[i].r.totalsize += resume; // -> full size
|
||||
if ((fread(back[i].r.adr, 1, (size_t) resume,
|
||||
fp)) != (size_t) resume) {
|
||||
back[i].r.totalsize += sz; // plus en fait
|
||||
if ((fread(back[i].r.adr, 1, sz, fp)) != sz) {
|
||||
back[i].status = STATUS_READY; // terminé (voir plus loin)
|
||||
back_set_finished(sback, i);
|
||||
strcpybuff(back[i].r.msg,
|
||||
@@ -3919,30 +3740,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
"No memory for partial file");
|
||||
}
|
||||
fclose(fp);
|
||||
} else { // open failed
|
||||
} else { // Argh..
|
||||
back[i].status = STATUS_READY; // terminé (voir plus loin)
|
||||
back_set_finished(sback, i);
|
||||
strcpybuff(back[i].r.msg,
|
||||
"Can not open partial file");
|
||||
}
|
||||
}
|
||||
} else if (sz >=
|
||||
0) { // unusable range -> restart whole file
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"Unusable partial-content range for %s%s "
|
||||
"(have " LLintP " bytes, got " LLintP
|
||||
"-" LLintP "/" LLintP "), restarting",
|
||||
back[i].url_adr, back[i].url_fil,
|
||||
(LLint) sz, back[i].r.crange_start,
|
||||
back[i].r.crange_end, back[i].r.crange);
|
||||
url_savename_refname_remove(opt, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
UNLINK(back[i].url_sav);
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
strcpybuff(back[i].r.msg,
|
||||
"Unusable partial content, restarting");
|
||||
} else { // partial not found
|
||||
} else { // Non trouvé??
|
||||
back[i].status = STATUS_READY; // terminé (voir plus loin)
|
||||
back_set_finished(sback, i);
|
||||
strcpybuff(back[i].r.msg, "Can not find partial file");
|
||||
@@ -3957,6 +3762,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
deletehttp(&back[i].r);
|
||||
}
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
//back[i].r.statuscode=206; ????????
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
if (strnotempty(back[i].r.msg))
|
||||
strcpybuff(back[i].r.msg,
|
||||
@@ -3977,13 +3783,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
&& (back[i].r.adr = (char *) malloct(2))) {
|
||||
back[i].r.adr[0] = 0;
|
||||
}
|
||||
/* locked = name pending; the waiter finalizes after
|
||||
patching url_sav (else: cached as .delayed, #5) */
|
||||
if (!back[i].locked) {
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing empty");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
}
|
||||
} else if (!back[i].r.is_chunk) { // pas de chunk
|
||||
hts_log_print(opt, LOG_TRACE, "finalizing empty");
|
||||
back_finalize(opt, cache, sback, i);
|
||||
} else if (!back[i].r.is_chunk) { // pas de chunk
|
||||
//if (back[i].r.http11!=2) { // pas de chunk
|
||||
back[i].is_chunk = 0;
|
||||
back[i].status = 1; // start body
|
||||
} else {
|
||||
@@ -4025,6 +3828,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/*} */
|
||||
@@ -4044,11 +3848,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].status < 0) {
|
||||
if (!back[i].testmode) { // pas en test
|
||||
UNLINK(back[i].url_sav); // éliminer fichier (endommagé)
|
||||
//printf("&& %s\n",back[i].url_sav);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
/* funny log for commandline users */
|
||||
//if (!opt->quiet) {
|
||||
// petite animation
|
||||
if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
|
||||
if (back[i].status == STATUS_READY) {
|
||||
if (back[i].r.statuscode == HTTP_OK)
|
||||
@@ -4061,43 +3868,23 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
//}
|
||||
|
||||
} // status>0
|
||||
} // for
|
||||
} // for
|
||||
|
||||
// vérifier timeouts
|
||||
if (gestion_timeout) {
|
||||
TStamp act;
|
||||
|
||||
act = time_local(); // temps en secondes
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
// for(i=0;i<back_max;i++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
if (back[i].status > 0) { // réception/connexion/..
|
||||
if (back[i].timeout > 0) {
|
||||
// a stuck connect with a fallback address: retry the next one well
|
||||
// before the full timeout (dead IPv6 on a dual-stack host, ...)
|
||||
if (back[i].status == STATUS_CONNECTING) {
|
||||
const hts_connect_fallback *const cf =
|
||||
&sback->connect_fallback[i];
|
||||
|
||||
if (back_connect_fallback_due(cf->addr_index, cf->addr_count,
|
||||
(int) (act - cf->connect_start),
|
||||
back[i].timeout)) {
|
||||
if (back_connect_next(opt, sback, i)) {
|
||||
continue; // reconnected to the next candidate
|
||||
}
|
||||
// fallback was due but no socket could be opened
|
||||
// (back_connect_next closed the dead one): stop now rather than
|
||||
// spin on an invalid fd
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_CONNERROR;
|
||||
strcpybuff(back[i].r.msg, "Connect Error");
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
//printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
|
||||
if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
|
||||
hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
@@ -4173,11 +3960,6 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Grace left to the smooth stop before in-flight transfers are aborted. */
|
||||
static int back_maxtime_grace(const int maxtime) {
|
||||
return maximum(5, minimum(30, maxtime / 10));
|
||||
}
|
||||
|
||||
int back_checkmirror(httrackp * opt) {
|
||||
// Check max size
|
||||
if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
|
||||
@@ -4194,19 +3976,13 @@ int back_checkmirror(httrackp * opt) {
|
||||
*/
|
||||
}
|
||||
// Check max time
|
||||
if (opt->maxtime > 0) {
|
||||
const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
|
||||
|
||||
if (elapsed >= opt->maxtime) {
|
||||
if (!opt->state.stop) { /* not yet stopped */
|
||||
hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
|
||||
opt->maxtime);
|
||||
/* cancel mirror smoothly */
|
||||
hts_request_stop(opt, 0);
|
||||
}
|
||||
/* smooth stop starved past the grace period: stop waiting (#481) */
|
||||
if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
|
||||
return 0;
|
||||
if ((opt->maxtime > 0)
|
||||
&& ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
|
||||
if (!opt->state.stop) { /* not yet stopped */
|
||||
hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
|
||||
opt->maxtime);
|
||||
/* cancel mirror smoothly */
|
||||
hts_request_stop(opt, 0);
|
||||
}
|
||||
}
|
||||
return 1; /* Ok, go on */
|
||||
|
||||
@@ -113,12 +113,6 @@ void back_set_locked(struct_back * sback, const int p);
|
||||
void back_set_unlocked(struct_back * sback, const int p);
|
||||
int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
const int p);
|
||||
/* Discard back's on-disk .delayed placeholder and its refname. */
|
||||
void back_delayed_discard(httrackp *opt, lien_back *back);
|
||||
/* Move back's .delayed placeholder (and open stream) to newname;
|
||||
HTS_FALSE = file lost, slot flagged in error. */
|
||||
hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
|
||||
const char *newname);
|
||||
void back_index_unlock(struct_back * sback, const int p);
|
||||
int back_clear_entry(lien_back * back);
|
||||
int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
@@ -142,8 +136,6 @@ void back_solve(httrackp * opt, lien_back * sback);
|
||||
int host_wait(httrackp * opt, lien_back * sback);
|
||||
#endif
|
||||
int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
|
||||
/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
|
||||
the -E deadline overran its grace period (callers must stop waiting). */
|
||||
int back_checkmirror(httrackp * opt);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -55,6 +55,8 @@ Please visit our Website: http://www.httrack.com
|
||||
// inline function in Wspiapi.h.
|
||||
#include <ws2tcpip.h>
|
||||
#include <Wspiapi.h>
|
||||
// #include <winsock2.h>
|
||||
// #include <tpipv6.h>
|
||||
|
||||
#endif
|
||||
|
||||
@@ -127,8 +129,6 @@ typedef enum HTTPStatusCode {
|
||||
HTTP_UNSUPPORTED_MEDIA_TYPE = 415,
|
||||
HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416,
|
||||
HTTP_EXPECTATION_FAILED = 417,
|
||||
HTTP_TOO_MANY_REQUESTS = 429,
|
||||
HTTP_UNAVAILABLE_FOR_LEGAL_REASONS = 451,
|
||||
HTTP_INTERNAL_SERVER_ERROR = 500,
|
||||
HTTP_NOT_IMPLEMENTED = 501,
|
||||
HTTP_BAD_GATEWAY = 502,
|
||||
@@ -146,8 +146,7 @@ typedef enum BackStatusCode {
|
||||
STATUSCODE_NON_FATAL = -5,
|
||||
STATUSCODE_SSL_HANDSHAKE = -6,
|
||||
STATUSCODE_TOO_BIG = -7,
|
||||
STATUSCODE_TEST_OK = -10,
|
||||
STATUSCODE_EXCLUDED = -11 /* aborted: MIME excluded by a -mime: filter */
|
||||
STATUSCODE_TEST_OK = -10
|
||||
} BackStatusCode;
|
||||
|
||||
/** HTTrack status ('status' member of of 'lien_back') **/
|
||||
|
||||
@@ -3,12 +3,12 @@
|
||||
|
||||
# Change this to download files
|
||||
if false; then
|
||||
echo "mget https://www.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
|
||||
echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
|
||||
echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
|
||||
echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
|
||||
echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
|
||||
echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
|
||||
echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
|
||||
rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
|
||||
fi
|
||||
|
||||
|
||||
@@ -107,6 +107,7 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
|
||||
#if DEBUG_COOK
|
||||
printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
|
||||
cook_name, cook_value, domain, path);
|
||||
//printf(">>>cook: %s<<<\n",cookie->data);
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@@ -207,6 +208,8 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
char buffer[8192];
|
||||
|
||||
// cookie->data[0]='\0';
|
||||
|
||||
// Fusionner d'abord les éventuels cookies IE
|
||||
#ifdef _WIN32
|
||||
{
|
||||
|
||||
268
src/htscache.c
268
src/htscache.c
@@ -40,7 +40,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htscore.h"
|
||||
#include "htsbasenet.h"
|
||||
#include "htsmd5.h"
|
||||
#include <limits.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "htszlib.h"
|
||||
@@ -221,40 +220,6 @@ struct cache_back_zip_entry {
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* Consecutive entry write failures before the cache stream is declared dead. */
|
||||
#define CACHE_MAX_WRITE_FAILURES 8
|
||||
|
||||
/* Cache write failed: a fatal errno or a failure streak aborts the mirror
|
||||
(exit_xh); an isolated failure only drops the current entry. */
|
||||
static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
|
||||
const char *what, int zErr,
|
||||
hts_boolean entry_open, const char *url_adr,
|
||||
const char *url_fil) {
|
||||
const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
|
||||
|
||||
cache->zipWriteFailures++;
|
||||
if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
|
||||
if (!cache->zipWriteFailed) {
|
||||
cache->zipWriteFailed = HTS_TRUE;
|
||||
if (fatal_errno) {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: disk full or filesystem problems");
|
||||
} else {
|
||||
hts_log_print(opt, LOG_ERROR,
|
||||
"Mirror aborted: cache write failed (%s): %s", what,
|
||||
hts_get_zerror(zErr));
|
||||
}
|
||||
}
|
||||
opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
|
||||
} else {
|
||||
if (entry_open)
|
||||
zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"cache write failed (%s: %s), entry not cached: %s%s", what,
|
||||
hts_get_zerror(zErr), url_adr, url_fil);
|
||||
}
|
||||
}
|
||||
|
||||
/* Ajout d'un fichier en cache */
|
||||
void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
const char *url_adr, const char *url_fil, const char *url_save,
|
||||
@@ -265,14 +230,12 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
char BIGSTK headers[8192];
|
||||
int headersSize = 0;
|
||||
|
||||
//int entryBodySize = 0;
|
||||
//int entryFilenameSize = 0;
|
||||
zip_fileinfo fi;
|
||||
const char *url_save_suffix = url_save;
|
||||
int zErr;
|
||||
|
||||
/* already failed and aborting; don't touch the broken stream again */
|
||||
if (cache->zipWriteFailed)
|
||||
return;
|
||||
|
||||
// robots.txt hack
|
||||
if (url_save == NULL) {
|
||||
dataincache = 0; // testing links
|
||||
@@ -300,19 +263,10 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if (r->size < 0) // error
|
||||
return;
|
||||
|
||||
// data in cache: the body must fit the 32-bit zip write API
|
||||
if (dataincache && (LLint) (int) r->size != r->size) {
|
||||
if (r->is_write && url_save != NULL && strnotempty(url_save)) {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"file too large for the cache, storing headers only: %s%s",
|
||||
url_adr, url_fil);
|
||||
dataincache = 0;
|
||||
} else {
|
||||
hts_log_print(opt, LOG_WARNING,
|
||||
"entry too large for the cache, not cached: %s%s", url_adr,
|
||||
url_fil);
|
||||
return;
|
||||
}
|
||||
// data in cache
|
||||
if (dataincache) {
|
||||
assertf(((int) r->size) == r->size);
|
||||
//entryBodySize = (int) r->size;
|
||||
}
|
||||
|
||||
/* Fields */
|
||||
@@ -357,6 +311,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil); // Original URI filename
|
||||
ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix); // Original save filename
|
||||
|
||||
//entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
|
||||
|
||||
/* Filename */
|
||||
if (!link_has_authority(url_adr)) {
|
||||
strcpybuff(filename, "http://");
|
||||
@@ -390,9 +346,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
*/
|
||||
headers, (uInt) strlen(headers), NULL, 0, NULL, /* comment */
|
||||
Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
return;
|
||||
int zip_zipOpenNewFileInZip_failed = 0;
|
||||
|
||||
assertf(zip_zipOpenNewFileInZip_failed);
|
||||
}
|
||||
|
||||
/* Write data in cache */
|
||||
@@ -402,9 +358,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if ((zErr =
|
||||
zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
|
||||
(int) r->size)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
|
||||
HTS_TRUE, url_adr, url_fil);
|
||||
return;
|
||||
int zip_zipWriteInFileInZip_failed = 0;
|
||||
|
||||
assertf(zip_zipWriteInFileInZip_failed);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -425,10 +381,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
if ((zErr =
|
||||
zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
|
||||
(int) nl)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
|
||||
HTS_TRUE, url_adr, url_fil);
|
||||
fclose(fp);
|
||||
return;
|
||||
int zip_zipWriteInFileInZip_failed = 0;
|
||||
|
||||
assertf(zip_zipWriteInFileInZip_failed);
|
||||
}
|
||||
}
|
||||
} while(nl > 0);
|
||||
@@ -442,19 +397,17 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
|
||||
/* Close */
|
||||
if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
|
||||
cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
return;
|
||||
int zip_zipCloseFileInZip_failed = 0;
|
||||
|
||||
assertf(zip_zipCloseFileInZip_failed);
|
||||
}
|
||||
|
||||
/* Flush */
|
||||
if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
|
||||
cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
|
||||
url_adr, url_fil);
|
||||
return;
|
||||
}
|
||||
int zip_zipFlush_failed = 0;
|
||||
|
||||
cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
|
||||
assertf(zip_zipFlush_failed);
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
@@ -623,18 +576,15 @@ htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
|
||||
return cache_readex(opt, cache, adr, fil, save, location, NULL, 1);
|
||||
}
|
||||
|
||||
htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
|
||||
const char *adr, const char *fil,
|
||||
char *return_save) {
|
||||
htsblk r = cache_readex(opt, cache, adr, fil, NULL, NULL, return_save, 0);
|
||||
htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
|
||||
const char *adr, const char *fil) {
|
||||
htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL);
|
||||
|
||||
if (r.statuscode == -1) {
|
||||
lien_back *itemback = NULL;
|
||||
|
||||
if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
||||
r = itemback->r;
|
||||
if (return_save != NULL)
|
||||
strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2);
|
||||
/* cleanup */
|
||||
back_clear_entry(itemback); /* delete entry content */
|
||||
freet(itemback); /* delete item */
|
||||
@@ -721,6 +671,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
char BIGSTK headerBuff[8192 + 2];
|
||||
int readSizeHeader;
|
||||
|
||||
//int totalHeader = 0;
|
||||
int dataincache = 0;
|
||||
|
||||
/* For BIG comments */
|
||||
@@ -771,10 +722,13 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
HTS_URLMAXSIZE * 2);
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
|
||||
sizeof(r.cdispo));
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
|
||||
sizeof(previous_save_));
|
||||
}
|
||||
} while (offset < readSizeHeader && !lineEof);
|
||||
} while(offset < readSizeHeader && !lineEof);
|
||||
//totalHeader = offset;
|
||||
|
||||
/* Previous entry */
|
||||
if (previous_save_[0] != '\0') {
|
||||
@@ -791,15 +745,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
|
||||
}
|
||||
|
||||
/* A negative X-Size is corrupt; so is one >= INT_MAX when the data
|
||||
is in the zip (the write path asserts int-sized). Headers-only
|
||||
entries legitimately exceed INT_MAX (>2GB body on disk): keep
|
||||
them, or every update would re-fetch the file. */
|
||||
if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Read Error : Bad Size");
|
||||
}
|
||||
|
||||
/* Complete fields */
|
||||
r.totalsize = r.size;
|
||||
r.adr = NULL;
|
||||
@@ -826,8 +771,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
} // otherwise, the ZIP file is supposed to be consistent with data.
|
||||
}
|
||||
/* Read data ? */
|
||||
else if (r.statuscode !=
|
||||
STATUSCODE_INVALID) { /* ne pas lire uniquement header */
|
||||
else { /* ne pas lire uniquement header */
|
||||
int ok = 0;
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
@@ -956,6 +900,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg,
|
||||
"Cache Write Error : Unable to Create File");
|
||||
//printf("%s\n",save);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -990,10 +935,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg,
|
||||
"Previous cache file not found (empty filename)");
|
||||
}
|
||||
} else if (r.size >= INT_MAX) { /* too big to read in memory */
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Read Error : Bad Size");
|
||||
} else { /* Read in memory from disk */
|
||||
} else { /* Read in memory from disk */
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
@@ -1033,6 +975,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r.adr + r.size) = '\0';
|
||||
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
|
||||
} else { // erreur
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Memory Error");
|
||||
@@ -1211,14 +1154,17 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
// sécurité
|
||||
r.adr = NULL;
|
||||
r.out = NULL;
|
||||
////r.location=NULL; non, fixée lors des 301 ou 302
|
||||
r.fp = NULL;
|
||||
|
||||
if ((r.statuscode >= 0) && (r.statuscode <= 999)
|
||||
&& (r.notmodified >= 0) && (r.notmodified <= 9)) { // petite vérif intégrité
|
||||
if ((save) && (!header_only)) { /* ne pas lire uniquement header */
|
||||
if ((save) && (!header_only)) { /* ne pas lire uniquement header */
|
||||
//int to_file=0;
|
||||
|
||||
r.adr = NULL;
|
||||
r.soc = INVALID_SOCKET;
|
||||
// // r.location=NULL;
|
||||
|
||||
#if HTS_DIRECTDISK
|
||||
// Court-circuit:
|
||||
@@ -1227,11 +1173,12 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
int ok = 0;
|
||||
|
||||
r.is_write = 1; // écrire
|
||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
|
||||
save))) { // un fichier existe déja
|
||||
if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) { // un fichier existe déja
|
||||
//if (fsize_utf8(fconv(save))==r.size) { // même taille -- NON tant pis (taille mal declaree)
|
||||
ok = 1; // plus rien à faire
|
||||
filenote(&opt->state.strc, save, NULL); // noter comme connu
|
||||
file_notify(opt, adr, fil, save, 0, 0, 0);
|
||||
//}
|
||||
}
|
||||
|
||||
if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
|
||||
@@ -1282,6 +1229,7 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg,
|
||||
"Cache Write Error : Unable to Create File");
|
||||
//printf("%s\n",save);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1329,13 +1277,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
|
||||
strcpybuff(r.msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r.adr + r.size) = '\0';
|
||||
//printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
|
||||
} else { // erreur
|
||||
r.statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r.msg, "Cache Memory Error");
|
||||
}
|
||||
}
|
||||
}
|
||||
} // si save==null, ne rien charger (juste en tête)
|
||||
} // si save==null, ne rien charger (juste en tête)
|
||||
} else {
|
||||
#if DEBUGCA
|
||||
printf("Cache Read Error : Bad Data");
|
||||
@@ -1448,86 +1397,6 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
|
||||
return rename(a, b);
|
||||
}
|
||||
|
||||
/* Pathname of a file inside the mirror dir (rotating concat buffer). */
|
||||
static char *reconcile_path(httrackp *opt, const char *name) {
|
||||
return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), name);
|
||||
}
|
||||
|
||||
/* Interrupted-run heuristic: prefer the old generation when the new cache
|
||||
stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
|
||||
arbitrary thresholds). */
|
||||
#define CACHE_RECONCILE_NEW_TINY 32768
|
||||
#define CACHE_RECONCILE_OLD_SOLID 65536
|
||||
|
||||
/* Replace the new-generation file by the old one, when the old one exists. */
|
||||
static void reconcile_promote(httrackp *opt, const char *oldname,
|
||||
const char *newname) {
|
||||
if (fexist(reconcile_path(opt, oldname))) {
|
||||
remove(reconcile_path(opt, newname));
|
||||
rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
|
||||
}
|
||||
}
|
||||
|
||||
void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
|
||||
switch (mode) {
|
||||
case CACHE_RECONCILE_PROMOTE:
|
||||
/* Previous run rotated new.* to old.* then died before writing: promote
|
||||
the old generation back, whichever format it uses. */
|
||||
if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
|
||||
!fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
break;
|
||||
case CACHE_RECONCILE_INTERRUPTED:
|
||||
/* Aborted run: keep the larger generation when the new cache is
|
||||
suspiciously small next to the old one. The new file must exist: fsize()
|
||||
is -1 for a missing file, which would spuriously pass the "< TINY" test
|
||||
and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
|
||||
*/
|
||||
if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
|
||||
break;
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.zip")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.zip")))
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
|
||||
fsize(reconcile_path(opt, "hts-cache/new.dat")) <
|
||||
CACHE_RECONCILE_NEW_TINY &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.dat")) >
|
||||
CACHE_RECONCILE_OLD_SOLID &&
|
||||
fsize(reconcile_path(opt, "hts-cache/old.dat")) >
|
||||
fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
break;
|
||||
case CACHE_RECONCILE_ROLLBACK:
|
||||
/* Nothing transferred: restore the previous generation and sidecars. */
|
||||
reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
|
||||
if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
|
||||
fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
|
||||
reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
|
||||
reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
|
||||
}
|
||||
reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
|
||||
reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// renvoyer uniquement en tête, ou NULL si erreur
|
||||
// return NULL upon error, and set -1 to r.statuscode
|
||||
htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
|
||||
@@ -1558,10 +1427,25 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
|
||||
HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
if ((fexist(fconcat(
|
||||
OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.zip")))) { // a previous cache exists.. rename it
|
||||
if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) { // il existe déja un cache précédent.. renommer
|
||||
/* Previous cache from the previous cache version */
|
||||
#if 0
|
||||
/* No.. reuse with old httrack releases! */
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.dat"));
|
||||
if (fexist
|
||||
(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx")))
|
||||
remove(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"));
|
||||
#endif
|
||||
/* Previous cache version */
|
||||
if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
|
||||
rename(fconcat
|
||||
@@ -1602,13 +1486,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
} else {
|
||||
hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
|
||||
}
|
||||
} else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.dat"))) &&
|
||||
(fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/new.ndx")))) { // a previous cache
|
||||
// exists.. rename it
|
||||
} else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) { // il existe déja un cache précédent.. renommer
|
||||
#if DEBUGCA
|
||||
printf("work with former cache\n");
|
||||
#endif
|
||||
@@ -1637,7 +1515,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log),
|
||||
"hts-cache/old.ndx"));
|
||||
} else { // one or both cache files missing: remove the remaining one
|
||||
} else { // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
|
||||
#if DEBUGCA
|
||||
printf("new cache\n");
|
||||
#endif
|
||||
@@ -2006,6 +1884,12 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"hts-cache/new.lst"), "wb");
|
||||
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
|
||||
opt->state.strc.lst = cache->lst;
|
||||
//{
|
||||
//filecreate_params tmp;
|
||||
//strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin
|
||||
//tmp.lst=cache->lst; // fichier lst
|
||||
//filenote("",&tmp); // initialiser filecreate
|
||||
//}
|
||||
|
||||
// supprimer old.txt
|
||||
if (fexist
|
||||
@@ -2094,6 +1978,12 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"hts-cache/new.lst"), "wb");
|
||||
strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
|
||||
opt->state.strc.lst = cache->lst;
|
||||
//{
|
||||
// filecreate_params tmp;
|
||||
// strcpybuff(tmp.path,StringBuff(opt->path_html)); // chemin
|
||||
// tmp.lst=cache->lst; // fichier lst
|
||||
// filenote("",&tmp); // initialiser filecreate
|
||||
//}
|
||||
|
||||
// supprimer old.txt
|
||||
if (fexist
|
||||
@@ -2125,6 +2015,8 @@ void cache_init(cache_back * cache, httrackp * opt) {
|
||||
"statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
|
||||
LF);
|
||||
}
|
||||
// test
|
||||
// cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
|
||||
} // cache->ndx!=NULL
|
||||
} //cache->zipOutput != NULL
|
||||
|
||||
|
||||
@@ -66,11 +66,8 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, char *location);
|
||||
htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, char *location);
|
||||
/* Like cache_read, but also yields entries whose transfer broke; return_save
|
||||
(optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
|
||||
htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
|
||||
const char *adr, const char *fil,
|
||||
char *return_save);
|
||||
htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
|
||||
const char *adr, const char *fil);
|
||||
htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, const char *save, char *location,
|
||||
char *return_save, int readonly);
|
||||
@@ -78,17 +75,6 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
|
||||
const char *fil, htsblk * r);
|
||||
void cache_init(cache_back * cache, httrackp * opt);
|
||||
|
||||
/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
|
||||
typedef enum {
|
||||
CACHE_RECONCILE_PROMOTE, /* no new cache: promote the old generation */
|
||||
CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
|
||||
CACHE_RECONCILE_ROLLBACK /* nothing transferred: restore the old one */
|
||||
} hts_cache_reconcile_mode;
|
||||
|
||||
/* Reconcile the on-disk cache generations according to mode; a no-op when
|
||||
the involved files are absent. */
|
||||
void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
|
||||
|
||||
int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
|
||||
const char *str2, char *outbuff, int len);
|
||||
int cache_readdata(cache_back * cache, const char *str1, const char *str2,
|
||||
|
||||
@@ -47,8 +47,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htslib.h"
|
||||
#include "htszlib.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -318,328 +316,6 @@ static int disk_fallback_selftest(httrackp *opt) {
|
||||
return fail;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
size_t budget; /**< bytes allowed through before writes start failing */
|
||||
int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
|
||||
int writes; /**< zwrite call count, to detect re-entry into the stream */
|
||||
int fail_once; /**< recover (unlimited budget) after the first failure */
|
||||
} writefail_inject;
|
||||
|
||||
/* zwrite that copies until the budget runs out, then fails with inj->fail_errno
|
||||
(the #174/#219 condition). Counts calls so the test can prove a flagged cache
|
||||
never re-enters the stream. */
|
||||
static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
|
||||
const void *buf, uLong size) {
|
||||
writefail_inject *inj = (writefail_inject *) opaque;
|
||||
|
||||
inj->writes++;
|
||||
if (inj->budget >= (size_t) size) {
|
||||
inj->budget -= (size_t) size;
|
||||
return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
|
||||
}
|
||||
if (inj->fail_once)
|
||||
inj->budget = (size_t) -1; /* the backend recovers after this failure */
|
||||
errno = inj->fail_errno;
|
||||
return 0; /* short write -> the minizip op returns an error */
|
||||
}
|
||||
|
||||
/* Open a ZIP whose writes fail past inj->budget, so cache_add() hits an error.
|
||||
*/
|
||||
static zipFile selftest_open_failing_zip(const char *path,
|
||||
writefail_inject *inj) {
|
||||
zlib_filefunc_def ff;
|
||||
|
||||
fill_fopen_filefunc(&ff); /* real fopen/read/seek/close; ignores opaque */
|
||||
ff.zwrite_file = selftest_failing_zwrite;
|
||||
ff.opaque = inj;
|
||||
return zipOpen2(path, APPEND_STATUS_CREATE, NULL, &ff);
|
||||
}
|
||||
|
||||
/* Store one octet-stream body into `cache` (all-in-cache, body in the ZIP). */
|
||||
static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
|
||||
const char *body, size_t body_len) {
|
||||
htsblk r;
|
||||
char locbuf[4];
|
||||
char *bodycopy = malloct(body_len);
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
r.statuscode = 200;
|
||||
r.size = (LLint) body_len;
|
||||
strcpybuff(r.msg, "OK");
|
||||
strcpybuff(r.contenttype, "application/octet-stream");
|
||||
locbuf[0] = '\0';
|
||||
r.location = locbuf;
|
||||
r.is_write = 0;
|
||||
memcpy(bodycopy, body, body_len);
|
||||
r.adr = bodycopy;
|
||||
cache_add(opt, cache, &r, "example.com", fil, "example.com/blob.bin", 1,
|
||||
NULL);
|
||||
freet(bodycopy);
|
||||
}
|
||||
|
||||
/* Store an entry claiming a >2GB body; the degrade path never reads data. */
|
||||
static void writefail_store_oversized(httrackp *opt, cache_back *cache,
|
||||
const char *fil, int is_write) {
|
||||
htsblk r;
|
||||
char locbuf[4];
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
r.statuscode = 200;
|
||||
r.size = (LLint) INT_MAX + 1;
|
||||
strcpybuff(r.msg, "OK");
|
||||
strcpybuff(r.contenttype, "application/octet-stream");
|
||||
locbuf[0] = '\0';
|
||||
r.location = locbuf;
|
||||
r.is_write = (short int) is_write;
|
||||
cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
|
||||
}
|
||||
|
||||
/* Read back `entryname`: extra field (cached headers) and body. Returns the
|
||||
body length, or -1 if the entry is absent or unreadable. */
|
||||
static int writefail_read_entry(const char *path, const char *entryname,
|
||||
char *extra, size_t extralen, char *body,
|
||||
size_t bodylen) {
|
||||
unzFile z = unzOpen(path);
|
||||
int n = -1;
|
||||
|
||||
if (z == NULL)
|
||||
return -1;
|
||||
if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
|
||||
unzOpenCurrentFile(z) == UNZ_OK) {
|
||||
const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
|
||||
|
||||
if (elen >= 0) {
|
||||
extra[elen] = '\0';
|
||||
n = unzReadCurrentFile(z, body, (unsigned) bodylen);
|
||||
}
|
||||
unzCloseCurrentFile(z);
|
||||
}
|
||||
unzClose(z);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
|
||||
stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
|
||||
*/
|
||||
int cache_write_failure_selftest(httrackp *opt, const char *dir) {
|
||||
int fail = 0;
|
||||
char path[HTS_URLMAXSIZE];
|
||||
/* incompressible + big, so deflate flushes (and fails) mid-write, before
|
||||
* close */
|
||||
static const size_t body_len = 256 * 1024;
|
||||
char *body = malloct(body_len);
|
||||
int phase;
|
||||
|
||||
gen_body(body, body_len, 1 /* incompressible */);
|
||||
fconcat(path, sizeof(path), dir, "/wfail.zip");
|
||||
|
||||
/* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
|
||||
drops entries until the streak caps out, then aborts. */
|
||||
for (phase = 0; phase < 2; phase++) {
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
int writes_after_fail;
|
||||
|
||||
inj.budget = (phase == 0) ? 4096 : 0;
|
||||
inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
if (cache.zipOutput == NULL) {
|
||||
fprintf(stderr, "cache-writefail: could not open injected ZIP\n");
|
||||
fail++;
|
||||
continue;
|
||||
}
|
||||
|
||||
opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
|
||||
if (phase == 0) {
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
} else {
|
||||
/* the abort must land exactly on the 8th consecutive failure */
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 7; i++) {
|
||||
char fil[32];
|
||||
|
||||
snprintf(fil, sizeof(fil), "/b%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
}
|
||||
if (cache.zipWriteFailed) {
|
||||
fprintf(stderr, "cache-writefail: phase 1: aborted before the "
|
||||
"8th consecutive failure\n");
|
||||
fail++;
|
||||
}
|
||||
writefail_store(opt, &cache, "/b7.bin", body, 16);
|
||||
}
|
||||
if (!cache.zipWriteFailed) {
|
||||
fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
|
||||
phase);
|
||||
fail++;
|
||||
}
|
||||
if (opt->state.exit_xh != -1) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: phase %d: mirror not aborted (exit_xh=%d)\n",
|
||||
phase, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
|
||||
/* a flagged cache must no-op a sibling write: no further backend write */
|
||||
writes_after_fail = inj.writes;
|
||||
writefail_store(opt, &cache, "/blob2.bin", body, 16);
|
||||
if (inj.writes != writes_after_fail) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: phase %d: sibling write re-entered the broken "
|
||||
"stream (%d extra backend writes)\n",
|
||||
phase, inj.writes - writes_after_fail);
|
||||
fail++;
|
||||
}
|
||||
|
||||
if (cache.zipOutput != NULL) {
|
||||
zipClose(cache.zipOutput,
|
||||
NULL); /* best-effort; may fail on the backend */
|
||||
cache.zipOutput = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* failures with successes in between reset the streak: never aborts */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
int i;
|
||||
|
||||
inj.budget = (size_t) -1;
|
||||
inj.fail_errno = EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
for (i = 0; i < 10; i++) {
|
||||
char fil[32];
|
||||
|
||||
inj.budget = 0; /* this store fails */
|
||||
snprintf(fil, sizeof(fil), "/s%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
|
||||
snprintf(fil, sizeof(fil), "/ok%d.bin", i);
|
||||
writefail_store(opt, &cache, fil, body, 16);
|
||||
}
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: scattered: non-consecutive failures aborted "
|
||||
"the mirror (flagged=%d, exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
}
|
||||
|
||||
/* isolated failure: only that entry drops; a later sibling round-trips */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
char extra[8192];
|
||||
char rbody[64];
|
||||
int n;
|
||||
|
||||
inj.budget = 4096;
|
||||
inj.fail_errno = EIO;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 1;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
writefail_store(opt, &cache, "/blob.bin", body, body_len);
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: skip: isolated failure aborted the mirror "
|
||||
"(flagged=%d, exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
writefail_store(opt, &cache, "/blob2.bin", body, 16);
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody));
|
||||
if (n != 16 || memcmp(rbody, body, 16) != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: skip: sibling entry lost after a skipped "
|
||||
"entry (%d)\n",
|
||||
n);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
|
||||
/* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
|
||||
{
|
||||
cache_back cache;
|
||||
writefail_inject inj;
|
||||
char extra[8192];
|
||||
char rbody[64];
|
||||
int n;
|
||||
|
||||
inj.budget = (size_t) -1; /* no injected failure */
|
||||
inj.fail_errno = 0;
|
||||
inj.writes = 0;
|
||||
inj.fail_once = 0;
|
||||
memset(&cache, 0, sizeof(cache));
|
||||
cache.type = 1;
|
||||
cache.log = stderr;
|
||||
cache.errlog = stderr;
|
||||
cache.hashtable = coucal_new(0);
|
||||
cache.zipOutput = selftest_open_failing_zip(path, &inj);
|
||||
opt->state.exit_xh = 0;
|
||||
|
||||
writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
|
||||
writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
|
||||
zipClose(cache.zipOutput, NULL);
|
||||
cache.zipOutput = NULL;
|
||||
|
||||
if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: mirror aborted (flagged=%d, "
|
||||
"exit_xh=%d)\n",
|
||||
(int) cache.zipWriteFailed, opt->state.exit_xh);
|
||||
fail++;
|
||||
}
|
||||
if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody)) >= 0) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: in-memory entry was stored\n");
|
||||
fail++;
|
||||
}
|
||||
n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
|
||||
sizeof(extra), rbody, sizeof(rbody));
|
||||
if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
|
||||
fprintf(stderr,
|
||||
"cache-writefail: oversize: on-disk entry not stored "
|
||||
"headers-only (%d)\n",
|
||||
n);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
|
||||
freet(body);
|
||||
return fail;
|
||||
}
|
||||
|
||||
int cache_selftests(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
cache_back cache;
|
||||
@@ -909,494 +585,3 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* --- hts_cache_reconcile() policies -------------------------------------- */
|
||||
|
||||
/* All reconcile inputs/outputs, wiped between cases. */
|
||||
static const char *const reconcile_files[] = {
|
||||
"hts-cache/new.zip", "hts-cache/old.zip", "hts-cache/new.dat",
|
||||
"hts-cache/old.dat", "hts-cache/new.ndx", "hts-cache/old.ndx",
|
||||
"hts-cache/new.lst", "hts-cache/old.lst", "hts-cache/new.txt",
|
||||
"hts-cache/old.txt", "hts-in_progress.lock"};
|
||||
|
||||
static char *reconcile_st_path(httrackp *opt, const char *name) {
|
||||
return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), name);
|
||||
}
|
||||
|
||||
static void reconcile_wipe(httrackp *opt) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
|
||||
remove(reconcile_st_path(opt, reconcile_files[i]));
|
||||
}
|
||||
|
||||
/* Create a filler file of exactly `size` bytes. */
|
||||
static void reconcile_put(httrackp *opt, const char *name, size_t size) {
|
||||
FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
|
||||
static const char filler[1024] = {'x'};
|
||||
|
||||
assertf(fp != NULL);
|
||||
while (size > 0) {
|
||||
const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
|
||||
|
||||
assertf(fwrite(filler, 1, n, fp) == n);
|
||||
size -= n;
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
|
||||
static int reconcile_expect(httrackp *opt, const char *name, off_t size,
|
||||
const char *what) {
|
||||
const off_t got = fsize(reconcile_st_path(opt, name));
|
||||
|
||||
if (got != size) {
|
||||
fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
|
||||
name, (int) got, (int) size);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cache_reconcile_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
/* around the interrupted-run thresholds (new < 32768, old > 65536) */
|
||||
static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
|
||||
|
||||
golden_setup(opt, dir);
|
||||
#ifdef _WIN32
|
||||
mkdir(reconcile_st_path(opt, "hts-cache"));
|
||||
#else
|
||||
mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
|
||||
#endif
|
||||
|
||||
/* PROMOTE: a zip old generation replaces a missing new one */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
|
||||
|
||||
/* PROMOTE: an existing new.zip is left alone */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
|
||||
|
||||
/* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
|
||||
zip cache existed) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
|
||||
|
||||
/* PROMOTE: a half-written legacy new pair is replaced by the old pair */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
|
||||
|
||||
/* INTERRUPTED: no lock file, no action */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
|
||||
|
||||
/* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
|
||||
spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
|
||||
|
||||
/* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
|
||||
zip caches: the arm was gated on a legacy new.dat) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
|
||||
|
||||
/* INTERRUPTED: old below the confidence threshold, keep new */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
|
||||
|
||||
/* INTERRUPTED: new big enough to trust, keep it */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.zip", MID);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
|
||||
|
||||
/* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-in_progress.lock", 0);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/new.ndx", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
|
||||
failures +=
|
||||
reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
|
||||
|
||||
/* ROLLBACK: the old zip generation is restored (a zip cache used to lose
|
||||
its only good generation here) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.zip", SOLID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
|
||||
failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
|
||||
|
||||
/* ROLLBACK: sidecars are restored regardless of format */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.lst", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.lst", MID);
|
||||
reconcile_put(opt, "hts-cache/old.txt", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
|
||||
|
||||
/* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.dat", TINY);
|
||||
reconcile_put(opt, "hts-cache/new.ndx", TINY);
|
||||
reconcile_put(opt, "hts-cache/old.dat", SOLID);
|
||||
reconcile_put(opt, "hts-cache/old.ndx", MID);
|
||||
reconcile_put(opt, "hts-cache/old.lst", MID);
|
||||
reconcile_put(opt, "hts-cache/old.txt", MID);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
|
||||
failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
|
||||
|
||||
/* ROLLBACK: nothing to restore, the new generation stays */
|
||||
reconcile_wipe(opt);
|
||||
reconcile_put(opt, "hts-cache/new.zip", TINY);
|
||||
hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
|
||||
failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
|
||||
|
||||
reconcile_wipe(opt);
|
||||
return failures;
|
||||
}
|
||||
|
||||
/* --- read-side corruption injection --------------------------------------- */
|
||||
|
||||
/* canary read back intact after each corruption; victim gets the byte surgery
|
||||
*/
|
||||
#define CORRUPT_ADR "corrupt.example.com"
|
||||
static char corrupt_body_a[33 + 1];
|
||||
static char corrupt_body_b[44 + 1];
|
||||
|
||||
/* Write a fresh two-entry cache: /canary.html then /victim.html. */
|
||||
static void corrupt_build(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
|
||||
strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
|
||||
is later overwritten with a forged oversized X-Size (same byte length). */
|
||||
static void corrupt_build_etag(httrackp *opt) {
|
||||
cache_back cache;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
|
||||
corrupt_body_b, strlen(corrupt_body_b));
|
||||
selftest_close(&cache);
|
||||
}
|
||||
|
||||
/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
|
||||
body on disk): the shape every non-html file is stored with. */
|
||||
static void corrupt_build_disk(httrackp *opt) {
|
||||
cache_back cache;
|
||||
htsblk w;
|
||||
char locw[4];
|
||||
char BIGSTK save[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
|
||||
char *path;
|
||||
FILE *fp;
|
||||
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
CORRUPT_ADR "/victim.bin");
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
strlen(corrupt_body_a));
|
||||
hts_init_htsblk(&w);
|
||||
w.statuscode = 200;
|
||||
w.size = (LLint) sizeof(corrupt_body_b) - 1;
|
||||
strcpybuff(w.msg, "OK");
|
||||
strcpybuff(w.contenttype, "application/octet-stream");
|
||||
strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
|
||||
locw[0] = '\0';
|
||||
w.location = locw;
|
||||
w.is_write = 0;
|
||||
cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
|
||||
0 /* all_in_cache */, NULL);
|
||||
selftest_close(&cache);
|
||||
/* the reader only checks this file exists; it never reads it here */
|
||||
path = fconv(catbuff, sizeof(catbuff), save);
|
||||
(void) structcheck(path);
|
||||
fp = FOPEN(path, "wb");
|
||||
assertf(fp != NULL);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
|
||||
static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
|
||||
const char *rep, size_t nth, size_t total) {
|
||||
LLint fsz = 0;
|
||||
char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
|
||||
const size_t n = (size_t) fsz;
|
||||
size_t k, hits = 0, at = 0;
|
||||
FILE *fp;
|
||||
|
||||
assertf(data != NULL);
|
||||
for (k = 0; k + patlen <= n; k++) {
|
||||
if (memcmp(data + k, pat, patlen) == 0) {
|
||||
hits++;
|
||||
if (hits == nth)
|
||||
at = k;
|
||||
}
|
||||
}
|
||||
assertf(hits == total);
|
||||
memcpy(data + at, rep, patlen);
|
||||
fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, n, fp) == n);
|
||||
fclose(fp);
|
||||
freet(data);
|
||||
}
|
||||
|
||||
/* Garbage the first bytes of the victim's deflated data (2nd local header). */
|
||||
static void corrupt_victim_body(httrackp *opt) {
|
||||
LLint fsz = 0;
|
||||
char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
|
||||
const size_t n = (size_t) fsz;
|
||||
size_t k, hits = 0, off = 0;
|
||||
FILE *fp;
|
||||
|
||||
assertf(data != NULL);
|
||||
for (k = 0; k + 4 <= n; k++) {
|
||||
if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
|
||||
const size_t namelen =
|
||||
(unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
|
||||
const size_t extralen =
|
||||
(unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
|
||||
|
||||
off = k + 30 + namelen + extralen;
|
||||
}
|
||||
}
|
||||
assertf(hits == 2);
|
||||
assertf(off != 0 && off + 4 <= n);
|
||||
memset(data + off, 0xFF, 4);
|
||||
fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
|
||||
assertf(fp != NULL);
|
||||
assertf(fwrite(data, 1, n, fp) == n);
|
||||
fclose(fp);
|
||||
freet(data);
|
||||
}
|
||||
|
||||
/* Read the corrupt /victim.html and, in the SAME read session, the intact
|
||||
/canary.html: the victim must be rejected (wantmsg pins which path) and the
|
||||
canary must still decode byte-exact, proving one bad entry never taints a
|
||||
sibling read. */
|
||||
static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
|
||||
const char *wantmsg, const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v, c;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK lc[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = lc[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
|
||||
if (v.statuscode != STATUSCODE_INVALID) {
|
||||
fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
|
||||
selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
|
||||
fail++;
|
||||
}
|
||||
if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
|
||||
fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
|
||||
selftest_tag, what, v.msg, wantmsg);
|
||||
fail++;
|
||||
}
|
||||
c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
|
||||
if (c.statuscode != 200 || c.adr == NULL ||
|
||||
c.size != (LLint) strlen(corrupt_body_a) ||
|
||||
memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
|
||||
fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
|
||||
c.statuscode);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
if (c.adr != NULL)
|
||||
freet(c.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
|
||||
const char *what) {
|
||||
return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
|
||||
}
|
||||
|
||||
/* Headers-only probe of the disk victim: must parse OK with the size kept. */
|
||||
static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
|
||||
const char *what) {
|
||||
cache_back cache;
|
||||
htsblk v;
|
||||
char BIGSTK lv[HTS_URLMAXSIZE * 2];
|
||||
int fail = 0;
|
||||
|
||||
selftest_open_for_read(&cache, opt);
|
||||
lv[0] = '\0';
|
||||
v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
|
||||
if (v.statuscode != 200 || v.size != wantsize) {
|
||||
fprintf(stderr,
|
||||
"%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
|
||||
selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
|
||||
fail++;
|
||||
}
|
||||
if (v.adr != NULL)
|
||||
freet(v.adr);
|
||||
selftest_close(&cache);
|
||||
return fail;
|
||||
}
|
||||
|
||||
/* One zip corruption case: build, patch, then check victim+canary in-session.
|
||||
*/
|
||||
static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
|
||||
size_t nth, size_t total, const char *wantmsg,
|
||||
const char *what) {
|
||||
corrupt_build(opt);
|
||||
corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
|
||||
return corrupt_expect_victim(opt, wantmsg, what);
|
||||
}
|
||||
|
||||
int cache_corruption_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
selftest_tag = "cache-corrupt";
|
||||
golden_setup(opt, dir);
|
||||
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
|
||||
"Cache Read Error : Read Data", "oversized X-Size");
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
|
||||
"Cache Read Error : Bad Size", "negative X-Size");
|
||||
/* both entries carry the line; the victim's is the second */
|
||||
failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
|
||||
"Previous cache file not found (empty filename)",
|
||||
"blanked X-In-Cache");
|
||||
/* smashed local file header: the entry is dropped at index load */
|
||||
failures +=
|
||||
corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
|
||||
"File Cache Entry Not Found", "smashed local header");
|
||||
|
||||
corrupt_build(opt);
|
||||
corrupt_victim_body(opt);
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
|
||||
"garbled deflate stream");
|
||||
|
||||
/* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
|
||||
truncates negative in the (int) cast the malloc uses: a wraparound alloc.
|
||||
cache_add asserts size fits an int, so such a value only reaches the reader
|
||||
from a corrupt/foreign cache; inject it by overwriting the victim's long
|
||||
Etag line with a same-length forged X-Size line (the parser keeps the last
|
||||
X-Size it sees), keeping the zip byte-length and offsets intact. */
|
||||
corrupt_build_etag(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483648AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
|
||||
"X-Size above INT_MAX");
|
||||
|
||||
/* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
|
||||
is how every >2GB non-html file is stored. It must survive a header probe
|
||||
(or every update re-fetches the file); an in-memory read still rejects. */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483648AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
|
||||
"headers-only X-Size above INT_MAX");
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"in-memory X-Size above INT_MAX");
|
||||
|
||||
/* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: 2147483647AAAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"in-memory X-Size at INT_MAX");
|
||||
|
||||
/* the negative check must stay global, headers-only included */
|
||||
corrupt_build_disk(opt);
|
||||
corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
|
||||
"X-Size: -2147483648AAAAAAA", 1, 1);
|
||||
failures += corrupt_expect_victim_fil(opt, "/victim.bin",
|
||||
"Cache Read Error : Bad Size",
|
||||
"headers-only negative X-Size");
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
@@ -52,19 +52,6 @@ int cache_selftests(httrackp *opt, const char *dir);
|
||||
committed file, never by the test). Returns the failed-check count. */
|
||||
int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
|
||||
|
||||
/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
|
||||
drop just the entry otherwise. Returns the failed-check count. */
|
||||
int cache_write_failure_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
/* Exercise the hts_cache_reconcile() generation policies on file fixtures
|
||||
under <dir>. Returns the failed-check count. */
|
||||
int cache_reconcile_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
|
||||
under <dir> and assert every case degrades to STATUSCODE_INVALID without
|
||||
tainting a sibling entry. */
|
||||
int cache_corruption_selftest(httrackp *opt, const char *dir);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -64,7 +64,7 @@ Please visit our Website: http://www.httrack.com
|
||||
// catch_url_init(&port,&return_host);
|
||||
HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) {
|
||||
T_SOC soc;
|
||||
int try_to_listen_to[] = {8080, 3128, 80, 81, 82, 8081, 3129, 0, -1};
|
||||
int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 };
|
||||
int i = 0;
|
||||
|
||||
do {
|
||||
@@ -175,9 +175,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
|
||||
//
|
||||
socinput(soc, line, 1000);
|
||||
if (strnotempty(line)) {
|
||||
/* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
|
||||
protocol[256] */
|
||||
if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
|
||||
if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
|
||||
lien_adrfil af;
|
||||
|
||||
// méthode en majuscule
|
||||
@@ -197,6 +195,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
|
||||
htsblk blkretour;
|
||||
|
||||
hts_init_htsblk(&blkretour);
|
||||
//memset(&blkretour, 0, sizeof(htsblk)); // effacer
|
||||
blkretour.location = loc; // si non nul, contiendra l'adresse véritable en cas de moved xx
|
||||
// Lire en têtes restants
|
||||
sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
|
||||
@@ -206,8 +205,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
|
||||
strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
|
||||
strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
|
||||
}
|
||||
// CR/LF final de l'en tête inutile car déja placé via la ligne vide
|
||||
// juste au dessus
|
||||
// CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
|
||||
//strcatbuff(data,"\r\n");
|
||||
if (blkretour.totalsize > 0) {
|
||||
int len = (int) min(blkretour.totalsize, 32000);
|
||||
int pos = (int) strlen(data);
|
||||
|
||||
@@ -841,6 +841,71 @@ static unsigned int nlz8(unsigned char x) {
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* Sample. */
|
||||
#if 0
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
int i;
|
||||
int hex = 0;
|
||||
|
||||
#define READ_INT(DEST) \
|
||||
( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1) \
|
||||
|| (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
|
||||
|
||||
for(i = 1 ; i < argc ; i++) {
|
||||
unsigned int uc, from, to;
|
||||
|
||||
if (strcmp(argv[i], "--hex") == 0) {
|
||||
hex = 1;
|
||||
}
|
||||
else if (strcmp(argv[i], "--decimal") == 0) {
|
||||
hex = 0;
|
||||
}
|
||||
else if (strcmp(argv[i], "--decode") == 0) {
|
||||
#define RD fgetc_unlocked(stdin)
|
||||
#define WR(C) do { \
|
||||
if (C != -1) { \
|
||||
printf("%04x\n", C); \
|
||||
} else if (!feof(stdin)) { \
|
||||
fprintf(stderr, "read error\n"); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while(0)
|
||||
while(!feof(stdin)) {
|
||||
READ_UNICODE(RD, WR);
|
||||
}
|
||||
#undef RD
|
||||
#undef WR
|
||||
}
|
||||
else if (strcmp(argv[i], "-range") == 0
|
||||
&& i + 2 < argc
|
||||
&& (++i, 1)
|
||||
&& READ_INT(from)
|
||||
&& (++i, 1)
|
||||
&& READ_INT(to)
|
||||
) {
|
||||
unsigned int i;
|
||||
for(i = from ; i < to ; i++) {
|
||||
#define EM(C) fputc_unlocked(C, stdout)
|
||||
EMIT_UNICODE(i, EM);
|
||||
#undef EM
|
||||
}
|
||||
}
|
||||
else if (READ_INT(uc)) {
|
||||
#define EM(C) fputc_unlocked(C, stdout)
|
||||
EMIT_UNICODE(uc, EM);
|
||||
#undef EM
|
||||
}
|
||||
else {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/* IDNA helpers. */
|
||||
#undef ADD_BYTE
|
||||
#undef INCREASE_CAPA
|
||||
|
||||
@@ -54,6 +54,10 @@ Please visit our Website: http://www.httrack.com
|
||||
// extension par défaut pour fichiers n'en ayant pas
|
||||
#define DEFAULT_EXT ".html"
|
||||
#define DEFAULT_EXT_SHORT ".htm"
|
||||
// #define DEFAULT_BIN_EXT ".bin"
|
||||
// #define DEFAULT_BIN_EXT_SHORT ".bin"
|
||||
// #define DEFAULT_EXT ".txt"
|
||||
// #define DEFAULT_EXT_SHORT ".txt"
|
||||
|
||||
// éviter les /nul, /con..
|
||||
#define HTS_OVERRIDE_DOS_FOLDERS 1
|
||||
@@ -76,6 +80,10 @@ Please visit our Website: http://www.httrack.com
|
||||
// always direct-to-disk (0/1)
|
||||
#define HTS_DIRECTDISK_ALWAYS 1
|
||||
|
||||
// gérer une table de hachage?
|
||||
// REMOVED
|
||||
// #define HTS_HASH 1
|
||||
|
||||
// fast cache (build hash table)
|
||||
#define HTS_FAST_CACHE 1
|
||||
|
||||
@@ -89,10 +97,27 @@ Please visit our Website: http://www.httrack.com
|
||||
// always transform a '//' into a sigle '/'
|
||||
#define HTS_STRIP_DOUBLE_SLASH 0
|
||||
|
||||
// case-sensitive pour les dossiers et fichiers (0/1)
|
||||
// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
|
||||
// pas très utile..
|
||||
// ..et pas bcp respecté]
|
||||
// REMOVED
|
||||
// #define HTS_CASSE 0
|
||||
|
||||
// Un fichier ayant une taille différente du content-length doit il être annulé?
|
||||
// SEE opt.tolerant and opt.http10
|
||||
// #define HTS_CL_IS_FATAL 0
|
||||
|
||||
// une erreur supprime le fichier sur disque
|
||||
// (non fixé pour cause de retry)
|
||||
#define HTS_REMOVE_BAD_FILES 0
|
||||
|
||||
// en cas de Range: xx- donnant un Content-length: xx
|
||||
// alors skipper le fichier, considéré comme transmis
|
||||
// #define HTS_SKIP_FULL_RANGE 1
|
||||
|
||||
// nombre max de filtres que l'utilisateur peut fixer
|
||||
// #define HTS_FILTERSMAX 10000
|
||||
#define HTS_FILTERSINC 1000
|
||||
|
||||
// connect non bloquant? (poll sur write)
|
||||
|
||||
751
src/htscore.c
751
src/htscore.c
File diff suppressed because it is too large
Load Diff
@@ -152,15 +152,6 @@ struct lien_adrfilsave {
|
||||
char save[HTS_URLMAXSIZE * 2]; /**< local save path (with directory) */
|
||||
};
|
||||
|
||||
/** Per-slot connect-fallback bookkeeping (parallel to struct_back.lnk).
|
||||
Tracks which resolved address the slot is currently connecting to so a
|
||||
stuck connect can be retried against the next one. */
|
||||
typedef struct hts_connect_fallback {
|
||||
int addr_index; /**< candidate being connected (0-based) */
|
||||
int addr_count; /**< resolved addresses; -1 = not yet probed */
|
||||
TStamp connect_start; /**< when the current candidate's connect began */
|
||||
} hts_connect_fallback;
|
||||
|
||||
/** The download-slot ring: the set of concurrent transfers in flight.
|
||||
Allocated/owned by the engine; consumers (status callbacks, the loop)
|
||||
read it but do not resize or free it. */
|
||||
@@ -177,7 +168,6 @@ struct struct_back {
|
||||
int count; /**< number of usable slots (back_max) */
|
||||
coucal ready; /**< index of slots whose transfer completed */
|
||||
LLint ready_size_bytes; /**< total bytes buffered in completed slots */
|
||||
hts_connect_fallback *connect_fallback; /**< per-slot, count+1 entries */
|
||||
};
|
||||
|
||||
typedef struct cache_back_zip_entry cache_back_zip_entry;
|
||||
@@ -214,9 +204,6 @@ struct cache_back {
|
||||
cache_back_zip_entry *zipEntries;
|
||||
int zipEntriesOffs;
|
||||
int zipEntriesCapa;
|
||||
hts_boolean
|
||||
zipWriteFailed; /**< a cache write failed; stop touching the stream */
|
||||
int zipWriteFailures; /**< consecutive entry write failures; reset on store */
|
||||
};
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_hash_struct
|
||||
@@ -235,12 +222,8 @@ struct hash_struct {
|
||||
coucal adrfil;
|
||||
/* former address+path -> link index (renamed/moved entries) */
|
||||
coucal former_adrfil;
|
||||
/* effective urlhack sub-flags: www.==host / // collapse / query-arg sort */
|
||||
hts_boolean norm_host;
|
||||
hts_boolean norm_slash;
|
||||
hts_boolean norm_query;
|
||||
/* query-strip keys (not owned); set from opt->strip_query at hash_init */
|
||||
const char *strip_query;
|
||||
/* scratch buffers reused across lookups (not reentrant) */
|
||||
int normalized;
|
||||
char normfil[HTS_URLMAXSIZE * 2];
|
||||
char normfil2[HTS_URLMAXSIZE * 2];
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
@@ -363,42 +346,12 @@ void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
|
||||
|
||||
void usercommand_exe(const char *cmd, const char *file);
|
||||
|
||||
// Finish the makeindex index.html (footer + refresh meta), run usercommand.
|
||||
// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
|
||||
void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
|
||||
FILE **makeindex_fp, int makeindex_links,
|
||||
const char *makeindex_firstlink,
|
||||
const char *template_footer, const char *adr,
|
||||
const char *fil);
|
||||
|
||||
// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
|
||||
// closed+NULLed on write. Precondition: ht_len>0.
|
||||
void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
|
||||
FILE **fp, const char *ht_buff, size_t ht_len,
|
||||
const char *adr, const char *fil, const char *save);
|
||||
|
||||
int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
|
||||
|
||||
int fspc(httrackp * opt, FILE * fp, const char *type);
|
||||
|
||||
char *next_token(char *p, int flag);
|
||||
|
||||
/* Like fil_normalized(), but first drops query keys in STRIP (comma-separated,
|
||||
"*" = all); STRIP NULL/empty behaves exactly like fil_normalized(). */
|
||||
char *fil_normalized_filtered(const char *source, char *dest,
|
||||
const char *strip);
|
||||
|
||||
/* As fil_normalized_filtered(), but DO_SLASH/DO_QUERY gate the // collapse and
|
||||
the query-argument sort independently (the urlhack sub-flags). */
|
||||
char *fil_normalized_filtered_ex(const char *source, char *dest,
|
||||
const char *strip, int do_slash, int do_query);
|
||||
|
||||
/* For URL ADR/FIL, return (in DEST) the comma keylist to strip from the
|
||||
'\n'-separated "[pattern=]keys" RULES (patterns matched on host/path via
|
||||
strjoker, last wins); NULL if none match. Feeds fil_normalized_filtered(). */
|
||||
const char *hts_query_strip_keys(const char *rules, const char *adr,
|
||||
const char *fil, char *dest, size_t destsize);
|
||||
|
||||
/* Read a whole file into a freshly malloc'd, NUL-terminated buffer; the caller
|
||||
owns it and must release it with freet(). Return NULL on missing/unreadable
|
||||
file (readfile_or substitutes defaultdata instead). The byte content is NOT
|
||||
@@ -412,16 +365,13 @@ char *readfile_utf8(const char *fil);
|
||||
|
||||
char *readfile_or(const char *fil, const char *defaultdata);
|
||||
|
||||
#if 0
|
||||
void check_rate(TStamp stat_timestart, int maxrate);
|
||||
#endif
|
||||
|
||||
/* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
|
||||
Not thread-safe; call from the single crawl loop. */
|
||||
|
||||
/* True if a connecting slot should give up on the current address and try the
|
||||
next one: a fallback address remains (addr_index+1 < addr_count) and the
|
||||
candidate has been connecting for at least its deadline, min(timeout, an
|
||||
internal cap). elapsed/timeout in seconds. Exposed for the -#D self-test. */
|
||||
int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
|
||||
int timeout);
|
||||
|
||||
/* How many new sockets may be opened now, honoring maxsoc and the maxconn rate
|
||||
limit (>=0). _strict ignores reserved-slot headroom; the plain form leaves
|
||||
room for naming tests and stops at 0 when the stack is nearly full. */
|
||||
@@ -429,19 +379,6 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
|
||||
|
||||
int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
|
||||
|
||||
/* One engine-loop tick: refresh the transfer stats and run the loop callback
|
||||
for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
|
||||
hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
|
||||
|
||||
/* Wait until a test socket can be plugged, pumping transfers, stats and the
|
||||
loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
|
||||
hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
|
||||
cache_back *cache, int ptr);
|
||||
|
||||
/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
|
||||
timestamp seed so it is stable within one gap and rerolls per launch. */
|
||||
int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
|
||||
|
||||
/* Schedule more links from the heap into free slots. Returns the number queued,
|
||||
or <=0 if none could be added (no free slot / paused / stopped). */
|
||||
int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
@@ -463,6 +400,11 @@ int ask_continue(httrackp * opt);
|
||||
/* Number of decimal digits in n. */
|
||||
int nombre_digit(int n);
|
||||
|
||||
// Java
|
||||
#if 0
|
||||
int hts_add_file(char *file, int file_position);
|
||||
#endif
|
||||
|
||||
// Polling
|
||||
#if HTS_POLL
|
||||
int check_flot(T_SOC s);
|
||||
@@ -485,8 +427,4 @@ void voidf(void);
|
||||
/* HTML marker comment marking where the top index is spliced. */
|
||||
#define HTS_TOPINDEX "TOP_INDEX_HTTRACK"
|
||||
|
||||
/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
|
||||
#define HTS_HTMLESCAPE_MAXEXP 5 /* escape_for_html_print: '&'->"&" */
|
||||
#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */
|
||||
|
||||
#endif
|
||||
|
||||
1360
src/htscoremain.c
1360
src/htscoremain.c
File diff suppressed because it is too large
Load Diff
@@ -69,15 +69,11 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
|
||||
typedef struct t_hts_callbackarg t_hts_callbackarg;
|
||||
#endif
|
||||
|
||||
/* Marks a symbol an external wrapper module exports back to the engine.
|
||||
Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
|
||||
hide their own hts_plug()/hts_unplug() entry points. */
|
||||
/* Marks a symbol an external wrapper module exports back to the engine
|
||||
(dllexport on Windows, nothing elsewhere). */
|
||||
#ifndef EXTERNAL_FUNCTION
|
||||
#ifdef _WIN32
|
||||
#define EXTERNAL_FUNCTION __declspec(dllexport)
|
||||
#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) || \
|
||||
(defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
|
||||
#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
|
||||
#else
|
||||
#define EXTERNAL_FUNCTION
|
||||
#endif
|
||||
|
||||
@@ -1,359 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: htsdns_selftest.c subroutines: */
|
||||
/* in-process self-test for the DNS resolver and cache */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
/* Routes the resolver through a scripted getaddrinfo (hts_resolver_backend)
|
||||
instead of the network, so resolution and the DNS cache are testable for a
|
||||
fixed set of scenarios (IPv4/IPv6/dual-stack, errors, family filter,
|
||||
cache reuse) with no live DNS. */
|
||||
|
||||
#define HTS_INTERNAL_BYTECODE
|
||||
|
||||
#include "htsdns_selftest.h"
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htslib.h"
|
||||
#include "htsnet.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#if HTS_INET6 != 0
|
||||
|
||||
/* IPV6_resolver: 0 = v4+v6, 1 = v4 only, 2 = v6 only (htscoremain -@i). */
|
||||
extern int IPV6_resolver;
|
||||
|
||||
/* One scripted host: either a getaddrinfo error, or an ordered address list. */
|
||||
typedef struct mock_addr {
|
||||
int family; /* AF_INET / AF_INET6 */
|
||||
unsigned char addr[16]; /* 4 (v4) or 16 (v6) meaningful bytes */
|
||||
} mock_addr;
|
||||
|
||||
typedef struct mock_host {
|
||||
const char *name;
|
||||
int gai_err; /* non-zero: getaddrinfo returns this */
|
||||
int naddr;
|
||||
mock_addr addr[6];
|
||||
int calls; /* times the backend resolved this host */
|
||||
} mock_host;
|
||||
|
||||
static mock_host mock_hosts[] = {
|
||||
{"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
|
||||
{"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
|
||||
/* dual stack, IPv6 first (RFC 6724 order) then IPv4 */
|
||||
{"dual.test",
|
||||
0,
|
||||
2,
|
||||
{{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 2}}, {AF_INET, {5, 6, 7, 8}}},
|
||||
0},
|
||||
/* dual stack, IPv4 first: distinguishes "keep the first address" from
|
||||
"prefer a family", so the selection contract is actually pinned. */
|
||||
{"dual4.test",
|
||||
0,
|
||||
2,
|
||||
{{AF_INET, {9, 10, 11, 12}},
|
||||
{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 3}}},
|
||||
0},
|
||||
/* more addresses than HTS_MAXADDRNUM: the list must clamp to the cap. */
|
||||
{"many.test",
|
||||
0,
|
||||
6,
|
||||
{{AF_INET, {10, 0, 0, 1}},
|
||||
{AF_INET, {10, 0, 0, 2}},
|
||||
{AF_INET, {10, 0, 0, 3}},
|
||||
{AF_INET, {10, 0, 0, 4}},
|
||||
{AF_INET, {10, 0, 0, 5}},
|
||||
{AF_INET, {10, 0, 0, 6}}},
|
||||
0},
|
||||
{"nodns.test", EAI_NONAME, 0, {{0}}, 0},
|
||||
};
|
||||
|
||||
static mock_host *mock_find(const char *name) {
|
||||
for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
|
||||
if (strcmp(mock_hosts[i].name, name) == 0)
|
||||
return &mock_hosts[i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void mock_reset_calls(void) {
|
||||
for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
|
||||
mock_hosts[i].calls = 0;
|
||||
}
|
||||
|
||||
/* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
|
||||
static struct addrinfo *mock_mkai(const mock_addr *a) {
|
||||
struct addrinfo *ai = calloct(1, sizeof(*ai));
|
||||
|
||||
ai->ai_family = a->family;
|
||||
if (a->family == AF_INET) {
|
||||
struct sockaddr_in *sin = calloct(1, sizeof(*sin));
|
||||
|
||||
sin->sin_family = AF_INET;
|
||||
memcpy(&sin->sin_addr, a->addr, 4);
|
||||
ai->ai_addr = (struct sockaddr *) sin;
|
||||
ai->ai_addrlen = sizeof(*sin);
|
||||
} else {
|
||||
struct sockaddr_in6 *sin6 = calloct(1, sizeof(*sin6));
|
||||
|
||||
sin6->sin6_family = AF_INET6;
|
||||
memcpy(&sin6->sin6_addr, a->addr, 16);
|
||||
ai->ai_addr = (struct sockaddr *) sin6;
|
||||
ai->ai_addrlen = sizeof(*sin6);
|
||||
}
|
||||
return ai;
|
||||
}
|
||||
|
||||
static int mock_getaddrinfo(const char *node, const char *service,
|
||||
const struct addrinfo *hints,
|
||||
struct addrinfo **res) {
|
||||
mock_host *const h = mock_find(node);
|
||||
const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
|
||||
struct addrinfo *head = NULL, *tail = NULL;
|
||||
|
||||
(void) service;
|
||||
*res = NULL;
|
||||
if (h == NULL)
|
||||
return EAI_NONAME;
|
||||
h->calls++; /* a real backend hit; a cached host skips this */
|
||||
if (h->gai_err != 0)
|
||||
return h->gai_err;
|
||||
for (int i = 0; i < h->naddr; i++) {
|
||||
if (want != PF_UNSPEC && want != h->addr[i].family)
|
||||
continue; /* honor the requested family (v4/v6 only) */
|
||||
struct addrinfo *const ai = mock_mkai(&h->addr[i]);
|
||||
|
||||
if (head == NULL)
|
||||
head = ai;
|
||||
else
|
||||
tail->ai_next = ai;
|
||||
tail = ai;
|
||||
}
|
||||
if (head == NULL)
|
||||
return EAI_NONAME; /* filtered to empty, as the libc resolver does */
|
||||
*res = head;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void mock_freeaddrinfo(struct addrinfo *res) {
|
||||
while (res != NULL) {
|
||||
struct addrinfo *const next = res->ai_next;
|
||||
|
||||
freet(res->ai_addr);
|
||||
freet(res);
|
||||
res = next;
|
||||
}
|
||||
}
|
||||
|
||||
static const hts_resolver_backend mock_backend = {mock_getaddrinfo,
|
||||
mock_freeaddrinfo};
|
||||
|
||||
static int failures = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
failures++; \
|
||||
fprintf(stderr, "dns-selftest: FAIL at %s:%d: %s\n", __FILE__, __LINE__, \
|
||||
#cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* Resolve via the uncached entry point; return the address family, or
|
||||
AF_UNSPEC if the host did not resolve. */
|
||||
static int resolve_family_nocache(const char *host) {
|
||||
SOCaddr addr;
|
||||
const char *err = NULL;
|
||||
|
||||
if (hts_dns_resolve_nocache2(host, &addr, &err) == NULL)
|
||||
return AF_UNSPEC;
|
||||
return SOCaddr_sinfamily(addr);
|
||||
}
|
||||
|
||||
int dns_selftests(httrackp *opt) {
|
||||
failures = 0;
|
||||
hts_dns_set_resolver_backend(&mock_backend);
|
||||
|
||||
/* IPv4-only / IPv6-only hosts map to the right family. */
|
||||
IPV6_resolver = 0;
|
||||
CHECK(resolve_family_nocache("v4only.test") == AF_INET);
|
||||
CHECK(resolve_family_nocache("v6only.test") == AF_INET6);
|
||||
|
||||
/* Dual-stack: the single-address API returns the *first* resolved address.
|
||||
Both orderings pin selection by position, not a family preference. The
|
||||
multi-address API (resolve_all, below) exposes the whole list. */
|
||||
CHECK(resolve_family_nocache("dual.test") == AF_INET6); /* v6 listed first */
|
||||
CHECK(resolve_family_nocache("dual4.test") == AF_INET); /* v4 listed first */
|
||||
|
||||
/* Unknown host does not resolve. */
|
||||
CHECK(resolve_family_nocache("nodns.test") == AF_UNSPEC);
|
||||
|
||||
/* Family filter (-@i4 / -@i6) selects v4 / v6 out of the dual-stack host. */
|
||||
IPV6_resolver = 1;
|
||||
CHECK(resolve_family_nocache("dual.test") == AF_INET);
|
||||
IPV6_resolver = 2;
|
||||
CHECK(resolve_family_nocache("dual.test") == AF_INET6);
|
||||
IPV6_resolver = 0;
|
||||
|
||||
/* Cached driver resolves a host once and reuses the *same* address. */
|
||||
mock_reset_calls();
|
||||
{
|
||||
SOCaddr a1, a2;
|
||||
char ip1[64], ip2[64];
|
||||
const char *err = NULL;
|
||||
|
||||
CHECK(hts_dns_resolve2(opt, "v4only.test", &a1, &err) != NULL);
|
||||
CHECK(hts_dns_resolve2(opt, "v4only.test", &a2, &err) != NULL);
|
||||
CHECK(mock_find("v4only.test")->calls == 1);
|
||||
/* the cache returns the right address, not merely a hit for the key */
|
||||
SOCaddr_inetntoa(ip1, sizeof(ip1), a1);
|
||||
SOCaddr_inetntoa(ip2, sizeof(ip2), a2);
|
||||
CHECK(strcmp(ip1, "1.2.3.4") == 0);
|
||||
CHECK(strcmp(ip1, ip2) == 0);
|
||||
}
|
||||
|
||||
/* A negative result is cached too: a second lookup does not re-resolve. */
|
||||
{
|
||||
SOCaddr a1, a2;
|
||||
const char *err = NULL;
|
||||
|
||||
CHECK(hts_dns_resolve2(opt, "nodns.test", &a1, &err) == NULL);
|
||||
CHECK(hts_dns_resolve2(opt, "nodns.test", &a2, &err) == NULL);
|
||||
CHECK(mock_find("nodns.test")->calls == 1); /* resolved once, then cached */
|
||||
}
|
||||
|
||||
/* Multi-address resolution: count and order are the connect-fallback
|
||||
contract. A dead first address is retried against the next, so both must be
|
||||
exact. */
|
||||
mock_reset_calls();
|
||||
{
|
||||
SOCaddr addrs[HTS_MAXADDRNUM];
|
||||
char ip[64];
|
||||
const char *err = NULL;
|
||||
|
||||
/* dual-stack, in resolver order: [0]=v6, [1]=v4 */
|
||||
CHECK(hts_dns_resolve_all(opt, "dual.test", addrs, HTS_MAXADDRNUM, &err) ==
|
||||
2);
|
||||
CHECK(SOCaddr_sinfamily(addrs[0]) == AF_INET6);
|
||||
CHECK(SOCaddr_sinfamily(addrs[1]) == AF_INET);
|
||||
SOCaddr_inetntoa(ip, sizeof(ip), addrs[1]);
|
||||
CHECK(strcmp(ip, "5.6.7.8") == 0);
|
||||
CHECK(mock_find("dual.test")->calls ==
|
||||
1); /* one backend hit for the list */
|
||||
|
||||
/* single-address host: count 1 */
|
||||
CHECK(hts_dns_resolve_all(opt, "v4only.test", addrs, HTS_MAXADDRNUM,
|
||||
&err) == 1);
|
||||
SOCaddr_inetntoa(ip, sizeof(ip), addrs[0]);
|
||||
CHECK(strcmp(ip, "1.2.3.4") == 0);
|
||||
|
||||
/* does-not-resolve: count 0 (negative), no addresses */
|
||||
CHECK(hts_dns_resolve_all(opt, "nodns.test", addrs, HTS_MAXADDRNUM, &err) ==
|
||||
0);
|
||||
|
||||
/* more than the cap: the kept list is clamped to HTS_MAXADDRNUM, keeping
|
||||
the FIRST addresses in resolver order (not some other window) */
|
||||
CHECK(hts_dns_resolve_all(opt, "many.test", addrs, HTS_MAXADDRNUM, &err) ==
|
||||
HTS_MAXADDRNUM);
|
||||
SOCaddr_inetntoa(ip, sizeof(ip), addrs[0]);
|
||||
CHECK(strcmp(ip, "10.0.0.1") == 0);
|
||||
SOCaddr_inetntoa(ip, sizeof(ip), addrs[HTS_MAXADDRNUM - 1]);
|
||||
CHECK(strcmp(ip, "10.0.0.4") == 0);
|
||||
|
||||
/* family filter still applies through the list path */
|
||||
IPV6_resolver = 1;
|
||||
CHECK(hts_dns_resolve_all(opt, "dual4.test", addrs, HTS_MAXADDRNUM, &err) ==
|
||||
1);
|
||||
CHECK(SOCaddr_sinfamily(addrs[0]) == AF_INET);
|
||||
IPV6_resolver = 0;
|
||||
}
|
||||
|
||||
/* newhttp_addr() must connect to the addr_index-th address, not always the
|
||||
first: this is what back_connect_next relies on to reach the fallback. */
|
||||
{
|
||||
htsblk r;
|
||||
int count = -1;
|
||||
T_SOC s;
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
s = newhttp_addr(opt, "dual.test", &r, 80, 0, 0, &count);
|
||||
CHECK(count == 2);
|
||||
CHECK(SOCaddr_sinfamily(r.address) == AF_INET6); /* index 0 = v6 */
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
|
||||
hts_init_htsblk(&r);
|
||||
count = -1;
|
||||
s = newhttp_addr(opt, "dual.test", &r, 80, 0, 1, &count);
|
||||
CHECK(count == 2);
|
||||
CHECK(SOCaddr_sinfamily(r.address) == AF_INET); /* index 1 = v4 */
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
|
||||
/* out-of-range index: no address selected (address stays unset) */
|
||||
hts_init_htsblk(&r);
|
||||
s = newhttp_addr(opt, "dual.test", &r, 80, 0, 2, NULL);
|
||||
CHECK(s == INVALID_SOCKET);
|
||||
if (s != INVALID_SOCKET)
|
||||
deletesoc(s);
|
||||
}
|
||||
|
||||
/* Connect-fallback decision (consumer of the multi-address list): when a
|
||||
stuck connect should abandon the current address for the next one. */
|
||||
{
|
||||
/* no fallback for the last/only candidate, whatever the elapsed time */
|
||||
CHECK(back_connect_fallback_due(0, 1, 9999, 120) == 0);
|
||||
CHECK(back_connect_fallback_due(1, 2, 9999, 120) == 0);
|
||||
CHECK(back_connect_fallback_due(3, 4, 9999, 120) == 0);
|
||||
/* fallback available: wait the per-candidate deadline (cap 10s here) */
|
||||
CHECK(back_connect_fallback_due(0, 2, 9, 120) == 0);
|
||||
CHECK(back_connect_fallback_due(0, 2, 10, 120) == 1);
|
||||
CHECK(back_connect_fallback_due(2, 4, 10, 120) == 1);
|
||||
/* a shorter slot timeout shortens the deadline (min(timeout, cap)) */
|
||||
CHECK(back_connect_fallback_due(0, 2, 4, 5) == 0);
|
||||
CHECK(back_connect_fallback_due(0, 2, 5, 5) == 1);
|
||||
/* no timeout management: never force a fallback */
|
||||
CHECK(back_connect_fallback_due(0, 2, 9999, 0) == 0);
|
||||
}
|
||||
|
||||
hts_dns_set_resolver_backend(NULL);
|
||||
return failures;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
int dns_selftests(httrackp *opt) {
|
||||
(void) opt;
|
||||
return 0; /* resolver seam only exists in the IPv6 build */
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,51 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: htsdns_selftest.h */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSDNS_SELFTEST_DEFH
|
||||
#define HTSDNS_SELFTEST_DEFH
|
||||
|
||||
#ifdef HTS_INTERNAL_BYTECODE
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_httrackp
|
||||
#define HTS_DEF_FWSTRUCT_httrackp
|
||||
typedef struct httrackp httrackp;
|
||||
#endif
|
||||
|
||||
/* Drive the DNS resolver and cache through a scripted (mock) getaddrinfo,
|
||||
asserting address family, single-address selection, negative caching, the
|
||||
IPv4/IPv6 family filter, and that a cached host is resolved only once.
|
||||
Returns the number of failed checks (0 == success). */
|
||||
int dns_selftests(httrackp *opt);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -30,14 +30,12 @@ Please visit our Website: http://www.httrack.com
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "htscharset.h"
|
||||
#include "htsencoding.h"
|
||||
#include "htssafe.h"
|
||||
|
||||
/* static int decode_entity(const uint64_t hash, const size_t len);
|
||||
*/
|
||||
/* static int decode_entity(const unsigned int hash, const size_t len);
|
||||
*/
|
||||
#include "htsentities.h"
|
||||
|
||||
/* hexadecimal conversion */
|
||||
@@ -52,31 +50,30 @@ static int get_hex_value(char c) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
|
||||
*/
|
||||
#define HASH_INIT 0xcbf29ce484222325ULL
|
||||
#define HASH_PRIME 0x100000001b3ULL
|
||||
#define HASH_ADD(HASH, C) \
|
||||
do { \
|
||||
(HASH) ^= (unsigned char) (C); \
|
||||
(HASH) *= HASH_PRIME; \
|
||||
} while (0)
|
||||
/* Numerical Recipes,
|
||||
see <http://en.wikipedia.org/wiki/Linear_congruential_generator> */
|
||||
#define HASH_PRIME ( 1664525 )
|
||||
#define HASH_CONST ( 1013904223 )
|
||||
#define HASH_ADD(HASH, C) do { \
|
||||
(HASH) *= HASH_PRIME; \
|
||||
(HASH) += HASH_CONST; \
|
||||
(HASH) += (C); \
|
||||
} while(0)
|
||||
|
||||
int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
|
||||
size_t i, j, ampStart, ampStartDest;
|
||||
int uc;
|
||||
int hex;
|
||||
uint64_t hash;
|
||||
unsigned int hash;
|
||||
|
||||
assertf(max != 0);
|
||||
for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
|
||||
hash = HASH_INIT;
|
||||
src[i] != '\0'; i++) {
|
||||
for(i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0,
|
||||
uc = -1, hex = 0, hash = 0 ; src[i] != '\0' ; i++) {
|
||||
/* start of entity */
|
||||
if (src[i] == '&') {
|
||||
ampStart = i;
|
||||
ampStartDest = j;
|
||||
hash = HASH_INIT;
|
||||
hash = 0;
|
||||
uc = -1;
|
||||
}
|
||||
/* inside a potential entity */
|
||||
@@ -177,11 +174,14 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
|
||||
}
|
||||
/* alphanumerical entity */
|
||||
else {
|
||||
/* alphanum, capped at the longest name
|
||||
* '∳' (31) */
|
||||
if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') ||
|
||||
(src[i] >= 'A' && src[i] <= 'Z') ||
|
||||
(src[i] >= 'a' && src[i] <= 'z'))) {
|
||||
/* alphanum and not too far ('ϑ' is the longest) */
|
||||
if (i <= ampStart + 10 &&
|
||||
(
|
||||
(src[i] >= '0' && src[i] <= '9')
|
||||
|| (src[i] >= 'A' && src[i] <= 'Z')
|
||||
|| (src[i] >= 'a' && src[i] <= 'z')
|
||||
)
|
||||
) {
|
||||
/* compute hash */
|
||||
HASH_ADD(hash, (unsigned char) src[i]);
|
||||
} else {
|
||||
@@ -190,9 +190,9 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* reserve one byte for the trailing NUL written after the loop */
|
||||
if (j + 1 >= max) {
|
||||
|
||||
/* copy */
|
||||
if (j + 1 > max) {
|
||||
/* overflow */
|
||||
return -1;
|
||||
}
|
||||
@@ -300,11 +300,6 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
|
||||
|
||||
/* Was the character read successfully ? */
|
||||
if (nRead == utfBufferSize) {
|
||||
/* the 'continue' below skips the NUL-reserve guard: re-check */
|
||||
if (utfBufferJ + utfBufferSize >= max) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Rollback write position to sequence start write position */
|
||||
j = utfBufferJ;
|
||||
|
||||
@@ -319,8 +314,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
|
||||
}
|
||||
}
|
||||
|
||||
/* reserve one byte for the trailing NUL written after the loop */
|
||||
if (j + 1 >= max) {
|
||||
/* Check for overflow */
|
||||
if (j + 1 > max) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
13612
src/htsentities.h
13612
src/htsentities.h
File diff suppressed because it is too large
Load Diff
@@ -1,92 +1,75 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Regenerate htsentities.h from the WHATWG named character references.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
src=entities.json
|
||||
url=https://html.spec.whatwg.org/entities.json
|
||||
src=html40.txt
|
||||
url=http://www.w3.org/TR/1998/REC-html40-19980424/html40.txt
|
||||
dest=htsentities.h
|
||||
|
||||
# 64-bit FNV-1a of $1, printed as a C constant. Must match the hash in
|
||||
# htsencoding.c. The offset basis is stored as its wrapped (signed) bit pattern;
|
||||
# bash arithmetic is 64-bit two's complement, so the result is bit-exact.
|
||||
fnv1a() {
|
||||
local s=$1 i c h=$((0xcbf29ce484222325))
|
||||
for ((i = 0; i < ${#s}; i++)); do
|
||||
printf -v c '%d' "'${s:i:1}"
|
||||
h=$(((h ^ (c & 0xff)) * 0x100000001b3))
|
||||
done
|
||||
printf '0x%016xULL' "$h"
|
||||
}
|
||||
(
|
||||
cat <<EOF
|
||||
/*
|
||||
-- ${dest} --
|
||||
FILE GENERATED BY $0, DO NOT MODIFY
|
||||
|
||||
if [ ! -f "$src" ]; then
|
||||
curl -fsS "$url" -o "$src"
|
||||
fi
|
||||
We compute the LCG hash
|
||||
(see <http://en.wikipedia.org/wiki/Linear_congruential_generator>)
|
||||
for each entity. We should in theory check using strncmp() that we
|
||||
actually have the correct entity, but this is actually statistically
|
||||
not needed.
|
||||
|
||||
# Keep ';'-terminated single-codepoint names; the ~93 multi-codepoint refs can't
|
||||
# fit decode_entity's single-codepoint return and are skipped (left verbatim).
|
||||
pairs=$(jq -r '
|
||||
to_entries
|
||||
| map(select((.key | endswith(";")) and (.value.codepoints | length == 1)))
|
||||
| sort_by(.key)
|
||||
| .[] | "\(.key | ltrimstr("&") | rtrimstr(";"))\t\(.value.codepoints[0])"' "$src")
|
||||
We may want to do better, but we expect the hash function to be uniform, and
|
||||
let the compiler be smart enough to optimize the switch (for example by
|
||||
checking in log2() intervals)
|
||||
|
||||
This code has been generated using the evil $0 script.
|
||||
*/
|
||||
|
||||
# Skipped multi-codepoint names, kept to prove none aliases an emitted hash.
|
||||
skipped=$(jq -r '
|
||||
to_entries
|
||||
| map(select((.key | endswith(";")) and (.value.codepoints | length > 1)))
|
||||
| .[] | .key | ltrimstr("&") | rtrimstr(";")' "$src")
|
||||
|
||||
cases=""
|
||||
emit_hashes=""
|
||||
while IFS=$'\t' read -r name cp; do
|
||||
hash=$(fnv1a "$name")
|
||||
cases+=" /* $name */"$'\n'
|
||||
cases+=" case $hash:"$'\n'
|
||||
cases+=" if (len == ${#name}) {"$'\n'
|
||||
cases+=" return $cp;"$'\n'
|
||||
cases+=" }"$'\n'
|
||||
cases+=" break;"$'\n'
|
||||
emit_hashes+="$hash"$'\n'
|
||||
done <<<"$pairs"
|
||||
|
||||
skip_hashes=""
|
||||
while IFS= read -r name; do
|
||||
[ -n "$name" ] && skip_hashes+="$(fnv1a "$name")"$'\n'
|
||||
done <<<"$skipped"
|
||||
|
||||
# The switch keys on the hash alone, so the dispatch is correct only while every
|
||||
# emitted name hashes uniquely; prove it here, no runtime name compare needed.
|
||||
dups=$(printf '%s' "$emit_hashes" | sort | uniq -d || true)
|
||||
if [ -n "$dups" ]; then
|
||||
echo "FATAL: two entity names share a hash (duplicate switch case); change the hash:" >&2
|
||||
echo "$dups" >&2
|
||||
exit 1
|
||||
fi
|
||||
# A skipped name colliding with an emitted hash would mis-decode instead of
|
||||
# staying verbatim; forbid that too.
|
||||
aliased=$(comm -12 <(printf '%s' "$emit_hashes" | sort -u) <(printf '%s' "$skip_hashes" | sort -u) || true)
|
||||
if [ -n "$aliased" ]; then
|
||||
echo "FATAL: a skipped multi-codepoint name aliases an emitted hash:" >&2
|
||||
echo "$aliased" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat >"$dest" <<EOF
|
||||
/* GENERATED by $0 from the WHATWG named character references
|
||||
(${url}). DO NOT EDIT.
|
||||
Dispatch keys on a 64-bit FNV-1a hash of the entity name; the generator
|
||||
aborts on any hash collision, so no runtime name compare is needed. */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
static int decode_entity(const uint64_t hash, const size_t len) {
|
||||
static int decode_entity(const unsigned int hash, const size_t len) {
|
||||
switch(hash) {
|
||||
${cases} }
|
||||
EOF
|
||||
(
|
||||
if test -f ${src}; then
|
||||
cat ${src}
|
||||
else
|
||||
GET "${url}"
|
||||
fi
|
||||
) |
|
||||
grep -E '^<!ENTITY [a-zA-Z0-9_]' |
|
||||
sed \
|
||||
-e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
|
||||
-e 's/-->$//' \
|
||||
-e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/' |
|
||||
(
|
||||
read -r A
|
||||
while test -n "$A"; do
|
||||
ent="${A%% *}"
|
||||
code=$(echo "$A" | cut -f2 -d' ')
|
||||
# compute hash
|
||||
hash=0
|
||||
i=0
|
||||
a=1664525
|
||||
c=1013904223
|
||||
m="$((1 << 32))"
|
||||
while test "$i" -lt ${#ent}; do
|
||||
d="$(echo -n "${ent:${i}:1}" | hexdump -v -e '/1 "%d"')"
|
||||
hash="$((((hash * a) % (m) + d + c) % (m)))"
|
||||
i=$((i + 1))
|
||||
done
|
||||
echo -e " /* $A */"
|
||||
echo -e " case ${hash}u:"
|
||||
echo -e " if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
|
||||
echo -e " return ${code};"
|
||||
echo -e " }"
|
||||
echo -e " break;"
|
||||
|
||||
# next
|
||||
read -r A
|
||||
done
|
||||
)
|
||||
cat <<EOF
|
||||
}
|
||||
/* unknown */
|
||||
return -1;
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "wrote $dest ($(grep -c '^ case ' "$dest") entities)" >&2
|
||||
) >${dest}
|
||||
|
||||
@@ -76,8 +76,7 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
|
||||
}
|
||||
if (size)
|
||||
sz = *size;
|
||||
/* size unknown (scan time): no size pointer => size tests stay neutral */
|
||||
if (strjoker(nom, filters[i] + filteroffs, size ? &sz : NULL, size_flag)) {
|
||||
if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) { // reconnu
|
||||
if (size)
|
||||
if (sz != *size)
|
||||
sizelimit = sz;
|
||||
@@ -102,8 +101,9 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
|
||||
// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
|
||||
// * = toute lettre
|
||||
// --?-- : spécifique à HTTrack et aux ?
|
||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
LLint *size, int *size_flag) {
|
||||
HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
|
||||
int *size_flag) {
|
||||
//int err=0;
|
||||
if (strnotempty(joker) == 0) { // fin de chaine joker
|
||||
if (strnotempty(chaine) == 0) // fin aussi pour la chaine: ok
|
||||
return chaine;
|
||||
@@ -192,12 +192,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
int len = (int) strlen(joker);
|
||||
|
||||
while((joker[i] != RIGHT) && (joker[i]) && (i < len)) {
|
||||
// '\' escapes the next char as a literal member, e.g. *[\[\]]
|
||||
if (joker[i] == '\\' && joker[i + 1] != '\0') {
|
||||
i++;
|
||||
pass[(int) (unsigned char) joker[i]] = 1;
|
||||
i++;
|
||||
} else if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
|
||||
if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
|
||||
int lsize = 0;
|
||||
int lverdict;
|
||||
|
||||
@@ -225,9 +220,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
while(isdigit((unsigned char) joker[i]))
|
||||
i++;
|
||||
}
|
||||
} else if (joker[i + 1] == '-' && joker[i + 2] != '\0') {
|
||||
// range *[A-Z]; the '\0' guard rejects a truncated *[a- (else
|
||||
// i+=3 overshoots the NUL)
|
||||
} else if (joker[i + 1] == '-') { // 2 car, ex: *[A-Z]
|
||||
if ((int) (unsigned char) joker[i + 2] >
|
||||
(int) (unsigned char) joker[i]) {
|
||||
int j;
|
||||
@@ -235,9 +228,14 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
for(j = (int) (unsigned char) joker[i];
|
||||
j <= (int) (unsigned char) joker[i + 2]; j++)
|
||||
pass[j] = 1;
|
||||
|
||||
}
|
||||
// else err=1;
|
||||
i += 3;
|
||||
} else { // 1 car, ex: *[ ]
|
||||
} else { // 1 car, ex: *[ ]
|
||||
if (joker[i + 2] == '\\' && joker[i + 3] != 0) { // escaped char, such as *[\[] or *[\]]
|
||||
i++;
|
||||
}
|
||||
pass[(int) (unsigned char) joker[i]] = 1;
|
||||
i++;
|
||||
}
|
||||
@@ -259,6 +257,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
for(i = 0; i < 256; i++)
|
||||
pass[i] = 1; // tout autoriser
|
||||
jmp = 1;
|
||||
////if (joker[2]==LEFT) jmp=3; // permet de recher *<crochet ouvrant>
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
185
src/htsftp.c
185
src/htsftp.c
@@ -54,9 +54,15 @@ Please visit our Website: http://www.httrack.com
|
||||
#endif
|
||||
|
||||
// ftp mode passif
|
||||
// #if HTS_INET6==0
|
||||
#define FTP_PASV 1
|
||||
// #else
|
||||
// no passive mode for v6
|
||||
// #define FTP_PASV 0
|
||||
// #endif
|
||||
|
||||
#define FTP_DEBUG 0
|
||||
//#define FORK_DEBUG 0
|
||||
|
||||
#if USE_BEGINTHREAD
|
||||
|
||||
@@ -122,33 +128,6 @@ void launch_ftp(FTPDownloadStruct * params) {
|
||||
return 0; \
|
||||
}
|
||||
|
||||
/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
|
||||
void ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size) {
|
||||
size_t n = 0;
|
||||
|
||||
assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
|
||||
|
||||
while (src[n] != '\0' && src[n] != ':') {
|
||||
if (n < user_size - 1)
|
||||
user[n] = src[n];
|
||||
n++;
|
||||
}
|
||||
user[n < user_size ? n : user_size - 1] = '\0';
|
||||
pass[0] = '\0';
|
||||
if (src[n] == ':') { // password follows the colon
|
||||
const size_t base = n + 1;
|
||||
size_t k = 0;
|
||||
|
||||
while (&src[base + k + 1] < end && src[base + k] != '\0') {
|
||||
if (k < pass_size - 1)
|
||||
pass[k] = src[base + k];
|
||||
k++;
|
||||
}
|
||||
pass[k < pass_size ? k : pass_size - 1] = '\0';
|
||||
}
|
||||
}
|
||||
|
||||
// la véritable fonction une fois lancées les routines thread/fork
|
||||
int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
lien_back *back = pStruct->pBack;
|
||||
@@ -194,14 +173,43 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
while(*real_adr == '/')
|
||||
real_adr++; // sauter /
|
||||
if ((adr = jump_identification(real_adr)) != real_adr) { // user
|
||||
ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
|
||||
int i = -1;
|
||||
|
||||
pass[0] = '\0';
|
||||
do {
|
||||
i++;
|
||||
user[i] = real_adr[i];
|
||||
} while((real_adr[i] != ':') && (real_adr[i]));
|
||||
user[i] = '\0';
|
||||
if (real_adr[i] == ':') { // pass
|
||||
int j = -1;
|
||||
|
||||
i++; // oui on saute aussi le :
|
||||
do {
|
||||
j++;
|
||||
pass[j] = real_adr[i + j];
|
||||
} while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
|
||||
pass[j] = '\0';
|
||||
}
|
||||
}
|
||||
// Calculer RETR <nom>
|
||||
{
|
||||
char *a;
|
||||
|
||||
#if 0
|
||||
a = back->url_fil + strlen(back->url_fil) - 1;
|
||||
while((a > back->url_fil) && (*a != '/'))
|
||||
a--;
|
||||
if (*a != '/') {
|
||||
a = NULL;
|
||||
}
|
||||
#else
|
||||
a = back->url_fil;
|
||||
#endif
|
||||
if (a != NULL && *a != '\0') {
|
||||
#if 0
|
||||
a++; // sauter /
|
||||
#endif
|
||||
ftp_filename = a;
|
||||
if (strnotempty(a)) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
@@ -225,6 +233,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unexpected PORT error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
}
|
||||
@@ -258,6 +267,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"Unable to get server's address: %s", error);
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_NON_FATAL;
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
@@ -270,17 +280,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
|
||||
if (soc_ctl == INVALID_SOCKET) {
|
||||
strcpybuff(back->r.msg, "Unable to create a socket");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
|
||||
SOCaddr_initport(server, port);
|
||||
// server.sin_port = htons((unsigned short int) port);
|
||||
|
||||
// connexion (bloquante, on est en thread)
|
||||
strcpybuff(back->info, "connect");
|
||||
|
||||
if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
|
||||
strcpybuff(back->r.msg, "Unable to connect to the server");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
_HALT_FTP return 0;
|
||||
#ifdef _WIN32
|
||||
@@ -322,22 +335,64 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
// ok
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "TYPE I error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#if 0
|
||||
// --CWD--
|
||||
char *a;
|
||||
|
||||
a = back->url_fil + strlen(back->url_fil) - 1;
|
||||
while((a > back->url_fil) && (*a != '/'))
|
||||
a--;
|
||||
if (*a == '/') { // ok repéré
|
||||
char BIGSTK target[1024];
|
||||
|
||||
target[0] = '\0';
|
||||
strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
|
||||
if (strnotempty(target) == 0)
|
||||
strcatbuff(target, "/");
|
||||
strcpybuff(back->info, "cwd");
|
||||
snprintf(line, sizeof(line), "CWD %s", target);
|
||||
send_line(soc_ctl, line);
|
||||
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
|
||||
_CHECK_HALT_FTP;
|
||||
if (line[0] == '2') {
|
||||
send_line(soc_ctl, "TYPE I");
|
||||
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
|
||||
_CHECK_HALT_FTP;
|
||||
if (line[0] == '2') {
|
||||
// ok..
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "TYPE I error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unexpected ftp error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#endif
|
||||
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
|
||||
@@ -404,8 +459,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
// -- fin analyse de l'adresse IP et du port --
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
@@ -436,13 +491,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
}
|
||||
@@ -548,8 +603,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
deletesoc(soc_dat);
|
||||
soc_dat = INVALID_SOCKET;
|
||||
//
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"RETR command error: %s", linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
|
||||
linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
@@ -560,20 +616,23 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
soc_dat = INVALID_SOCKET;
|
||||
//
|
||||
strcpybuff(back->r.msg, "Unable to connect");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unable to create a socket");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"Unable to resolve IP %s: %s", adr_ip, error);
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
} // sinon on est prêts
|
||||
#else
|
||||
@@ -594,16 +653,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
//T_SOC soc_dat;
|
||||
if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
|
||||
strcpybuff(back->r.msg, "Unable to accept connection");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg),
|
||||
"RETR command error: %s", linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
@@ -613,6 +673,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
#endif
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unable to listen to a port");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#endif
|
||||
@@ -636,18 +697,21 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
int len = 1;
|
||||
int read_len = 1024;
|
||||
|
||||
//HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues
|
||||
|
||||
while((len > 0) && (!stop_ftp(back))) {
|
||||
// attendre les données
|
||||
len = 1; // pas d'erreur pour le moment
|
||||
switch (wait_socket_receive(soc_dat, timeout)) {
|
||||
case -1:
|
||||
strcpybuff(back->r.msg, "FTP read error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // fin
|
||||
break;
|
||||
case 0:
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
|
||||
timeout);
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // fin
|
||||
break;
|
||||
@@ -669,14 +733,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
*/
|
||||
strcpybuff(back->r.msg, "Write error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
len = 0; // error
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unexpected write error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else { // Erreur ou terminé
|
||||
} else { // Erreur ou terminé
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = 0;
|
||||
if (back->r.totalsize > 0
|
||||
&& back->r.size != back->r.totalsize) {
|
||||
@@ -685,6 +752,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
}
|
||||
read_len = 1024;
|
||||
//HTS_TOTAL_RECV_CHECK(read_len); // Diminuer au besoin si trop de données reçues
|
||||
}
|
||||
}
|
||||
if (back->r.fp) {
|
||||
@@ -693,6 +761,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "Unable to write file");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
@@ -708,14 +777,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
get_ftp_line(soc_ctl, line, sizeof(line), timeout);
|
||||
if (line[0] == '2') { // OK
|
||||
strcpybuff(back->r.msg, "OK");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = HTTP_OK;
|
||||
} else {
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
|
||||
linejmp(line));
|
||||
snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
strcpybuff(back->r.msg, "FTP read error");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
}
|
||||
@@ -741,6 +812,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
back->r.statuscode = HTTP_OK;
|
||||
strcpybuff(back->r.msg, "OK");
|
||||
}
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -907,11 +979,13 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
|
||||
break;
|
||||
}
|
||||
|
||||
//HTS_TOTAL_RECV_CHECK(dummy); // Diminuer au besoin si trop de données reçues
|
||||
switch (recv(soc, &b, 1, 0)) {
|
||||
//case 0: break; // pas encore --> erreur (on attend)!
|
||||
case 1:
|
||||
HTS_STAT.HTS_TOTAL_RECV += 1; // compter flux entrant
|
||||
if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
|
||||
data[i++] = b; // truncate hostile over-long reply lines
|
||||
if ((b != 10) && (b != 13))
|
||||
data[i++] = b;
|
||||
break;
|
||||
default:
|
||||
if (ptrline)
|
||||
@@ -1034,6 +1108,7 @@ int wait_socket_receive(T_SOC soc, int timeout) {
|
||||
int stop_ftp(lien_back * back) {
|
||||
if (back->stop_ftp) {
|
||||
strcpybuff(back->r.msg, "Cancelled by User");
|
||||
// back->status=STATUS_FTP_READY; // fini
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -70,11 +70,6 @@ int back_launch_ftp(FTPDownloadStruct * params);
|
||||
int run_launch_ftp(FTPDownloadStruct * params);
|
||||
int send_line(T_SOC soc, const char *data);
|
||||
int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
|
||||
/* Split a "user[:pass]@" prefix (end = jump_identification result) into
|
||||
bounded, NUL-terminated user/pass buffers, truncating to fit.
|
||||
Both sizes must be nonzero. */
|
||||
void ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size);
|
||||
T_SOC get_datasocket(char *to_send, size_t to_send_size);
|
||||
int stop_ftp(lien_back * back);
|
||||
char *linejmp(char *line);
|
||||
|
||||
@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
|
||||
configure.ac, decoupled from these). VERSION is the display form, VERSIONID
|
||||
the dotted numeric form, AFF_VERSION the short form shown in footers,
|
||||
LIB_VERSION the data/cache format generation. */
|
||||
#define HTTRACK_VERSION "3.49-11"
|
||||
#define HTTRACK_VERSIONID "3.49.11"
|
||||
#define HTTRACK_VERSION "3.49-8"
|
||||
#define HTTRACK_VERSIONID "3.49.8"
|
||||
#define HTTRACK_AFF_VERSION "3.x"
|
||||
#define HTTRACK_LIB_VERSION "2.0"
|
||||
|
||||
@@ -229,10 +229,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_DEFAULT_FOOTER \
|
||||
"<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION \
|
||||
" " HTTRACK_AFF_AUTHORS ", %s -->"
|
||||
/* Honest crawler User-Agent; no fake OS/browser to go stale. */
|
||||
#define HTS_DEFAULT_USER_AGENT \
|
||||
"Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION \
|
||||
"; +https://www.httrack.com/)"
|
||||
#define HTTRACK_WEB "http://www.httrack.com"
|
||||
#define HTS_UPDATE_WEBSITE \
|
||||
"http://www.httrack.com/" \
|
||||
@@ -251,23 +247,13 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_NOPARAM "(none)"
|
||||
#define HTS_NOPARAM2 "\"(none)\""
|
||||
|
||||
/* Boolean flag for option fields and API yes/no returns. Int-backed, not an
|
||||
enum: an enum makes C++ reject `field = 1` / `f(0)` on the exported fields
|
||||
and params. Int-sized, so the httrackp layout and the ABI are unchanged. */
|
||||
/* Boolean flag for option fields and API yes/no returns. An enum (not C bool)
|
||||
so it stays int-sized: option fields keep the httrackp layout/ABI, and a
|
||||
return type stays compatible with the int it replaces. */
|
||||
#ifndef HTS_DEF_DEFSTRUCT_hts_boolean
|
||||
#define HTS_DEF_DEFSTRUCT_hts_boolean
|
||||
|
||||
typedef int hts_boolean;
|
||||
#define HTS_FALSE 0
|
||||
#define HTS_TRUE 1
|
||||
#endif
|
||||
|
||||
#ifndef HTS_DEF_DEFSTRUCT_hts_tristate
|
||||
#define HTS_DEF_DEFSTRUCT_hts_tristate
|
||||
/* Tri-state hts_boolean: HTS_DEFAULT (-1) = "unspecified" (copy_htsopt leaves
|
||||
the target untouched); HTS_FALSE/HTS_TRUE = off/on. */
|
||||
typedef int hts_tristate;
|
||||
#define HTS_DEFAULT (-1)
|
||||
typedef enum hts_boolean { HTS_FALSE = 0, HTS_TRUE = 1 } hts_boolean;
|
||||
#endif
|
||||
|
||||
/* Larger/smaller of two values. Macros: arguments are evaluated twice. */
|
||||
@@ -412,10 +398,6 @@ typedef int T_SOC;
|
||||
/* Buffer size for a printed network address (IPv4 or IPv6, NUL included). */
|
||||
#define HTS_MAXADDRLEN 64
|
||||
|
||||
/* Max resolved addresses kept per host for connect fallback (dead IPv6 etc.).
|
||||
*/
|
||||
#define HTS_MAXADDRNUM 4
|
||||
|
||||
#ifdef _WIN32
|
||||
#else
|
||||
#define __cdecl
|
||||
|
||||
@@ -106,10 +106,10 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
|
||||
const lien_url*const lien = (const lien_url*) value;
|
||||
const char *const adr = !former ? lien->adr : lien->former_adr;
|
||||
const char *const fil = !former ? lien->fil : lien->former_fil;
|
||||
const char *const adr_norm =
|
||||
adr != NULL ? (hash->norm_host ? jump_normalized_const(adr)
|
||||
: jump_identification_const(adr))
|
||||
: NULL;
|
||||
const char *const adr_norm = adr != NULL ?
|
||||
( hash->normalized ? jump_normalized_const(adr)
|
||||
: jump_identification_const(adr) )
|
||||
: NULL;
|
||||
|
||||
// copy address
|
||||
assertf(adr_norm != NULL);
|
||||
@@ -117,18 +117,10 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
|
||||
|
||||
// copy link
|
||||
assertf(fil != NULL);
|
||||
{
|
||||
/* resolve the per-URL strip keys; strip applies even when urlhack is off */
|
||||
char BIGSTK keybuf[HTS_URLMAXSIZE];
|
||||
const char *const keys = hts_query_strip_keys(hash->strip_query, adr, fil,
|
||||
keybuf, sizeof(keybuf));
|
||||
|
||||
if (hash->norm_slash || hash->norm_query || keys != NULL) {
|
||||
fil_normalized_filtered_ex(fil, &hash->normfil[strlen(hash->normfil)],
|
||||
keys, hash->norm_slash, hash->norm_query);
|
||||
} else {
|
||||
strcpy(&hash->normfil[strlen(hash->normfil)], fil);
|
||||
}
|
||||
if (hash->normalized) {
|
||||
fil_normalized(fil, &hash->normfil[strlen(hash->normfil)]);
|
||||
} else {
|
||||
strcpy(&hash->normfil[strlen(hash->normfil)], fil);
|
||||
}
|
||||
|
||||
// hash
|
||||
@@ -140,7 +132,8 @@ static int key_adrfil_equals_generic(void *arg,
|
||||
coucal_key_const a_,
|
||||
coucal_key_const b_,
|
||||
const int former) {
|
||||
hash_struct *const hash = (hash_struct *) arg;
|
||||
hash_struct *const hash = (hash_struct*) arg;
|
||||
const int normalized = hash->normalized;
|
||||
const lien_url*const a = (const lien_url*) a_;
|
||||
const lien_url*const b = (const lien_url*) b_;
|
||||
const char *const a_adr = !former ? a->adr : a->former_adr;
|
||||
@@ -157,10 +150,10 @@ static int key_adrfil_equals_generic(void *arg,
|
||||
assertf(b_fil != NULL);
|
||||
|
||||
// skip scheme and authentication to the domain (possibly without www.)
|
||||
ja = hash->norm_host ? jump_normalized_const(a_adr)
|
||||
: jump_identification_const(a_adr);
|
||||
jb = hash->norm_host ? jump_normalized_const(b_adr)
|
||||
: jump_identification_const(b_adr);
|
||||
ja = normalized
|
||||
? jump_normalized_const(a_adr) : jump_identification_const(a_adr);
|
||||
jb = normalized
|
||||
? jump_normalized_const(b_adr) : jump_identification_const(b_adr);
|
||||
assertf(ja != NULL);
|
||||
assertf(jb != NULL);
|
||||
if (strcasecmp(ja, jb) != 0) {
|
||||
@@ -168,23 +161,12 @@ static int key_adrfil_equals_generic(void *arg,
|
||||
}
|
||||
|
||||
// now compare pathes
|
||||
{
|
||||
char BIGSTK ka[HTS_URLMAXSIZE], kb[HTS_URLMAXSIZE];
|
||||
const char *const keysa =
|
||||
hts_query_strip_keys(hash->strip_query, a_adr, a_fil, ka, sizeof(ka));
|
||||
const char *const keysb =
|
||||
hts_query_strip_keys(hash->strip_query, b_adr, b_fil, kb, sizeof(kb));
|
||||
|
||||
if (hash->norm_slash || hash->norm_query || keysa != NULL ||
|
||||
keysb != NULL) {
|
||||
fil_normalized_filtered_ex(a_fil, hash->normfil, keysa, hash->norm_slash,
|
||||
hash->norm_query);
|
||||
fil_normalized_filtered_ex(b_fil, hash->normfil2, keysb, hash->norm_slash,
|
||||
hash->norm_query);
|
||||
return strcmp(hash->normfil, hash->normfil2) == 0;
|
||||
} else {
|
||||
return strcmp(a_fil, b_fil) == 0;
|
||||
}
|
||||
if (normalized) {
|
||||
fil_normalized(a_fil, hash->normfil);
|
||||
fil_normalized(b_fil, hash->normfil2);
|
||||
return strcmp(hash->normfil, hash->normfil2) == 0;
|
||||
} else {
|
||||
return strcmp(a_fil, b_fil) == 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,17 +222,11 @@ static int key_former_adrfil_equals(void *arg,
|
||||
return key_adrfil_equals_generic(arg, a, b, 1);
|
||||
}
|
||||
|
||||
void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized) {
|
||||
void hash_init(httrackp *opt, hash_struct * hash, int normalized) {
|
||||
hash->sav = coucal_new(0);
|
||||
hash->adrfil = coucal_new(0);
|
||||
hash->former_adrfil = coucal_new(0);
|
||||
/* urlhack is the umbrella; per-feature negatives opt out of each part */
|
||||
hash->norm_host = normalized && !opt->no_www_dedup;
|
||||
hash->norm_slash = normalized && !opt->no_slash_dedup;
|
||||
hash->norm_query = normalized && !opt->no_query_dedup;
|
||||
/* snapshot the query-strip list (not owned; valid for the hash lifetime) */
|
||||
hash->strip_query =
|
||||
StringNotEmpty(opt->strip_query) ? StringBuff(opt->strip_query) : NULL;
|
||||
hash->normalized = normalized;
|
||||
|
||||
hts_set_hash_handler(hash->sav, opt);
|
||||
hts_set_hash_handler(hash->adrfil, opt);
|
||||
@@ -306,26 +282,6 @@ void hash_free(hash_struct *hash) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Test helper: do the two URLs dedupe to the same key under opt's urlhack
|
||||
flags? Exercises the live hash compare (norm_host/slash/query resolution). */
|
||||
hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
|
||||
const char *adrb, const char *filb) {
|
||||
hash_struct hash;
|
||||
lien_url la, lb;
|
||||
hts_boolean eq;
|
||||
|
||||
memset(&la, 0, sizeof(la));
|
||||
memset(&lb, 0, sizeof(lb));
|
||||
la.adr = key_duphandler(NULL, adra);
|
||||
la.fil = key_duphandler(NULL, fila);
|
||||
lb.adr = key_duphandler(NULL, adrb);
|
||||
lb.fil = key_duphandler(NULL, filb);
|
||||
hash_init(opt, &hash, opt->urlhack);
|
||||
eq = key_adrfil_equals(&hash, &la, &lb);
|
||||
hash_free(&hash);
|
||||
return eq;
|
||||
}
|
||||
|
||||
// retour: position ou -1 si non trouvé
|
||||
int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
|
||||
hash_struct_type type) {
|
||||
|
||||
@@ -51,12 +51,8 @@ typedef enum hash_struct_type {
|
||||
} hash_struct_type;
|
||||
|
||||
// tables de hachage
|
||||
void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized);
|
||||
void hash_init(httrackp *opt, hash_struct *hash, int normalized);
|
||||
void hash_free(hash_struct *hash);
|
||||
/* Test helper: HTS_TRUE if the two URLs dedupe together under opt's urlhack
|
||||
flags. */
|
||||
hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
|
||||
const char *adrb, const char *filb);
|
||||
int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
|
||||
hash_struct_type type);
|
||||
void hash_write(hash_struct * hash, size_t lpos);
|
||||
|
||||
@@ -152,6 +152,16 @@ void help_wizard(httrackp * opt) {
|
||||
#define str (buffers->str)
|
||||
#define argv (buffers->argv)
|
||||
|
||||
//char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
|
||||
//char *mainpath = (char *) malloct(256);
|
||||
//char *projname = (char *) malloct(256);
|
||||
//char *stropt = (char *) malloct(2048); // options
|
||||
//char *stropt2 = (char *) malloct(2048); // options longues
|
||||
//char *strwild = (char *) malloct(2048); // wildcards
|
||||
//char *cmd = (char *) malloct(4096);
|
||||
//char *str = (char *) malloct(256);
|
||||
//char **argv = (char **) malloct(256 * sizeof(char *));
|
||||
|
||||
//
|
||||
char *a;
|
||||
|
||||
@@ -217,6 +227,8 @@ void help_wizard(httrackp * opt) {
|
||||
strcatbuff(stropt2, str);
|
||||
strcatbuff(stropt2, projname);
|
||||
strcatbuff(stropt2, "\" ");
|
||||
// Créer si ce n'est fait un index.html 1er niveau
|
||||
make_empty_index(str);
|
||||
//
|
||||
printf("\n");
|
||||
printf("Enter URLs (separated by commas or blank spaces) :");
|
||||
@@ -337,6 +349,8 @@ void help_wizard(httrackp * opt) {
|
||||
}
|
||||
hts_main(argc, argv);
|
||||
}
|
||||
//} else {
|
||||
// help("httrack",1);
|
||||
}
|
||||
|
||||
/* Free buffers */
|
||||
@@ -441,6 +455,23 @@ void help_catchurl(const char *dest_path) {
|
||||
printf("Unable to create a temporary proxy (no remaining port)\n");
|
||||
}
|
||||
|
||||
// Créer un index.html vide
|
||||
void make_empty_index(const char *str) {
|
||||
#if 0
|
||||
if (!fexist(fconcat(str, "index.html"))) {
|
||||
FILE *fp = fopen(fconcat(str, "index.html"), "wb");
|
||||
|
||||
if (fp) {
|
||||
fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
|
||||
fprintf(fp,
|
||||
"<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
|
||||
CRLF);
|
||||
fclose(fp);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// mini-aide (h: help)
|
||||
// y
|
||||
void help(const char *app, int more) {
|
||||
@@ -490,7 +521,6 @@ void help(const char *app, int more) {
|
||||
infomsg(" EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)");
|
||||
infomsg(" AN maximum transfer rate in bytes/seconds (1000=1KB/s max)");
|
||||
infomsg(" %cN maximum number of connections/seconds (*%c10)");
|
||||
infomsg(" %G random pause of MIN[:MAX] seconds between files (e.g. %G5:10)");
|
||||
infomsg
|
||||
(" GN pause transfer if N bytes reached, and wait until lock file is deleted");
|
||||
infomsg("");
|
||||
@@ -533,7 +563,6 @@ void help(const char *app, int more) {
|
||||
(" %x do not include any password for external password protected websites (%x0 include)");
|
||||
infomsg
|
||||
(" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
|
||||
infomsg(" %g strip query keys for dedup ([host/pattern=]key1,key2,...)");
|
||||
infomsg
|
||||
(" o *generate output html file in case of error (404..) (o0 don't generate)");
|
||||
infomsg(" X *purge old files after update (X0 keep delete)");
|
||||
@@ -542,7 +571,6 @@ void help(const char *app, int more) {
|
||||
infomsg("");
|
||||
infomsg("Spider options:");
|
||||
infomsg(" bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
|
||||
infomsg(" %K load extra cookies from a Netscape cookies.txt");
|
||||
infomsg
|
||||
(" u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
|
||||
infomsg
|
||||
@@ -559,9 +587,6 @@ void help(const char *app, int more) {
|
||||
(" %s update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)");
|
||||
infomsg
|
||||
(" %u url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)");
|
||||
infomsg(" opt out of one url-hack part: --keep-www-prefix "
|
||||
"(www.foo.com<>foo.com), --keep-double-slashes (//), "
|
||||
"--keep-query-order (?b&a)");
|
||||
infomsg
|
||||
(" %A assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)");
|
||||
infomsg(" shortcut: '--assume standard' is equivalent to -%A "
|
||||
@@ -621,7 +646,9 @@ void help(const char *app, int more) {
|
||||
infomsg("");
|
||||
infomsg("Guru options: (do NOT use if possible)");
|
||||
infomsg(" #X *use optimized engine (limited memory boundary checks)");
|
||||
infomsg(" #test list engine self-tests (run one with -#test=NAME [args])");
|
||||
infomsg(" #0 filter test (-#0 '*.gif' 'www.bar.com/foo.gif')");
|
||||
infomsg(" #1 simplify test (-#1 ./foo/bar/../foobar)");
|
||||
infomsg(" #2 type test (-#2 /foo/bar.php)");
|
||||
infomsg(" #C cache list (-#C '*.com/spider*.gif'");
|
||||
infomsg(" #R cache repair (damaged cache)");
|
||||
infomsg(" #d debug parser");
|
||||
@@ -777,4 +804,7 @@ void help(const char *app, int more) {
|
||||
infomsg("[compiled: " HTS_PLATFORM_NAME "]");
|
||||
#endif
|
||||
infomsg(NULL);
|
||||
|
||||
// infomsg(" R *relative links (e.g ../link)\n");
|
||||
// infomsg(" A absolute links (e.g /www.adr/link)\n");
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ typedef struct httrackp httrackp;
|
||||
|
||||
void infomsg(const char *msg);
|
||||
void help(const char *app, int more);
|
||||
void make_empty_index(const char *str);
|
||||
void help_wizard(httrackp * opt);
|
||||
int help_query(const char *list, int def);
|
||||
void help_catchurl(const char *dest_path);
|
||||
|
||||
@@ -142,6 +142,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
char keyword[KEYW_LEN + 32];
|
||||
int i = 0;
|
||||
|
||||
//
|
||||
//int WordIndexSize = 1024;
|
||||
coucal WordIndexHash = NULL;
|
||||
FILE *tmpfp = NULL;
|
||||
|
||||
@@ -176,6 +178,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
|| (strfield2(mime, "text/css"))
|
||||
) {
|
||||
inscript = 1;
|
||||
//} else if (strfield2(mime, "text/vnd.wap.wml")) { // humm won't work in many cases
|
||||
// inscript=0;
|
||||
} else
|
||||
return 0;
|
||||
|
||||
@@ -271,6 +275,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
|
||||
// Process indexing for this page
|
||||
{
|
||||
//FILE* fp=NULL;
|
||||
//fp=fopen(concat(indexpath,"index.txt"),"ab");
|
||||
if (fp_tmpproject) {
|
||||
while(!feof(tmpfp)) {
|
||||
char line[KEYW_LEN + 32];
|
||||
@@ -280,6 +286,7 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
intptr_t e = 0;
|
||||
|
||||
if (coucal_read(WordIndexHash, line, &e)) {
|
||||
//if (e) {
|
||||
char BIGSTK savelst[HTS_URLMAXSIZE * 2];
|
||||
|
||||
e++; /* 0 means "once" */
|
||||
@@ -293,9 +300,11 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
|
||||
fprintf(fp_tmpproject, "%s %d %s\n", line,
|
||||
(int) (KEYW_SORT_MAXCOUNT - e), savelst);
|
||||
hts_primindex_size++;
|
||||
//}
|
||||
}
|
||||
}
|
||||
}
|
||||
//fclose(fp);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,6 +330,7 @@ void index_finish(const char *indexpath, int mode) {
|
||||
off_t size = fpsize(fp_tmpproject);
|
||||
|
||||
if (size > 0) {
|
||||
//FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
|
||||
if (fp_tmpproject) {
|
||||
tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
|
||||
if (tab) {
|
||||
@@ -388,6 +398,8 @@ void index_finish(const char *indexpath, int mode) {
|
||||
if (total_hit) {
|
||||
if (mode == 1) // TEXT
|
||||
fprintf(fp, "\t=%d\r\n", total_hit);
|
||||
//else // HTML
|
||||
// fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
|
||||
if ((((total_hit * 1000) / hts_primindex_words) >=
|
||||
KEYW_USELESS1K)
|
||||
|| (((total_line * 1000) / index) >=
|
||||
@@ -404,6 +416,8 @@ void index_finish(const char *indexpath, int mode) {
|
||||
if (mode == 1) // TEXT
|
||||
fprintf(fp, "\t(%d)\r\n",
|
||||
((total_hit * 1000) / hts_primindex_words));
|
||||
//else // HTML
|
||||
// fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
|
||||
}
|
||||
}
|
||||
if (mode == 1) // TEXT
|
||||
|
||||
@@ -63,9 +63,6 @@ Please visit our Website: http://www.httrack.com
|
||||
/* This file */
|
||||
#include "htsjava.h"
|
||||
|
||||
/* calloct/freet wrappers */
|
||||
#include "htssafe.h"
|
||||
|
||||
static int reverse_endian(void) {
|
||||
int endian = 1;
|
||||
|
||||
@@ -175,14 +172,15 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
#if JAVADEBUG
|
||||
printf("fopen\n");
|
||||
#endif
|
||||
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
|
||||
NULL) {
|
||||
if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
|
||||
//fprintf(stderr, "Cannot open input file.\n");
|
||||
sprintf(str->err_msg, "Unable to open file %s", file);
|
||||
return 0; // une erreur..
|
||||
}
|
||||
#if JAVADEBUG
|
||||
printf("fread\n");
|
||||
#endif
|
||||
//if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) { // pas complet..
|
||||
if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
|
||||
fclose(fpout);
|
||||
sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
|
||||
@@ -206,16 +204,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
|
||||
the file size (hostile .class ~68 MB alloc DoS). */
|
||||
if (!hts_count_fits(header.count, (LLint) fsize(file))) {
|
||||
fclose(fpout);
|
||||
sprintf(str->err_msg,
|
||||
"Invalid constant pool count %u (file len " LLintP ")",
|
||||
(unsigned) header.count, (LLint) fsize(file));
|
||||
return 0;
|
||||
}
|
||||
tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
|
||||
tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
|
||||
if (!tab) {
|
||||
sprintf(str->err_msg, "Unable to alloc %d bytes",
|
||||
(int) sizeof(RESP_STRUCT));
|
||||
@@ -241,7 +230,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
} else { // ++ une erreur est survenue!
|
||||
if (strnotempty(str->err_msg) == 0)
|
||||
strcpy(str->err_msg, "Internal readtable error");
|
||||
freet(tab);
|
||||
free(tab);
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
@@ -256,10 +245,12 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
printf("addfiles\n");
|
||||
#endif
|
||||
{
|
||||
//unsigned int acess;
|
||||
unsigned int Class;
|
||||
unsigned int SClass;
|
||||
int i;
|
||||
|
||||
//acess = readshort(fpout);
|
||||
Class = readshort(fpout);
|
||||
SClass = readshort(fpout);
|
||||
|
||||
@@ -297,7 +288,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
|
||||
#if JAVADEBUG
|
||||
printf("end\n");
|
||||
#endif
|
||||
freet(tab);
|
||||
free(tab);
|
||||
if (fpout) {
|
||||
fclose(fpout);
|
||||
fpout = NULL;
|
||||
@@ -404,9 +395,11 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
|
||||
p = &buffer[0];
|
||||
|
||||
//fflush(fp);
|
||||
trans.file_position = ftell(fp);
|
||||
length = readshort(fp);
|
||||
if (length < HTS_URLMAXSIZE) {
|
||||
// while ((length > 0) && (length<500)) {
|
||||
while(length > 0) {
|
||||
*p++ = fgetc(fp);
|
||||
|
||||
@@ -414,13 +407,17 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
}
|
||||
*p = '\0';
|
||||
|
||||
//#if JDEBUG
|
||||
// if(tris(buffer)==1) printf("%s\n ",buffer);
|
||||
// if(tris(buffer)==2) printf("%s\n ",printname(buffer));
|
||||
//#endif
|
||||
if (tris(str->opt, buffer) == 1)
|
||||
str->addLink(str, buffer); /* trans.file_position */
|
||||
else if (tris(str->opt, buffer) == 2)
|
||||
str->addLink(str, printname(rname, buffer));
|
||||
|
||||
strcpy(trans.name, buffer);
|
||||
} else { // gros pb
|
||||
} else { // gros pb
|
||||
while((length > 0) && (!feof(fp))) {
|
||||
fgetc(fp);
|
||||
length--;
|
||||
@@ -436,6 +433,7 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// printf("Type inconnue\n");
|
||||
// on arrête tout
|
||||
sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
|
||||
*error = 1;
|
||||
@@ -498,6 +496,7 @@ static char *printname(char rname[1024], char name[1024]) {
|
||||
return rname; // ""
|
||||
}
|
||||
p += 2;
|
||||
//rname=(char*)calloct(strlen(name)+8,sizeof(char));
|
||||
p1 = rname;
|
||||
for(j = 0; name[j] != '\0'; j++, p++) {
|
||||
if (*p == '/')
|
||||
|
||||
@@ -33,19 +33,15 @@ Please visit our Website: http://www.httrack.com
|
||||
#ifndef HTSJAVA_DEFH
|
||||
#define HTSJAVA_DEFH
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
|
||||
#define HTS_DEF_FWSTRUCT_JAVA_HEADER
|
||||
typedef struct JAVA_HEADER JAVA_HEADER;
|
||||
#endif
|
||||
/* 10-byte on-disk .class header image, fread() directly: fields need exact
|
||||
widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
|
||||
struct JAVA_HEADER {
|
||||
uint32_t magic;
|
||||
uint16_t minor;
|
||||
uint16_t major;
|
||||
uint16_t count;
|
||||
unsigned long int magic;
|
||||
unsigned short int minor;
|
||||
unsigned short int major;
|
||||
unsigned short int count;
|
||||
};
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
|
||||
|
||||
1023
src/htslib.c
1023
src/htslib.c
File diff suppressed because it is too large
Load Diff
56
src/htslib.h
56
src/htslib.h
@@ -147,13 +147,11 @@ struct OLD_htsblk {
|
||||
#define HTS_DEF_FWSTRUCT_t_dnscache
|
||||
typedef struct t_dnscache t_dnscache;
|
||||
#endif
|
||||
// One DNS cache record, stored as a coucal value keyed by hostname.
|
||||
struct t_dnscache {
|
||||
// resolved addresses, in resolver (RFC 6724) order; host_count==0 means the
|
||||
// name does not resolve (negative cache). host_count<=HTS_MAXADDRNUM.
|
||||
int host_count;
|
||||
size_t host_length[HTS_MAXADDRNUM]; // sockaddr length of each (16 or 28)
|
||||
char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
|
||||
struct t_dnscache *next;
|
||||
const char *iadr;
|
||||
size_t host_length; // length ; (4 or 16) ; 0 for error
|
||||
char host_addr[HTS_MAXADDRLEN];
|
||||
};
|
||||
|
||||
/* Library internal definictions */
|
||||
@@ -171,6 +169,7 @@ HTSEXT_API const char* hts_version(void);
|
||||
// fonctions unix/winsock
|
||||
int hts_read(htsblk * r, char *buff, int size);
|
||||
|
||||
//int HTS_TOTAL_RECV_CHECK(int var);
|
||||
LLint check_downloadable_bytes(int rate);
|
||||
|
||||
HTSEXT_API int hts_uninit_module(void);
|
||||
@@ -189,15 +188,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
|
||||
int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
|
||||
const char *path, char *dst, size_t dst_size);
|
||||
|
||||
//int newhttp(char* iadr,char* err=NULL);
|
||||
T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
|
||||
int waitconnect);
|
||||
/* Like newhttp(), but connect to the addr_index-th resolved address of the host
|
||||
(0-based) instead of always the first; *addr_count, if non-NULL, is set to
|
||||
the total resolved addresses. newhttp() == newhttp_addr(...,0,NULL). Used by
|
||||
the slot scheduler to try the next address when a connect fails (dead IPv6
|
||||
etc.). */
|
||||
T_SOC newhttp_addr(httrackp *opt, const char *iadr, htsblk *retour, int port,
|
||||
int waitconnect, int addr_index, int *addr_count);
|
||||
HTS_INLINE void deletehttp(htsblk * r);
|
||||
HTS_INLINE int deleteaddr(htsblk * r);
|
||||
HTS_INLINE void deletesoc(T_SOC soc);
|
||||
@@ -222,14 +215,9 @@ void treatfirstline(htsblk * retour, const char *rcvd);
|
||||
|
||||
// sous-fonctions
|
||||
LLint http_xfread1(htsblk * r, int bufl);
|
||||
/* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
|
||||
resolver order), returning the count (0 = does not resolve, negative-cached).
|
||||
Resolves once per host; later calls read the DNS cache. Must hold no lock
|
||||
(brackets opt->state.lock itself). */
|
||||
int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
|
||||
const char **error);
|
||||
HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,
|
||||
SOCaddr *const addr, const char **error);
|
||||
HTS_INLINE SOCaddr* hts_dns_resolve2(httrackp * opt, const char *iadr,
|
||||
SOCaddr *const addr,
|
||||
const char **error);
|
||||
HTS_INLINE SOCaddr* hts_dns_resolve(httrackp * opt, const char *iadr,
|
||||
SOCaddr *const addr);
|
||||
HTSEXT_API SOCaddr* hts_dns_resolve_nocache2(const char *const hostname,
|
||||
@@ -242,9 +230,8 @@ HTSEXT_API int check_hostname_dns(const char *const hostname);
|
||||
int ftp_available(void);
|
||||
|
||||
#if HTS_DNSCACHE
|
||||
/* Return opt's DNS cache hashtable (hostname -> t_dnscache record), creating it
|
||||
on first use. Records are owned by the table and freed on coucal_delete. */
|
||||
coucal hts_cache(httrackp *opt);
|
||||
void hts_cache_free(t_dnscache *const cache);
|
||||
t_dnscache *hts_cache(httrackp * opt);
|
||||
#endif
|
||||
|
||||
// outils divers
|
||||
@@ -283,9 +270,6 @@ int ishttperror(int err);
|
||||
int get_userhttptype(httrackp * opt, char *s, const char *fil);
|
||||
int give_mimext(char *s, size_t ssize, const char *st);
|
||||
|
||||
/* Advertised Accept-Encoding value (no header name/CRLF); see htslib.c. */
|
||||
const char *hts_acceptencoding(hts_boolean compressible);
|
||||
|
||||
int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename);
|
||||
int may_unknown2(httrackp * opt, const char *mime, const char *filename);
|
||||
|
||||
@@ -497,22 +481,10 @@ HTS_STATIC int strcmpnocase(const char *a, const char *b) {
|
||||
|
||||
// is this MIME an hypertext MIME (text/html), html/js-style or other script/text type?
|
||||
#define HTS_HYPERTEXT_DEFAULT_MIME "text/html"
|
||||
/* Sentinel stored when the server declared no Content-Type. It is html-ish
|
||||
for every type test (so a typeless response still parses/stores as today),
|
||||
but the naming code (wire_patches_ext) treats it as "no declared type" and
|
||||
keeps the URL extension. It rides the cache, so updates name consistently. */
|
||||
#define HTS_UNKNOWN_MIME "unknown/unknown"
|
||||
/* Map the no-declared-type sentinel back to a real type for any header or
|
||||
record we EMIT or PERSIST, so "unknown/unknown" never reaches a consumer
|
||||
(a served Content-Type, a ProxyTrack .arc record, ...). */
|
||||
#define hts_effective_mime(m) \
|
||||
(strfield2((m), HTS_UNKNOWN_MIME) ? HTS_HYPERTEXT_DEFAULT_MIME : (m))
|
||||
|
||||
#define is_html_mime_type(a) \
|
||||
((strfield2((a), "text/html") != 0) || \
|
||||
(strfield2((a), "application/xhtml+xml") != 0) || \
|
||||
(strfield2((a), HTS_UNKNOWN_MIME) != \
|
||||
0) /* no declared type: treat as html */ \
|
||||
#define is_html_mime_type(a) \
|
||||
( (strfield2((a),"text/html")!=0)\
|
||||
|| (strfield2((a),"application/xhtml+xml")!=0) \
|
||||
)
|
||||
#define is_hypertext_mime__(a) \
|
||||
( \
|
||||
|
||||
@@ -55,6 +55,12 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
|
||||
|
||||
/* >>> Put all modules variables here */
|
||||
|
||||
#if 0
|
||||
t_gzopen gzopen = NULL;
|
||||
t_gzread gzread = NULL;
|
||||
t_gzclose gzclose = NULL;
|
||||
#endif
|
||||
|
||||
int V6_is_available = HTS_INET6;
|
||||
|
||||
static char WHAT_is_available[64] = "";
|
||||
@@ -106,6 +112,18 @@ int hts_parse_externals(htsmoduleStruct * str) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
|
||||
// while(chain->next != NULL) {
|
||||
// chain = chain->next;
|
||||
// }
|
||||
// chain->next = calloct(1, sizeof(htscallbacks));
|
||||
// assertf(chain->next != NULL);
|
||||
// chain = chain->next;
|
||||
// memset(chain, 0, sizeof(*chain));
|
||||
// chain->exitFnc = exitFnc;
|
||||
// chain->moduleHandle = moduleHandle;
|
||||
//}
|
||||
|
||||
void clearCallbacks(htscallbacks * chain_);
|
||||
void clearCallbacks(htscallbacks * chain_) {
|
||||
htscallbacks *chain;
|
||||
|
||||
440
src/htsname.c
440
src/htsname.c
@@ -41,10 +41,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htstools.h"
|
||||
#include "htscharset.h"
|
||||
#include "htsencoding.h"
|
||||
#include "htssniff.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
#include <ctype.h>
|
||||
|
||||
#define ADD_STANDARD_PATH \
|
||||
@@ -74,6 +70,31 @@ static const char *hts_tbdev[] = {
|
||||
""
|
||||
};
|
||||
|
||||
#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
|
||||
int prev = opt->state._hts_in_html_parsing; \
|
||||
while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
|
||||
opt->state. _hts_in_html_parsing = 6; \
|
||||
/* Wait .. */ \
|
||||
back_wait(sback,opt,cache,0); \
|
||||
/* Transfer rate */ \
|
||||
engine_stats(); \
|
||||
/* Refresh various stats */ \
|
||||
HTS_STAT.stat_nsocket=back_nsoc(sback); \
|
||||
HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
|
||||
HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
|
||||
HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
|
||||
HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
|
||||
HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
|
||||
/* Check */ \
|
||||
{ \
|
||||
if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
|
||||
return -1; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
opt->state._hts_in_html_parsing = prev; \
|
||||
} while(0)
|
||||
|
||||
/* Strip all // */
|
||||
static void cleanDoubleSlash(char *s) {
|
||||
int i, j;
|
||||
@@ -117,191 +138,36 @@ static void cleanEndingSpaceOrDot(char *s) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Wire Content-Type vs URL extension: a patchable wire type wins over an
|
||||
unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
|
||||
(#267 guard), a declared disagreement is CONTESTED (sniffed below). */
|
||||
typedef enum wire_verdict {
|
||||
WIRE_KEEPS_EXT,
|
||||
WIRE_WINS,
|
||||
WIRE_CONTESTED
|
||||
} wire_verdict;
|
||||
|
||||
static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
|
||||
const char *file, char *urlmime,
|
||||
size_t urlmime_size) {
|
||||
if (may_unknown2(opt, wiremime, file))
|
||||
return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
|
||||
urlmime[0] = '\0';
|
||||
/* type implied by the URL extension, only when confidently known (flag 0) */
|
||||
if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
|
||||
return WIRE_WINS; /* URL ext implies no known type */
|
||||
if (strfield2(wiremime, urlmime))
|
||||
return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
|
||||
if (!is_hypertext_mime(opt, urlmime, file) &&
|
||||
strfield2(wiremime, HTS_UNKNOWN_MIME))
|
||||
return WIRE_KEEPS_EXT; /* no declared type */
|
||||
return WIRE_CONTESTED;
|
||||
}
|
||||
|
||||
/* Optional evidence for a contested wire-vs-ext verdict. */
|
||||
typedef struct sniff_src {
|
||||
struct_back *sback; /* live backing (looked up by adr/fil) */
|
||||
const lien_back *headers; /* snapshot: r.adr, else the url_sav file */
|
||||
const char *adr, *fil;
|
||||
const char *prev_save; /* previous run's save name (cache X-Save) */
|
||||
} sniff_src;
|
||||
|
||||
#if HTS_USEZLIB
|
||||
/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
|
||||
static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
|
||||
size_t out_len) {
|
||||
z_stream zs;
|
||||
size_t n = 0;
|
||||
int err;
|
||||
|
||||
memset(&zs, 0, sizeof(zs));
|
||||
if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
|
||||
return 0;
|
||||
zs.next_in = (const Bytef *) in;
|
||||
zs.avail_in = (uInt) in_len;
|
||||
zs.next_out = (Bytef *) out;
|
||||
zs.avail_out = (uInt) out_len;
|
||||
err = inflate(&zs, Z_SYNC_FLUSH);
|
||||
if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
|
||||
n = out_len - zs.avail_out;
|
||||
inflateEnd(&zs);
|
||||
return n;
|
||||
}
|
||||
#endif
|
||||
|
||||
static size_t sniff_read_head(const char *path, void *buf, size_t len) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
|
||||
size_t n = 0;
|
||||
|
||||
if (fp != NULL) {
|
||||
n = fread(buf, 1, len, fp);
|
||||
fclose(fp);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or
|
||||
tmpfile for a compressed stream); inflated so the sniff sees the final body.
|
||||
*/
|
||||
static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
|
||||
const htsblk *const r = &slot->r;
|
||||
size_t n = 0;
|
||||
|
||||
if (r->adr != NULL && r->size > 0) {
|
||||
n = (size_t) r->size < len ? (size_t) r->size : len;
|
||||
memcpy(buf, r->adr, n);
|
||||
} else {
|
||||
if (r->out != NULL)
|
||||
fflush(r->out);
|
||||
if (slot->url_sav[0] != '\0')
|
||||
n = sniff_read_head(slot->url_sav, buf, len);
|
||||
if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0')
|
||||
n = sniff_read_head(slot->tmpfile, buf, len);
|
||||
}
|
||||
if (n > 0 && r->compressed) {
|
||||
#if HTS_USEZLIB
|
||||
unsigned char raw[HTS_SNIFF_LEN];
|
||||
|
||||
if (n > sizeof(raw))
|
||||
n = sizeof(raw);
|
||||
memcpy(raw, buf, n);
|
||||
n = sniff_inflate_head(raw, n, buf, len);
|
||||
#else
|
||||
n = 0;
|
||||
#endif
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Up to len leading body bytes; 0 when unavailable, and always in
|
||||
non-delayed mode (its HEAD-probe first run couldn't sniff either). */
|
||||
static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf,
|
||||
size_t len) {
|
||||
size_t n = 0;
|
||||
|
||||
if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE)
|
||||
return 0;
|
||||
/* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */
|
||||
if (src->sback != NULL && src->adr != NULL && src->fil != NULL) {
|
||||
const int b = back_index(opt, src->sback, src->adr, src->fil, NULL);
|
||||
|
||||
if (b >= 0)
|
||||
n = sniff_slot_head(&src->sback->lnk[b], buf, len);
|
||||
}
|
||||
if (n == 0 && src->headers != NULL)
|
||||
n = sniff_slot_head(src->headers, buf, len);
|
||||
return n;
|
||||
}
|
||||
|
||||
/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */
|
||||
static int wire_patches_ext(httrackp *opt, const sniff_src *src,
|
||||
const char *wiremime, const char *file) {
|
||||
/* Should the wire Content-Type override the URL's own extension when naming the
|
||||
saved file? True when the type is patchable (may_unknown2) and either the URL
|
||||
extension implies no specific type or the server declared a disagreeing one.
|
||||
A URL extension mapping to a specific non-HTML type is kept only when the
|
||||
server sent NO Content-Type (the #267 mangle guard): a typeless .png stays
|
||||
.png, but a .pdf explicitly served as text/html is named .html. */
|
||||
static int wire_patches_ext(httrackp *opt, const char *wiremime,
|
||||
const char *file, int contenttype_given) {
|
||||
char urlmime[256];
|
||||
|
||||
switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
|
||||
case WIRE_KEEPS_EXT:
|
||||
if (may_unknown2(opt, wiremime, file))
|
||||
return 0; /* type kept verbatim (keep-list / bogus-multiple) */
|
||||
urlmime[0] = '\0';
|
||||
/* type implied by the URL extension, only when confidently known (flag 0) */
|
||||
if (!get_httptype_sized(opt, urlmime, sizeof(urlmime), file, 0))
|
||||
return 1; /* URL ext implies no known type: trust the wire type */
|
||||
if (strfield2(wiremime, urlmime))
|
||||
return 0; /* wire agrees with the ext: keep it (no .htm->.html churn) */
|
||||
/* wire disagrees with a specific non-HTML URL ext. Keep the ext only when
|
||||
the server sent NO Content-Type: a missing type is defaulted to text/html
|
||||
upstream and must not clobber e.g. a .png. An explicitly declared type is
|
||||
trusted, so a binary-looking URL that really serves HTML (login/error
|
||||
interstitial, soft-404) is named .html instead of kept as .pdf/.jpg. */
|
||||
if (!is_hypertext_mime(opt, urlmime, file) && !contenttype_given)
|
||||
return 0;
|
||||
case WIRE_WINS:
|
||||
return 1;
|
||||
case WIRE_CONTESTED:
|
||||
break;
|
||||
}
|
||||
if (src != NULL) {
|
||||
if (hts_sniff_mime_known(urlmime)) {
|
||||
unsigned char head[HTS_SNIFF_LEN];
|
||||
const size_t n = sniff_body_head(opt, src, head, sizeof(head));
|
||||
|
||||
if (n > 0)
|
||||
return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1;
|
||||
}
|
||||
/* no bytes: reproduce the previous run's verdict (cached X-Save name) */
|
||||
if (src->prev_save != NULL && src->prev_save[0] != '\0') {
|
||||
char prevmime[256];
|
||||
|
||||
prevmime[0] = '\0';
|
||||
if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save,
|
||||
0) &&
|
||||
strfield2(prevmime, urlmime))
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime,
|
||||
const char *file) {
|
||||
char urlmime[256];
|
||||
|
||||
return wiremime != NULL && strnotempty(wiremime) &&
|
||||
wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) ==
|
||||
WIRE_CONTESTED &&
|
||||
hts_sniff_mime_known(urlmime);
|
||||
}
|
||||
|
||||
/* Wire-metadata name change: a Content-Disposition filename wins (returns 2),
|
||||
else the declared type's ext when wire_patches_ext() allows (returns 1),
|
||||
else 0. ext receives the new extension or replacement filename. */
|
||||
static int resolve_extension(httrackp *opt, const sniff_src *src,
|
||||
const char *cdispo, const char *contenttype,
|
||||
const char *fil, char *ext, size_t ext_size) {
|
||||
if (strnotempty(cdispo)) {
|
||||
strlcpybuff(ext, cdispo, ext_size);
|
||||
return 2;
|
||||
}
|
||||
if (wire_patches_ext(opt, src, contenttype, fil) &&
|
||||
give_mimext(ext, ext_size, contenttype))
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Build the local save name (save) from adr/fil; renames on collision
|
||||
// (e.g. INDEX.HTML vs index.html).
|
||||
// forme le nom du fichier à sauver (save) à partir de fil et adr
|
||||
// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
|
||||
int url_savename(lien_adrfilsave *const afs,
|
||||
lien_adrfil *const former,
|
||||
const char *referer_adr, const char *referer_fil,
|
||||
@@ -331,13 +197,6 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// copy of fil, used for lookups (see urlhack)
|
||||
const char *normadr = adr;
|
||||
const char *normfil = fil_complete;
|
||||
/* query keys to strip for this URL (NULL = none); decoupled from urlhack */
|
||||
char BIGSTK stripkeys[HTS_URLMAXSIZE];
|
||||
const char *const strip =
|
||||
StringNotEmpty(opt->strip_query)
|
||||
? hts_query_strip_keys(StringBuff(opt->strip_query), adr,
|
||||
fil_complete, stripkeys, sizeof(stripkeys))
|
||||
: NULL;
|
||||
const char *const print_adr = jump_protocol_const(adr);
|
||||
const char *start_pos = NULL, *nom_pos = NULL, *dot_pos = NULL; // Position nom et point
|
||||
|
||||
@@ -370,13 +229,9 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// www-42.foo.com -> foo.com
|
||||
// foo.com/bar//foobar -> foo.com/bar/foobar
|
||||
if (opt->urlhack) {
|
||||
// dedup-lookup key; honor the per-feature negatives like htshash.c so
|
||||
// distinct URLs keep distinct savenames (else keep normadr = adr)
|
||||
if (!opt->no_www_dedup)
|
||||
normadr = adr_normalized_sized(adr, normadr_, sizeof(normadr_));
|
||||
normfil =
|
||||
fil_normalized_filtered_ex(fil_complete, normfil_, strip,
|
||||
!opt->no_slash_dedup, !opt->no_query_dedup);
|
||||
// copy of adr (without protocol), used for lookups (see urlhack)
|
||||
normadr = adr_normalized_sized(adr, normadr_, sizeof(normadr_));
|
||||
normfil = fil_normalized(fil_complete, normfil_);
|
||||
} else {
|
||||
if (link_has_authority(adr_complete)) { // https or other protocols : in "http/" subfolder
|
||||
char *pos = strchr(adr_complete, ':');
|
||||
@@ -389,11 +244,6 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
normadr = normadr_;
|
||||
}
|
||||
}
|
||||
// strip still applies with urlhack off (host left untouched); no // or
|
||||
// query-sort here, to match the hash key (norm_slash/norm_query are 0 when
|
||||
// urlhack is off) so a URL is looked up under the key it was stored with
|
||||
if (strip != NULL)
|
||||
normfil = fil_normalized_filtered_ex(fil_complete, normfil_, strip, 0, 0);
|
||||
}
|
||||
|
||||
// à afficher sans ftp://
|
||||
@@ -538,30 +388,46 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
// si option check_type activée
|
||||
if (is_html < 0 && opt->check_type && !ext_chg) {
|
||||
int ishtest = 0;
|
||||
|
||||
if (protocol != PROTOCOL_FILE
|
||||
&& protocol != PROTOCOL_FTP
|
||||
) {
|
||||
// tester type avec requète HEAD si on ne connait pas le type du fichier
|
||||
if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/'))) // slash doit être html?
|
||||
if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
|
||||
ishtml(opt, fil) < 0) { // unsure whether it's html or a file
|
||||
(ishtest = ishtml(opt, fil)) <
|
||||
0) { // unsure whether it's html or a file
|
||||
// lire dans le cache
|
||||
char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
|
||||
htsblk r;
|
||||
htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
|
||||
|
||||
previous_save[0] = '\0';
|
||||
r = cache_read_including_broken(opt, cache, adr, fil,
|
||||
previous_save); // test uniquement
|
||||
if (r.statuscode != -1) { // pas d'erreur de lecture cache
|
||||
char s[32];
|
||||
|
||||
if (r.statuscode != -1) { // cache entry read OK
|
||||
s[0] = '\0';
|
||||
hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",
|
||||
adr_complete, fil_complete);
|
||||
if (!HTTP_IS_REDIRECT(r.statuscode)) {
|
||||
const sniff_src src = {sback, NULL, adr, fil, previous_save};
|
||||
|
||||
ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype,
|
||||
fil, ext, sizeof(ext));
|
||||
if (strnotempty(r.cdispo)) { /* filename given */
|
||||
ext_chg = 2; /* change filename */
|
||||
strcpybuff(ext, r.cdispo);
|
||||
} else if (wire_patches_ext(opt, r.contenttype, fil,
|
||||
r.contenttype_given)) {
|
||||
if (give_mimext(s, sizeof(s),
|
||||
r.contenttype)) { // recognized extension
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef DEFAULT_BIN_EXT
|
||||
// no extension and potentially bogus
|
||||
else if (ishtest == -2) {
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, DEFAULT_BIN_EXT + 1);
|
||||
}
|
||||
#endif
|
||||
//
|
||||
} else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
|
||||
is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
|
||||
Lookup mimetype not only by extension,
|
||||
@@ -585,13 +451,23 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
// fail later
|
||||
else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
|
||||
!opt->state.stop) {
|
||||
// Check if the file is ready in backing.
|
||||
// Check if the file is ready in backing. We basically take the same logic as later.
|
||||
// FIXME: we should cleanup and factorize this unholy mess
|
||||
if (headers != NULL && headers->status >= 0 && !is_redirect) {
|
||||
const sniff_src src = {sback, headers, adr, fil, NULL};
|
||||
|
||||
ext_chg = resolve_extension(opt, &src, headers->r.cdispo,
|
||||
headers->r.contenttype,
|
||||
headers->url_fil, ext, sizeof(ext));
|
||||
if (strnotempty(headers->r.cdispo)) { /* filename given */
|
||||
ext_chg = 2; /* change filename */
|
||||
strcpybuff(ext, headers->r.cdispo);
|
||||
} else if (wire_patches_ext(opt, headers->r.contenttype,
|
||||
headers->url_fil,
|
||||
headers->r.contenttype_given)) {
|
||||
char s[16];
|
||||
if (give_mimext(
|
||||
s, sizeof(s),
|
||||
headers->r.contenttype)) { // recognized extension
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (mime_type != NULL) {
|
||||
ext[0] = '\0';
|
||||
@@ -609,6 +485,13 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (!may_unknown2(opt, mime_type, fil)) {
|
||||
ext_chg = 1;
|
||||
}
|
||||
#ifdef DEFAULT_BIN_EXT
|
||||
// no extension and potentially bogus
|
||||
else if (ishtml(opt, fil) == -2) {
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, DEFAULT_BIN_EXT + 1);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
ext_chg = 0;
|
||||
}
|
||||
@@ -627,10 +510,11 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
int has_been_moved = 0;
|
||||
lien_adrfil current;
|
||||
|
||||
/* Wait for an available test slot, honoring the connection limits
|
||||
/* Ensure we don't use too many sockets by using a "testing" one
|
||||
If we have only 1 simultaneous connection authorized, wait for pending download
|
||||
Wait for an available slot
|
||||
*/
|
||||
if (!hts_wait_available_socket(sback, opt, cache, ptr))
|
||||
return -1;
|
||||
URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
|
||||
|
||||
/* Rock'in */
|
||||
current.adr[0] = current.fil[0] = '\0';
|
||||
@@ -660,11 +544,24 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (ptr >= 0) {
|
||||
back_fillmax(sback, opt, cache, ptr, numero_passe);
|
||||
}
|
||||
if (!hts_loop_tick(sback, opt, b, ptr)) {
|
||||
// on est obligé d'appeler le shell pour le refresh..
|
||||
// Transfer rate
|
||||
engine_stats();
|
||||
|
||||
// Refresh various stats
|
||||
HTS_STAT.stat_nsocket = back_nsoc(sback);
|
||||
HTS_STAT.stat_errors = fspc(opt, NULL, "error");
|
||||
HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
|
||||
HTS_STAT.stat_infos = fspc(opt, NULL, "info");
|
||||
HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
|
||||
HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
|
||||
|
||||
if (!RUN_CALLBACK7
|
||||
(opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
|
||||
(int) (time_local() - HTS_STAT.stat_timestart),
|
||||
&HTS_STAT)) {
|
||||
return -1;
|
||||
} else if (opt->state._hts_cancel ||
|
||||
!back_checkmirror(
|
||||
opt)) { // cancel level 2 or 1 (cancel parsing)
|
||||
} else if (opt->state._hts_cancel || !back_checkmirror(opt)) { // cancel 2 ou 1 (cancel parsing)
|
||||
back_delete(opt, cache, sback, b); // cancel test
|
||||
stop_looping = 1;
|
||||
}
|
||||
@@ -729,9 +626,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
"Loop with HEAD request (during prefetch) at %s%s",
|
||||
current.adr, current.fil);
|
||||
}
|
||||
if (!hts_wait_available_socket(sback, opt,
|
||||
cache, ptr))
|
||||
return -1;
|
||||
// Ajouter
|
||||
URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
|
||||
if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) { // OK
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"(during prefetch) %s (%d) to link %s at %s%s",
|
||||
@@ -779,16 +675,36 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
if (!has_been_moved) {
|
||||
if (back[b].r.statuscode != -10) { // erreur
|
||||
if (strnotempty(back[b].r.contenttype) == 0)
|
||||
strcpybuff(back[b].r.contenttype,
|
||||
HTS_UNKNOWN_MIME); // no declared type
|
||||
strcpybuff(back[b].r.contenttype, "text/html"); // message d'erreur en html
|
||||
// Finalement on, renvoie un erreur, pour ne toucher à rien dans le code
|
||||
// libérer emplacement backing
|
||||
}
|
||||
|
||||
// no error: change the type?
|
||||
ext_chg = resolve_extension(
|
||||
opt, NULL, back[b].r.cdispo, back[b].r.contenttype,
|
||||
back[b].url_fil, ext, sizeof(ext));
|
||||
{ // pas d'erreur, changer type?
|
||||
char s[16];
|
||||
|
||||
s[0] = '\0';
|
||||
if (strnotempty(back[b].r.cdispo)) { /* filename given */
|
||||
ext_chg = 2; /* change filename */
|
||||
strcpybuff(ext, back[b].r.cdispo);
|
||||
} else if (wire_patches_ext(opt, back[b].r.contenttype,
|
||||
back[b].url_fil,
|
||||
back[b].r.contenttype_given)) {
|
||||
if (give_mimext(
|
||||
s, sizeof(s),
|
||||
back[b].r.contenttype)) { // recognized extension
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, s);
|
||||
}
|
||||
}
|
||||
#ifdef DEFAULT_BIN_EXT
|
||||
// no extension and potentially bogus
|
||||
else if (ishtest == -2) {
|
||||
ext_chg = 1;
|
||||
strcpybuff(ext, DEFAULT_BIN_EXT + 1);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
// FIN Si non déplacé, forcer type?
|
||||
|
||||
@@ -845,9 +761,9 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
strcatbuff(fil, DEFAULT_HTML); // nommer page par défaut (à priori ici html depuis un proxy http)
|
||||
}
|
||||
}
|
||||
// Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
|
||||
// depending on the resolved type.
|
||||
if (ext_chg && !opt->no_type_change) {
|
||||
// Changer extension?
|
||||
// par exemple, php3 sera sauvé en html, cgi en html ou gif, xbm etc.. selon les cas
|
||||
if (ext_chg && !opt->no_type_change) { // changer ext
|
||||
char *a = fil + strlen(fil) - 1;
|
||||
|
||||
if ((opt->debug > 1) && (opt->log != NULL)) {
|
||||
@@ -859,19 +775,11 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
adr_complete, fil_complete, ext);
|
||||
}
|
||||
if (ext_chg == 1) {
|
||||
// Cut the old extension only when it is empty (a bare trailing dot), the
|
||||
// new one, or a recognized one; an unknown trailing ".token" (e.g.
|
||||
// /article-1.884291, #115) is part of the name, not an extension.
|
||||
const char *const old_ext = get_ext(catbuff, sizeof(catbuff), fil);
|
||||
const int known_ext = !*old_ext || strfield2(old_ext, ext) ||
|
||||
is_knowntype(opt, fil) || is_dyntype(old_ext) ||
|
||||
ishtml_ext(old_ext) != -1;
|
||||
|
||||
while((a > fil) && (*a != '.') && (*a != '/'))
|
||||
a--;
|
||||
if (*a == '.' && known_ext)
|
||||
*a = '\0'; // cut
|
||||
strcatbuff(fil, "."); // re-add the dot
|
||||
if (*a == '.')
|
||||
*a = '\0'; // couper
|
||||
strcatbuff(fil, "."); // recopier point
|
||||
} else {
|
||||
while((a > fil) && (*a != '/'))
|
||||
a--;
|
||||
@@ -879,7 +787,7 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
a++;
|
||||
*a = '\0';
|
||||
}
|
||||
strcatbuff(fil, ext); // append ext/name
|
||||
strcatbuff(fil, ext); // copier ext/nom
|
||||
}
|
||||
// Rechercher premier / et dernier .
|
||||
{
|
||||
@@ -1332,7 +1240,8 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
while((a > afs->save) && (*a != '.') && (*a != '/'))
|
||||
a--;
|
||||
if (*a != '.') { // agh pas de point
|
||||
if (*a != '.') { // agh pas de point
|
||||
//strcatbuff(save,".none"); // a éviter
|
||||
strcatbuff(afs->save, ".html"); // préférable!
|
||||
hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
|
||||
adr_complete, fil_complete, afs->save);
|
||||
@@ -1470,6 +1379,17 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
fil_simplifie(afs->save);
|
||||
|
||||
/* convert name to UTF-8 ? Note: already done while parsing. */
|
||||
//if (charset != NULL && charset[0] != '\0') {
|
||||
// char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
|
||||
|
||||
// if (s != NULL) {
|
||||
// hts_log_print(opt, LOG_DEBUG,
|
||||
// "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
|
||||
// save, s, charset);
|
||||
// strcpybuff(save, s);
|
||||
// free(s);
|
||||
// }
|
||||
//}
|
||||
|
||||
/* callback */
|
||||
RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
|
||||
@@ -1617,6 +1537,10 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
|
||||
int sameFil;
|
||||
|
||||
// NO - URL hack is only for stripping // and www.
|
||||
//if (opt->urlhack != 0)
|
||||
// sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
|
||||
//else
|
||||
sameFil = (strcmp(heap(i)->fil, normfil) == 0);
|
||||
if (sameAdr && sameFil) { // ok c'est le même lien, adresse déja définie
|
||||
/* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
|
||||
@@ -1682,13 +1606,17 @@ int url_savename(lien_adrfilsave *const afs,
|
||||
|
||||
strcpybuff(afs->save, tempo);
|
||||
|
||||
//printf("switched: %s\n",save);
|
||||
|
||||
} // if
|
||||
}
|
||||
#if DEBUG_SAVENAME
|
||||
printf("\nEnd search, %s\n", fil_complete);
|
||||
#endif
|
||||
} while (!nom_ok);
|
||||
} while(!nom_ok);
|
||||
|
||||
}
|
||||
//printf("'%s' %s %s\n",save,adr,fil);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1794,10 +1722,10 @@ char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
|
||||
StringBuff(opt->path_log), digest_filename);
|
||||
}
|
||||
|
||||
/* remove refname if any; HTS_TRUE if it was removed */
|
||||
hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
|
||||
const char *fil) {
|
||||
/* remove refname if any */
|
||||
void url_savename_refname_remove(httrackp * opt, const char *adr,
|
||||
const char *fil) {
|
||||
char *filename = url_savename_refname_fullpath(opt, adr, fil);
|
||||
|
||||
return UNLINK(filename) == 0 ? HTS_TRUE : HTS_FALSE;
|
||||
(void) UNLINK(filename);
|
||||
}
|
||||
|
||||
@@ -100,15 +100,12 @@ void standard_name(char *b, size_t bsize, const char *dot_pos,
|
||||
const char *nom_pos, const char *fil_complete,
|
||||
int short_ver);
|
||||
void url_savename_addstr(char *d, const char *s);
|
||||
/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */
|
||||
int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file);
|
||||
char *url_md5(char *digest_buffer, const char *fil_complete);
|
||||
void url_savename_refname(const char *adr, const char *fil, char *filename);
|
||||
char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
|
||||
const char *fil);
|
||||
/* Remove the temp-ref for (adr,fil); HTS_TRUE if it was removed. */
|
||||
hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
|
||||
const char *fil);
|
||||
void url_savename_refname_remove(httrackp * opt, const char *adr,
|
||||
const char *fil);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
16
src/htsnet.h
16
src/htsnet.h
@@ -304,22 +304,6 @@ static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
|
||||
/** Length type for socket APIs (getsockname, accept, ...). */
|
||||
typedef socklen_t SOClen;
|
||||
|
||||
#if HTS_INET6 != 0
|
||||
/** Resolver backend: getaddrinfo/freeaddrinfo as a swappable pair, so the
|
||||
self-test can script DNS answers (families, multiplicity, errors)
|
||||
in-process. The free function must match its getaddrinfo (a fake allocates
|
||||
its own chain), hence the pair. */
|
||||
typedef struct hts_resolver_backend {
|
||||
int (*getaddrinfo)(const char *node, const char *service,
|
||||
const struct addrinfo *hints, struct addrinfo **res);
|
||||
void (*freeaddrinfo)(struct addrinfo *res);
|
||||
} hts_resolver_backend;
|
||||
|
||||
/** Install a resolver backend for the process; NULL restores the libc default.
|
||||
Test-only seam, not thread-safe; callers must serialize against resolves. */
|
||||
void hts_dns_set_resolver_backend(const hts_resolver_backend *backend);
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
33
src/htsopt.h
33
src/htsopt.h
@@ -105,6 +105,7 @@ typedef struct htsfilters htsfilters;
|
||||
struct htsfilters {
|
||||
char ***filters; /**< pointer to the +/-pattern filter array */
|
||||
int *filptr; /**< pointer to the current filter count */
|
||||
// int* filter_max;
|
||||
};
|
||||
|
||||
/* User callbacks chain */
|
||||
@@ -240,7 +241,7 @@ struct htsoptstate {
|
||||
char *userhttptype;
|
||||
int verif_backblue_done; /**< backblue.gif/fade.gif already emitted */
|
||||
int verif_external_status;
|
||||
coucal dns_cache; /**< DNS resolution cache: hostname -> t_dnscache record */
|
||||
t_dnscache *dns_cache; /**< DNS resolution cache */
|
||||
int dns_cache_nthreads; /**< number of in-flight DNS resolver threads */
|
||||
/* HTML parsing state */
|
||||
char _hts_errmsg[HTS_CDLMAXSIZE + 256]; /**< last engine error message */
|
||||
@@ -427,11 +428,11 @@ struct httrackp {
|
||||
LLint maxfile_html; /**< max bytes per HTML file */
|
||||
int maxsoc; /**< max simultaneous sockets (-cN) */
|
||||
LLint fragment; /**< split site after this many bytes */
|
||||
hts_tristate
|
||||
hts_boolean
|
||||
nearlink; /**< also fetch images/data adjacent to a page but off-site */
|
||||
hts_boolean makeindex; /**< build a top-level index.html */
|
||||
hts_boolean kindex; /**< build a keyword index */
|
||||
hts_tristate delete_old; /**< delete locally obsolete files after update */
|
||||
hts_boolean delete_old; /**< delete locally obsolete files after update */
|
||||
int timeout; /**< connection timeout in seconds */
|
||||
int rateout; /**< minimum transfer rate (bytes/s) before abort */
|
||||
int maxtime; /**< max total mirror duration in seconds */
|
||||
@@ -439,6 +440,7 @@ struct httrackp {
|
||||
float maxconn; /**< max connections per second */
|
||||
int waittime; /**< scheduled start time (wall-clock seconds) */
|
||||
hts_cachemode cache; /**< cache generation mode */
|
||||
// int aff_progress; // progress bar
|
||||
hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
|
||||
t_proxy proxy; /**< proxy configuration */
|
||||
hts_savename_83
|
||||
@@ -463,13 +465,13 @@ struct httrackp {
|
||||
hts_boolean maketrack; /**< maintain an operations-statistics log */
|
||||
int parsejava; /**< Java/JS parsing mode; see htsparsejava_flags */
|
||||
int hostcontrol; /**< ban slow/timing-out hosts; see hts_hostcontrol bits */
|
||||
hts_tristate errpage; /**< generate an error page on 404 and similar */
|
||||
hts_boolean errpage; /**< generate an error page on 404 and similar */
|
||||
hts_boolean
|
||||
check_type; /**< probe unknown-type links (cgi/asp/dir) and follow moves
|
||||
*/
|
||||
hts_boolean all_in_cache; /**< keep all retrieved data in the cache */
|
||||
hts_robots robots; /**< robots.txt handling level */
|
||||
hts_tristate external; /**< render external links as error pages */
|
||||
hts_boolean external; /**< render external links as error pages */
|
||||
hts_boolean passprivacy; /**< strip passwords from external links */
|
||||
hts_boolean includequery; /**< include the query string in saved names */
|
||||
hts_boolean mirror_first_page; /**< only mirror the links of the first page */
|
||||
@@ -483,13 +485,14 @@ struct httrackp {
|
||||
hts_boolean sizehack; /**< treat same-size response as "updated" */
|
||||
hts_boolean urlhack; // force "url normalization" to avoid loops
|
||||
hts_boolean tolerant; /**< accept an incorrect Content-Length */
|
||||
hts_tristate
|
||||
hts_boolean
|
||||
parseall; /**< parse aggressively, including unknown tags with links */
|
||||
hts_boolean parsedebug; /**< parser debug mode */
|
||||
hts_boolean norecatch; /**< do not re-fetch files the user deleted locally */
|
||||
hts_verbosedisplay verbosedisplay; /**< animated text progress display */
|
||||
String footer; /**< footer/info line injected into pages */
|
||||
int maxcache; /**< in-memory cache backing limit (bytes) */
|
||||
// int maxcache_anticipate; // maximum links to anticipate (upper bound)
|
||||
hts_boolean ftp_proxy; /**< use the HTTP proxy for FTP too */
|
||||
String filelist; /**< file listing URLs to include */
|
||||
String urllist; /**< file listing filters to include */
|
||||
@@ -526,16 +529,6 @@ struct httrackp {
|
||||
htslibhandles libHandles; /**< loaded external module handles */
|
||||
//
|
||||
htsoptstate state; /**< embedded live engine state */
|
||||
String strip_query; /**< query keys to drop when deduping URLs (-strip-query);
|
||||
appended at the tail to keep field offsets stable */
|
||||
hts_boolean
|
||||
no_www_dedup; /**< with urlhack, keep www.host distinct from host */
|
||||
hts_boolean no_slash_dedup; /**< with urlhack, keep redundant // in paths */
|
||||
hts_boolean no_query_dedup; /**< with urlhack, keep query-argument order */
|
||||
String cookies_file; /**< extra Netscape cookies.txt to preload
|
||||
(--cookies-file) */
|
||||
int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
|
||||
int pause_max_ms; /**< inter-file pause upper bound, ms */
|
||||
};
|
||||
|
||||
/* Running statistics for a mirror. */
|
||||
@@ -546,6 +539,8 @@ typedef struct hts_stat_struct hts_stat_struct;
|
||||
struct hts_stat_struct {
|
||||
LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
|
||||
LLint stat_bytes; /**< total bytes written to disk */
|
||||
// int HTS_TOTAL_RECV_STATE; // status: 0 ok 1: slow down a little 2: slow
|
||||
// down 3: a lot
|
||||
TStamp stat_timestart; /**< mirror start time */
|
||||
//
|
||||
LLint total_packed; /**< compressed bytes received (on the wire) */
|
||||
@@ -656,6 +651,8 @@ struct htsblk {
|
||||
int debugid; /**< connection debug id */
|
||||
/* */
|
||||
htsrequest req; /**< parameters used for the request */
|
||||
/* a Content-Type header was received (else contenttype holds a default) */
|
||||
hts_boolean contenttype_given;
|
||||
/*char digest[32+2]; // md5 digest generated by the engine ("" if none) */
|
||||
};
|
||||
|
||||
@@ -677,6 +674,7 @@ struct lien_url {
|
||||
int depth; /**< remaining allowed depth; >0 strong, 0 weak */
|
||||
int pass2; /**< second-pass marker; -1 means handled in background */
|
||||
char link_import; /**< imported after a move; skip the usual up/down rules */
|
||||
// int moved; // pointer to moved
|
||||
int retry; /**< remaining retries */
|
||||
int testmode; /**< test only: send just a HEAD */
|
||||
};
|
||||
@@ -721,6 +719,9 @@ struct lien_back {
|
||||
LLint chunk_size; /**< size of the chunk being loaded */
|
||||
LLint chunk_blocksize; /**< data size declared by the chunk */
|
||||
LLint compressed_size; /**< compressed size (stats only) */
|
||||
//
|
||||
// int links_index; // to access liens[links_index]
|
||||
//
|
||||
char info[256]; /**< status text, e.g. for FTP */
|
||||
int stop_ftp; /**< stop flag for FTP */
|
||||
int finalized; /**< finalized (memory optimization) */
|
||||
|
||||
889
src/htsparse.c
889
src/htsparse.c
File diff suppressed because it is too large
Load Diff
@@ -66,6 +66,7 @@ struct htsmoduleStructExtended {
|
||||
char ***filters_;
|
||||
robots_wizard *robots_;
|
||||
hash_struct *hash_;
|
||||
//int *lien_max_;
|
||||
|
||||
/* Base & codebase */
|
||||
char *base;
|
||||
@@ -115,19 +116,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
|
||||
int hts_mirror_check_moved(htsmoduleStruct * str,
|
||||
htsmoduleStructExtended * stre);
|
||||
|
||||
/*
|
||||
Non-zero if a redirect (cur_adr,cur_fil)->(moved_adr,moved_fil) saves to the
|
||||
same local file, so it must be followed rather than turned into a
|
||||
self-pointing "moved" stub (#159). Mirrors the savename: scheme+userinfo
|
||||
stripped, www kept (www dedup is the crawl layer's job), path
|
||||
slash/query-normalized per the URL-hack flags. Not hash_url_equals: that keys
|
||||
on the dedup hash, which folds www and never collapses http<->https.
|
||||
*/
|
||||
hts_boolean hts_redirect_same_savefile(httrackp *opt, const char *cur_adr,
|
||||
const char *cur_fil,
|
||||
const char *moved_adr,
|
||||
const char *moved_fil);
|
||||
|
||||
/*
|
||||
Process user intercations: pause, add link, delete link..
|
||||
*/
|
||||
@@ -174,4 +162,27 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
|
||||
/* Apply changes */ \
|
||||
* str->ptr_ = ptr
|
||||
|
||||
#define WAIT_FOR_AVAILABLE_SOCKET() do { \
|
||||
int prev = opt->state._hts_in_html_parsing; \
|
||||
while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
|
||||
opt->state._hts_in_html_parsing = 6; \
|
||||
/* Wait .. */ \
|
||||
back_wait(sback,opt,cache,0); \
|
||||
/* Transfer rate */ \
|
||||
engine_stats(); \
|
||||
/* Refresh various stats */ \
|
||||
HTS_STAT.stat_nsocket=back_nsoc(sback); \
|
||||
HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
|
||||
HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
|
||||
HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
|
||||
HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
|
||||
HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
|
||||
/* Check */ \
|
||||
if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, -1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
|
||||
return -1; \
|
||||
} \
|
||||
} \
|
||||
opt->state._hts_in_html_parsing = prev; \
|
||||
} while(0)
|
||||
|
||||
#endif
|
||||
|
||||
167
src/htsrobots.c
167
src/htsrobots.c
@@ -44,84 +44,28 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
// -- robots --
|
||||
|
||||
/* RFC 9309 path-prefix match; '*' any run, '$' anchors end; linear. */
|
||||
static hts_boolean robots_pattern_match(const char *pattern, const char *path) {
|
||||
size_t patlen = strlen(pattern);
|
||||
hts_boolean anchored = HTS_FALSE;
|
||||
const char *p, *pend, *s;
|
||||
const char *star = NULL, *star_s = NULL;
|
||||
|
||||
if (patlen > 0 && pattern[patlen - 1] == '$') {
|
||||
anchored = HTS_TRUE;
|
||||
patlen--;
|
||||
}
|
||||
p = pattern;
|
||||
pend = pattern + patlen;
|
||||
s = path;
|
||||
while (*s != '\0') {
|
||||
if (p == pend) {
|
||||
if (!anchored)
|
||||
return HTS_TRUE; // prefix matched
|
||||
if (star != NULL) { // anchored: '*' must eat the rest
|
||||
p = star + 1;
|
||||
s = ++star_s;
|
||||
continue;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
if (*p == '*') {
|
||||
star = p++;
|
||||
star_s = s;
|
||||
} else if (*p == *s) {
|
||||
p++;
|
||||
s++;
|
||||
} else if (star != NULL) {
|
||||
p = star + 1;
|
||||
s = ++star_s;
|
||||
} else {
|
||||
return HTS_FALSE;
|
||||
}
|
||||
}
|
||||
while (p < pend && *p == '*')
|
||||
p++;
|
||||
return (p == pend) ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
// fil="" : vérifier si règle déja enregistrée
|
||||
int checkrobots(robots_wizard * robots, const char *adr, const char *fil) {
|
||||
while(robots) {
|
||||
if (strfield2(robots->adr, adr)) {
|
||||
if (fil[0]) {
|
||||
/* RFC 9309: longest pattern wins, Allow beats Disallow on ties. */
|
||||
int ptr = 0;
|
||||
char line[HTS_ROBOTS_TOKEN_SIZE];
|
||||
size_t toklen = strlen(robots->token);
|
||||
size_t best_len = 0;
|
||||
hts_boolean matched = HTS_FALSE;
|
||||
hts_boolean best_allow = HTS_FALSE;
|
||||
char line[250];
|
||||
|
||||
while (ptr < (int) toklen) {
|
||||
ptr += binput(robots->token + ptr, line, sizeof(line) - 1);
|
||||
if (line[0] != 'A' && line[0] != 'D')
|
||||
continue;
|
||||
{
|
||||
const hts_boolean is_allow =
|
||||
(line[0] == 'A') ? HTS_TRUE : HTS_FALSE;
|
||||
const char *pat = line + 1;
|
||||
|
||||
if (robots_pattern_match(pat, fil)) {
|
||||
const size_t len = strlen(pat);
|
||||
|
||||
if (!matched || len > best_len || (len == best_len && is_allow)) {
|
||||
matched = HTS_TRUE;
|
||||
best_len = len;
|
||||
best_allow = is_allow;
|
||||
if (strnotempty(robots->token)) {
|
||||
do {
|
||||
ptr += binput(robots->token + ptr, line, 200);
|
||||
if (line[0] == '/') { // absolu
|
||||
if (strfield(fil, line)) { // commence avec ligne
|
||||
return -1; // interdit
|
||||
}
|
||||
} else { // relatif
|
||||
if (strstrcase(fil, line)) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} while((strnotempty(line)) && (ptr < (int) strlen(robots->token)));
|
||||
}
|
||||
if (matched && !best_allow)
|
||||
return -1; // forbidden
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
@@ -130,93 +74,6 @@ int checkrobots(robots_wizard * robots, const char *adr, const char *fil) {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Append "<marker><pattern>\n" to the bounded rule blob if it fits. */
|
||||
static void robots_blob_add(char *blob, size_t blobsize, char marker,
|
||||
const char *pat) {
|
||||
const size_t used = strlen(blob);
|
||||
const size_t need = strlen(pat) + 2; // marker + '\n'
|
||||
|
||||
if (need < blobsize - used) { // overflow-safe: used <= blobsize-1
|
||||
blob[used] = marker;
|
||||
blob[used + 1] = '\0';
|
||||
strlcatbuff(blob, pat, blobsize);
|
||||
strlcatbuff(blob, "\n", blobsize);
|
||||
}
|
||||
}
|
||||
|
||||
void robots_parse(robots_wizard *robots, const char *adr, const char *body,
|
||||
size_t bodysize, char *info, size_t infosize,
|
||||
hts_boolean keep_root_disallow) {
|
||||
size_t bptr = 0;
|
||||
int record = 0;
|
||||
char BIGSTK line[1024];
|
||||
char BIGSTK blob[HTS_ROBOTS_TOKEN_SIZE];
|
||||
|
||||
blob[0] = '\0';
|
||||
if (info != NULL && infosize > 0)
|
||||
info[0] = '\0';
|
||||
#if DEBUG_ROBOTS
|
||||
printf("robots.txt dump:\n%s\n", body);
|
||||
#endif
|
||||
while (bptr < bodysize) {
|
||||
char *comm;
|
||||
int llen;
|
||||
|
||||
bptr += binput(body + bptr, line, sizeof(line) - 2);
|
||||
comm = strchr(line, '#'); // strip comment
|
||||
if (comm != NULL)
|
||||
*comm = '\0';
|
||||
llen = (int) strlen(line); // strip trailing spaces
|
||||
while (llen > 0 && is_realspace(line[llen - 1])) {
|
||||
line[llen - 1] = '\0';
|
||||
llen--;
|
||||
}
|
||||
if (strfield(line, "user-agent:")) {
|
||||
char *a = line + 11;
|
||||
|
||||
while (is_realspace(*a))
|
||||
a++;
|
||||
if (*a == '*') {
|
||||
if (record != 2)
|
||||
record = 1; // generic group applies to us
|
||||
} else if (strfield(a, "httrack") || strfield(a, "winhttrack") ||
|
||||
strfield(a, "webhttrack")) {
|
||||
blob[0] = '\0'; // explicit group: restart capture
|
||||
if (info != NULL && infosize > 0)
|
||||
info[0] = '\0';
|
||||
record = 2; // locked to the httrack group
|
||||
} else
|
||||
record = 0;
|
||||
} else if (record) {
|
||||
hts_boolean is_allow = strfield(line, "allow:");
|
||||
hts_boolean is_disallow = !is_allow && strfield(line, "disallow:");
|
||||
|
||||
if (is_allow || is_disallow) {
|
||||
char *a = line + (is_allow ? 6 : 9);
|
||||
|
||||
while (is_realspace(*a))
|
||||
a++;
|
||||
if (strnotempty(a)) {
|
||||
if (is_disallow && !keep_root_disallow && strcmp(a, "/") == 0) {
|
||||
// dropped: site-wide disallow ignored by option
|
||||
} else {
|
||||
robots_blob_add(blob, sizeof(blob), is_allow ? 'A' : 'D', a);
|
||||
if (is_disallow && info != NULL &&
|
||||
strlen(a) + 2 < infosize - strlen(info)) {
|
||||
if (strnotempty(info))
|
||||
strlcatbuff(info, ", ", infosize);
|
||||
strlcatbuff(info, a, infosize);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (strnotempty(blob))
|
||||
checkrobots_set(robots, adr, blob);
|
||||
}
|
||||
|
||||
int checkrobots_set(robots_wizard * robots, const char *adr, const char *data) {
|
||||
if (((int) strlen(adr)) >= sizeof(robots->adr) - 2)
|
||||
return 0;
|
||||
|
||||
@@ -39,27 +39,17 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_DEF_FWSTRUCT_robots_wizard
|
||||
typedef struct robots_wizard robots_wizard;
|
||||
#endif
|
||||
|
||||
/* Per-host blob: one rule per line, first byte 'A'/'D' then path pattern. */
|
||||
#define HTS_ROBOTS_TOKEN_SIZE 4096
|
||||
|
||||
struct robots_wizard {
|
||||
char adr[128];
|
||||
char token[HTS_ROBOTS_TOKEN_SIZE];
|
||||
char token[4096];
|
||||
struct robots_wizard *next;
|
||||
};
|
||||
|
||||
/* Library internal definictions */
|
||||
#ifdef HTS_INTERNAL_BYTECODE
|
||||
/* -1 if `fil` disallowed for `adr` (RFC 9309); empty: -1 if rules exist. */
|
||||
int checkrobots(robots_wizard * robots, const char *adr, const char *fil);
|
||||
void checkrobots_free(robots_wizard * robots);
|
||||
int checkrobots_set(robots_wizard * robots, const char *adr, const char *data);
|
||||
/* Parse robots.txt `body` for `adr`, storing the HTTrack group's rules; `info`
|
||||
gets a disallow summary, `keep_root_disallow` FALSE drops "Disallow: /". */
|
||||
void robots_parse(robots_wizard *robots, const char *adr, const char *body,
|
||||
size_t bodysize, char *info, size_t infosize,
|
||||
hts_boolean keep_root_disallow);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -456,13 +456,6 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
|
||||
return b->buf;
|
||||
}
|
||||
|
||||
/** True if 'count' records of >= 1 byte each fit in 'available' bytes; guards
|
||||
an attacker-controlled count driving a large allocation. */
|
||||
static HTS_INLINE HTS_UNUSED hts_boolean hts_count_fits(size_t count,
|
||||
LLint available) {
|
||||
return (available >= 0 && (LLint) count <= available) ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
/* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
|
||||
added bounds checking. freet() also NULLs the freed pointer and tolerates
|
||||
NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the
|
||||
|
||||
2196
src/htsselftest.c
2196
src/htsselftest.c
File diff suppressed because it is too large
Load Diff
@@ -1,52 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: htsselftest.h */
|
||||
/* named dispatch for the hidden engine self-tests */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSSELFTEST_DEFH
|
||||
#define HTSSELFTEST_DEFH
|
||||
|
||||
#ifdef HTS_INTERNAL_BYTECODE
|
||||
|
||||
#ifndef HTS_DEF_FWSTRUCT_httrackp
|
||||
#define HTS_DEF_FWSTRUCT_httrackp
|
||||
typedef struct httrackp httrackp;
|
||||
#endif
|
||||
|
||||
/* Run engine self-test `name` over the positional args argv[0..argc-1], or list
|
||||
the available tests when name is NULL, empty, or "list". Prints the result;
|
||||
returns the process exit code (0 == success). The caller owns option cleanup.
|
||||
Reached through the hidden `httrack -#test[=NAME ...]` subcommand. */
|
||||
int hts_selftest(httrackp *opt, const char *name, int argc, char **argv);
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -265,6 +265,7 @@ T_SOC smallserver_init(int *port, char *adr) {
|
||||
SOCaddr_initport(server, *port);
|
||||
if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
|
||||
if (listen(soc, 10) >= 0) {
|
||||
// SOCaddr_inetntoa(adr, 128, server2);
|
||||
strcpy(adr, h_loc);
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
@@ -357,12 +358,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
{NULL, 0}
|
||||
};
|
||||
initStrElt initStr[] = {
|
||||
{"user", HTS_DEFAULT_USER_AGENT},
|
||||
{"footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x "
|
||||
"[XR&CO'2014], %s -->"},
|
||||
{"url2",
|
||||
"+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
|
||||
{NULL, NULL}};
|
||||
{"user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"},
|
||||
{"footer",
|
||||
"<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2014], %s -->"},
|
||||
{"url2", "+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
|
||||
{NULL, NULL}
|
||||
};
|
||||
int i = 0;
|
||||
|
||||
for(i = 0; initInt[i].name; i++) {
|
||||
@@ -438,6 +439,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
meth = 10;
|
||||
} else {
|
||||
#ifdef _DEBUG
|
||||
// assert(FALSE);
|
||||
#endif
|
||||
}
|
||||
if (meth) {
|
||||
@@ -1358,6 +1360,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
|
||||
StringCat(headers, error_hdr);
|
||||
StringCat(output, error);
|
||||
//assert(file == NULL);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1385,10 +1388,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
|
||||
!= StringLength(output)))
|
||||
) {
|
||||
#ifdef _DEBUG
|
||||
//assert(FALSE);
|
||||
#endif
|
||||
}
|
||||
} else {
|
||||
#ifdef _DEBUG
|
||||
// assert(FALSE);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -1535,7 +1540,8 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
|
||||
} while(strnotempty(test));
|
||||
}
|
||||
|
||||
if (!strnotempty(test)) { // éviter doublons
|
||||
if (!strnotempty(test)) { // éviter doublons
|
||||
// conv_printf(key,key);
|
||||
const size_t len = strlen(intkey);
|
||||
char *const buff = (char *) malloc(len + 1);
|
||||
|
||||
@@ -1626,6 +1632,8 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
|
||||
intkey = "";
|
||||
} else {
|
||||
if (loops > 0) {
|
||||
//err_msg += intkey;
|
||||
//err_msg += " ";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1732,7 +1740,13 @@ static void LANG_DELETE(void) {
|
||||
}
|
||||
|
||||
// sélection de la langue
|
||||
static void LANG_INIT(const char *path) { LANG_T(path, 0); }
|
||||
static void LANG_INIT(const char *path) {
|
||||
//CWinApp* pApp = AfxGetApp();
|
||||
//if (pApp) {
|
||||
/* pApp->GetProfileInt("Language","IntId",0); */
|
||||
LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
|
||||
//}
|
||||
}
|
||||
|
||||
static int LANG_T(const char *path, int l) {
|
||||
if (l >= 0) {
|
||||
|
||||
352
src/htssniff.c
352
src/htssniff.c
@@ -1,352 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2017 Xavier Roche and other contributors
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Important notes:
|
||||
|
||||
- We hereby ask people using this source NOT to use it in purpose of grabbing
|
||||
emails addresses, or collecting any other private information on persons.
|
||||
This would disgrace our work, and spoil the many hours we spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: MIME magic-byte consistency checks */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#include "htssniff.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "htslib.h"
|
||||
|
||||
/* One magic rule: `len` bytes at `off` confirm `mime`. */
|
||||
typedef struct sniff_magic {
|
||||
const char *mime;
|
||||
unsigned short off;
|
||||
unsigned char len;
|
||||
const char *bytes;
|
||||
} sniff_magic;
|
||||
|
||||
/* Direction is mime -> magic (verify a claim, never classify); types with
|
||||
no reliable magic (plain text, css, js..) are deliberately absent. Patterns
|
||||
follow the WHATWG MIME Sniffing Standard tables where it defines them
|
||||
(https://mimesniff.spec.whatwg.org/); the rest covers httrack's wider MIME
|
||||
set. Spec-only types absent from our MIME tables (EOT, font/collection)
|
||||
are omitted as unreachable. */
|
||||
static const sniff_magic sniff_table[] = {
|
||||
/* images */
|
||||
{"image/jpeg", 0, 3, "\xff\xd8\xff"},
|
||||
{"image/pipeg", 0, 3, "\xff\xd8\xff"},
|
||||
{"image/pjpeg", 0, 3, "\xff\xd8\xff"},
|
||||
{"image/png", 0, 8, "\x89PNG\r\n\x1a\n"},
|
||||
{"image/gif", 0, 6, "GIF87a"},
|
||||
{"image/gif", 0, 6, "GIF89a"},
|
||||
{"image/bmp", 0, 2, "BM"},
|
||||
{"image/tiff", 0, 4, "II*\0"},
|
||||
{"image/tiff", 0, 4, "MM\0*"},
|
||||
{"image/x-icon", 0, 4, "\0\0\1\0"},
|
||||
{"image/x-icon", 0, 4, "\0\0\2\0"}, /* Windows cursor, per the spec */
|
||||
{"image/x-portable-bitmap", 0, 2, "P1"},
|
||||
{"image/x-portable-bitmap", 0, 2, "P4"},
|
||||
{"image/x-portable-pixmap", 0, 2, "P3"},
|
||||
{"image/x-portable-pixmap", 0, 2, "P6"},
|
||||
{"image/x-xpixmap", 0, 9, "/* XPM */"},
|
||||
{"image/x-xbitmap", 0, 7, "#define"},
|
||||
{"image/x-rgb", 0, 2, "\x01\xda"},
|
||||
{"image/x-cmu-raster", 0, 4, "\xf1\x00\x40\xbb"},
|
||||
/* audio */
|
||||
{"audio/mpeg", 0, 3, "ID3"},
|
||||
{"audio/basic", 0, 4, ".snd"},
|
||||
{"audio/mid", 0, 8, "MThd\0\0\0\6"},
|
||||
{"audio/midi", 0, 8, "MThd\0\0\0\6"},
|
||||
{"audio/x-pn-realaudio", 0, 4, ".ra\xfd"},
|
||||
{"audio/x-pn-realaudio", 0, 4, ".RMF"},
|
||||
{"audio/x-pn-realaudio-plugin", 0, 4, ".ra\xfd"},
|
||||
{"audio/x-pn-realaudio-plugin", 0, 4, ".RMF"},
|
||||
{"audio/flac", 0, 4, "fLaC"},
|
||||
{"audio/aac", 0, 4, "ADIF"},
|
||||
/* video */
|
||||
{"video/mpeg", 0, 4, "\x00\x00\x01\xba"},
|
||||
{"video/mpeg", 0, 4, "\x00\x00\x01\xb3"},
|
||||
{"video/x-sgi-movie", 0, 4, "MOVI"},
|
||||
/* archives / compression */
|
||||
{"application/x-gzip", 0, 3, "\x1f\x8b\x08"},
|
||||
{"multipart/x-gzip", 0, 3, "\x1f\x8b\x08"},
|
||||
{"application/x-compressed", 0, 3, "\x1f\x8b\x08"},
|
||||
{"application/x-compress", 0, 2, "\x1f\x9d"},
|
||||
{"application/x-bzip2", 0, 3, "BZh"},
|
||||
{"application/x-7z-compressed", 0, 6, "7z\xbc\xaf\x27\x1c"},
|
||||
/* 6-byte prefix common to RAR4 (spec) and RAR5 */
|
||||
{"application/x-rar-compressed", 0, 6, "Rar!\x1a\x07"},
|
||||
{"application/zstd", 0, 4, "\x28\xb5\x2f\xfd"},
|
||||
{"application/arj", 0, 2, "\x60\xea"},
|
||||
{"application/x-cpio", 0, 6, "070701"},
|
||||
{"application/x-cpio", 0, 6, "070707"},
|
||||
{"application/x-cpio", 0, 2, "\xc7\x71"},
|
||||
{"application/x-sv4cpio", 0, 6, "070701"},
|
||||
{"application/x-sv4crc", 0, 6, "070702"},
|
||||
{"application/x-stuffit", 0, 8, "StuffIt "},
|
||||
{"application/x-stuffit", 0, 4, "SIT!"},
|
||||
{"application/mac-binhex40", 0, 10, "(This file"},
|
||||
/* documents */
|
||||
{"application/pdf", 0, 5, "%PDF-"},
|
||||
{"application/postscript", 0, 2, "%!"},
|
||||
{"application/rtf", 0, 5, "{\\rtf"},
|
||||
{"application/x-dvi", 0, 2, "\xf7\x02"},
|
||||
{"application/x-hdf", 0, 4, "\x0e\x03\x13\x01"},
|
||||
{"application/x-hdf", 0, 8, "\x89HDF\r\n\x1a\n"},
|
||||
{"application/x-netcdf", 0, 4, "CDF\x01"},
|
||||
{"application/x-netcdf", 0, 4, "CDF\x02"},
|
||||
{"application/x-msaccess", 0, 19, "\0\1\0\0Standard Jet DB"},
|
||||
/* fonts */
|
||||
{"font/woff", 0, 4, "wOFF"},
|
||||
{"font/woff2", 0, 4, "wOF2"},
|
||||
{"font/ttf", 0, 4, "\0\1\0\0"},
|
||||
{"font/ttf", 0, 4, "true"},
|
||||
{"font/otf", 0, 4, "OTTO"},
|
||||
/* misc */
|
||||
{"application/x-shockwave-flash", 0, 3, "FWS"},
|
||||
{"application/x-shockwave-flash", 0, 3, "CWS"},
|
||||
{"application/x-shockwave-flash", 0, 3, "ZWS"},
|
||||
{"application/futuresplash", 0, 3, "FWS"},
|
||||
{"application/x-director", 0, 4, "RIFX"},
|
||||
{"application/x-director", 0, 4, "XFIR"},
|
||||
{"application/x-java-vm", 0, 4, "\xca\xfe\xba\xbe"},
|
||||
{"application/wasm", 0, 4, "\0asm"},
|
||||
{"application/x-msmetafile", 0, 4, "\xd7\xcd\xc6\x9a"},
|
||||
{"application/x-msmetafile", 0, 4, "\x01\x00\x09\x00"},
|
||||
{"application/x-x509-ca-cert", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs12", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs7-mime", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs7-signature", 0, 2, "\x30\x82"},
|
||||
{"application/x-pkcs7-certificates", 0, 2, "\x30\x82"},
|
||||
{"x-world/x-vrml", 0, 5, "#VRML"},
|
||||
{"application/x-bittorrent", 0, 11, "d8:announce"},
|
||||
{"drawing/x-dwf", 0, 4, "(DWF"},
|
||||
{"application/acad", 0, 4, "AC10"},
|
||||
{NULL, 0, 0, NULL}};
|
||||
|
||||
/* MIME families sharing a container magic */
|
||||
static const char *const zip_mimes[] = {
|
||||
"application/zip", "application/x-zip-compressed", "multipart/x-zip", NULL};
|
||||
static const char *const zip_mime_prefixes[] = {
|
||||
"application/vnd.openxmlformats-officedocument.",
|
||||
"application/vnd.oasis.opendocument.", NULL};
|
||||
static const char *const ole_mimes[] = {"application/msword",
|
||||
"application/excel",
|
||||
"application/vnd.ms-excel",
|
||||
"application/powerpoint",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.ms-project",
|
||||
"application/vnd.ms-works",
|
||||
"application/x-msmoney",
|
||||
"application/x-mspublisher",
|
||||
NULL};
|
||||
static const char *const tar_mimes[] = {
|
||||
"application/x-tar", "application/x-ustar", "application/x-gtar", NULL};
|
||||
static const char *const ogg_mimes[] = {"application/ogg", "audio/ogg",
|
||||
"video/ogg", "audio/opus", NULL};
|
||||
static const char *const ebml_mimes[] = {"video/webm", "audio/webm", NULL};
|
||||
/* ISO-BMFF, any 'ftyp' brand: containers overlap too much to split */
|
||||
static const char *const bmff_mimes[] = {"video/mp4", "audio/mp4",
|
||||
"video/quicktime", NULL};
|
||||
static const char *const avif_mimes[] = {"image/avif", NULL};
|
||||
static const char *const heic_mimes[] = {"image/heic", NULL};
|
||||
static const char *const asf_mimes[] = {"video/x-ms-asf", "video/x-ms-wmv",
|
||||
"video/x-la-asf", NULL};
|
||||
static const char *const xml_mimes[] = {"application/xml", "text/xml",
|
||||
"image/svg+xml", "image/svg-xml", NULL};
|
||||
static const char *const svg_mimes[] = {"image/svg+xml", "image/svg-xml", NULL};
|
||||
static const char *const html_mimes[] = {"text/html", NULL};
|
||||
static const char *const pem_mimes[] = {
|
||||
"application/x-x509-ca-cert", "application/x-pkcs7-certificates",
|
||||
"application/x-pkcs7-mime", "application/x-pkcs7-signature", NULL};
|
||||
|
||||
static hts_boolean mime_in(const char *const *list, const char *mime) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; list[i] != NULL; i++)
|
||||
if (strfield2(list[i], mime))
|
||||
return HTS_TRUE;
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
static hts_boolean mime_in_prefix(const char *const *list, const char *mime) {
|
||||
size_t i;
|
||||
|
||||
for (i = 0; list[i] != NULL; i++)
|
||||
if (strfield(mime, list[i]))
|
||||
return HTS_TRUE;
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
static hts_boolean has_bytes(const unsigned char *d, size_t n, size_t off,
|
||||
const char *bytes, size_t len) {
|
||||
/* overflow-safe: untrusted n alone on one side */
|
||||
return n >= off && len <= n - off && memcmp(d + off, bytes, len) == 0
|
||||
? HTS_TRUE
|
||||
: HTS_FALSE;
|
||||
}
|
||||
|
||||
static unsigned char ascii_lower(unsigned char c) {
|
||||
return c >= 'A' && c <= 'Z' ? (unsigned char) (c + 32) : c;
|
||||
}
|
||||
|
||||
/* Case-insensitive text prefix after an optional UTF-8 BOM and whitespace. */
|
||||
static hts_boolean has_text_prefix(const unsigned char *d, size_t n,
|
||||
const char *prefix) {
|
||||
const size_t len = strlen(prefix);
|
||||
size_t i, k;
|
||||
|
||||
i = n >= 3 && memcmp(d, "\xef\xbb\xbf", 3) == 0 ? 3 : 0;
|
||||
while (i < n && (d[i] == ' ' || d[i] == '\t' || d[i] == '\r' || d[i] == '\n'))
|
||||
i++;
|
||||
if (len > n - i) /* i <= n from the loop above */
|
||||
return HTS_FALSE;
|
||||
for (k = 0; k < len; k++)
|
||||
if (ascii_lower(d[i + k]) != ascii_lower((unsigned char) prefix[k]))
|
||||
return HTS_FALSE;
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
typedef enum sniff_op {
|
||||
SNIFF_QUERY_KNOWN, /* is any rule defined for this MIME? */
|
||||
SNIFF_QUERY_MATCH /* do the bytes confirm this MIME? */
|
||||
} sniff_op;
|
||||
|
||||
/* Single walk for both queries so the rule set can't drift apart. */
|
||||
static hts_boolean sniff_eval(sniff_op op, const unsigned char *d, size_t n,
|
||||
const char *mime) {
|
||||
size_t i;
|
||||
|
||||
/* KNOWN short-circuits; MATCH tests the magic */
|
||||
#define SNIFF_RULE(cond) \
|
||||
do { \
|
||||
if (op == SNIFF_QUERY_KNOWN) \
|
||||
return HTS_TRUE; \
|
||||
if (cond) \
|
||||
return HTS_TRUE; \
|
||||
} while (0)
|
||||
|
||||
for (i = 0; sniff_table[i].mime != NULL; i++) {
|
||||
if (strfield2(sniff_table[i].mime, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, sniff_table[i].off, sniff_table[i].bytes,
|
||||
sniff_table[i].len));
|
||||
}
|
||||
}
|
||||
if (mime_in(zip_mimes, mime) || mime_in_prefix(zip_mime_prefixes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "PK\3\4", 4) ||
|
||||
has_bytes(d, n, 0, "PK\5\6", 4));
|
||||
}
|
||||
if (mime_in(ole_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", 8));
|
||||
}
|
||||
if (mime_in(tar_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 257, "ustar", 5));
|
||||
}
|
||||
if (mime_in(ogg_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "OggS\0", 5));
|
||||
}
|
||||
if (mime_in(ebml_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "\x1a\x45\xdf\xa3", 4));
|
||||
}
|
||||
if (mime_in(bmff_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 4, "ftyp", 4));
|
||||
}
|
||||
if (mime_in(avif_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 4, "ftypavif", 8) ||
|
||||
has_bytes(d, n, 4, "ftypavis", 8));
|
||||
}
|
||||
if (mime_in(heic_mimes, mime)) {
|
||||
SNIFF_RULE(
|
||||
has_bytes(d, n, 4, "ftyphei", 7) || has_bytes(d, n, 4, "ftyphev", 7) ||
|
||||
has_bytes(d, n, 4, "ftypmif1", 8) || has_bytes(d, n, 4, "ftypmsf1", 8));
|
||||
}
|
||||
if (mime_in(asf_mimes, mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "\x30\x26\xb2\x75\x8e\x66\xcf\x11", 8));
|
||||
}
|
||||
if (strfield2("audio/x-wav", mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "WAVE", 4));
|
||||
}
|
||||
if (strfield2("video/x-msvideo", mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "AVI ", 4));
|
||||
}
|
||||
if (strfield2("image/webp", mime)) {
|
||||
SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) &&
|
||||
has_bytes(d, n, 8, "WEBPVP", 6));
|
||||
}
|
||||
if (strfield2("image/x-portable-anymap", mime)) {
|
||||
SNIFF_RULE(n >= 2 && d[0] == 'P' && d[1] >= '1' && d[1] <= '6');
|
||||
}
|
||||
if (strfield2("audio/x-aiff", mime)) {
|
||||
SNIFF_RULE(
|
||||
has_bytes(d, n, 0, "FORM", 4) &&
|
||||
(has_bytes(d, n, 8, "AIFF", 4) || has_bytes(d, n, 8, "AIFC", 4)));
|
||||
}
|
||||
if (strfield2("audio/mpeg", mime)) {
|
||||
/* MPEG audio frame sync (11 bits), valid layer and bitrate fields */
|
||||
SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xe0) == 0xe0 &&
|
||||
(d[1] & 0x06) != 0);
|
||||
}
|
||||
if (strfield2("audio/aac", mime)) {
|
||||
/* ADTS sync */
|
||||
SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xf6) == 0xf0);
|
||||
}
|
||||
if (strfield2("video/mp2t", mime)) {
|
||||
SNIFF_RULE(n >= 1 && d[0] == 0x47 && (n <= 188 || d[188] == 0x47));
|
||||
}
|
||||
if (mime_in(xml_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "<?xml"));
|
||||
}
|
||||
if (mime_in(svg_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "<svg") ||
|
||||
has_text_prefix(d, n, "<!DOCTYPE svg"));
|
||||
}
|
||||
if (mime_in(html_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "<!DOCTYPE") ||
|
||||
has_text_prefix(d, n, "<html") ||
|
||||
has_text_prefix(d, n, "<head"));
|
||||
}
|
||||
if (mime_in(pem_mimes, mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "-----BEGIN"));
|
||||
}
|
||||
if (strfield2("audio/x-mpegurl", mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "#EXTM3U"));
|
||||
}
|
||||
if (strfield2("text/x-vcard", mime)) {
|
||||
SNIFF_RULE(has_text_prefix(d, n, "BEGIN:VCARD"));
|
||||
}
|
||||
#undef SNIFF_RULE
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
hts_boolean hts_sniff_mime_known(const char *mime) {
|
||||
if (mime == NULL || *mime == '\0')
|
||||
return HTS_FALSE;
|
||||
return sniff_eval(SNIFF_QUERY_KNOWN, NULL, 0, mime);
|
||||
}
|
||||
|
||||
hts_boolean hts_sniff_mime_consistent(const void *data, size_t size,
|
||||
const char *mime) {
|
||||
if (data == NULL || size == 0 || mime == NULL || *mime == '\0')
|
||||
return HTS_FALSE;
|
||||
return sniff_eval(SNIFF_QUERY_MATCH, (const unsigned char *) data, size,
|
||||
mime);
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 1998-2017 Xavier Roche and other contributors
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Important notes:
|
||||
|
||||
- We hereby ask people using this source NOT to use it in purpose of grabbing
|
||||
emails addresses, or collecting any other private information on persons.
|
||||
This would disgrace our work, and spoil the many hours we spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: MIME magic-byte consistency checks */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTSSNIFF_DEFH
|
||||
#define HTSSNIFF_DEFH
|
||||
|
||||
#include <stddef.h>
|
||||
#include "htsglobal.h"
|
||||
|
||||
/* Leading-body window read to arbitrate a wire/extension MIME conflict. */
|
||||
#define HTS_SNIFF_LEN 512
|
||||
|
||||
/* Can a magic rule ever confirm this MIME? (whether sniffing is worth it) */
|
||||
hts_boolean hts_sniff_mime_known(const char *mime);
|
||||
|
||||
/* TRUE when the leading body bytes are consistent with the claimed MIME;
|
||||
FALSE on unknown MIME, unknown magic, or too-short data (fail-safe). */
|
||||
hts_boolean hts_sniff_mime_consistent(const void *data, size_t size,
|
||||
const char *mime);
|
||||
|
||||
#endif
|
||||
@@ -121,6 +121,9 @@ struct String {
|
||||
/** Byte at POS (read/write). No bounds check; POS must be < StringLength. **/
|
||||
#define StringSubRW(BLK, POS) (StringBuffRW(BLK)[POS])
|
||||
|
||||
/** Subcharacter (read/write) **/
|
||||
#define StringSubRW(BLK, POS) (StringBuffRW(BLK)[POS])
|
||||
|
||||
/** Byte POS positions from the end (read). POS==1 is the last byte. **/
|
||||
#define StringRight(BLK, POS) (StringBuff(BLK)[StringLength(BLK) - POS])
|
||||
|
||||
@@ -188,9 +191,8 @@ HTS_STATIC char *StringBuffN_(String *blk, int size) {
|
||||
asserts SIZE fits the existing content; does not (re)allocate. **/
|
||||
#define StringSetLength(BLK, SIZE) \
|
||||
do { \
|
||||
const int len__ = (SIZE); /* signed: negative means strlen(buffer_) */ \
|
||||
if (len__ >= 0) { \
|
||||
(BLK).length_ = len__; \
|
||||
if (SIZE >= 0) { \
|
||||
(BLK).length_ = SIZE; \
|
||||
} else { \
|
||||
(BLK).length_ = strlen((BLK).buffer_); \
|
||||
} \
|
||||
@@ -306,11 +308,10 @@ HTS_STATIC void StringAttach(String *blk, char **str) {
|
||||
#define StringCatN(BLK, STR, SIZE) \
|
||||
do { \
|
||||
const char *str__ = (STR); \
|
||||
const size_t usize__ = (SIZE); \
|
||||
if (str__ != NULL) { \
|
||||
size_t size__ = strlen(str__); \
|
||||
if (size__ > usize__) { \
|
||||
size__ = usize__; \
|
||||
if (size__ > (SIZE)) { \
|
||||
size__ = (SIZE); \
|
||||
} \
|
||||
StringMemcat(BLK, str__, size__); \
|
||||
} \
|
||||
|
||||
@@ -68,6 +68,16 @@ struct find_handle_struct {
|
||||
char path[2048];
|
||||
};
|
||||
#endif
|
||||
//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
|
||||
//#define HTS_DEF_FWSTRUCT_topindex_chain
|
||||
//typedef struct topindex_chain topindex_chain;
|
||||
//#endif
|
||||
//struct topindex_chain {
|
||||
// int level; /* sort level */
|
||||
// char *category; /* category */
|
||||
// char name[2048]; /* path */
|
||||
// struct topindex_chain *next; /* next element */
|
||||
//};
|
||||
|
||||
/* Tools */
|
||||
|
||||
@@ -283,6 +293,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
|
||||
char *curr;
|
||||
|
||||
//int n=0;
|
||||
char *a;
|
||||
int slash = 0;
|
||||
|
||||
@@ -327,6 +338,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
if (*curr == '/')
|
||||
curr++;
|
||||
l = link;
|
||||
//c=curr;
|
||||
// couper ce qui est commun
|
||||
while((streql(*link, *curr)) && (*link != 0)) {
|
||||
link++;
|
||||
@@ -338,6 +350,8 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
link--;
|
||||
curr--;
|
||||
}
|
||||
//if (*link=='/') link++;
|
||||
//if (*curr=='/') curr++;
|
||||
}
|
||||
|
||||
// calculer la profondeur du répertoire courant et remonter
|
||||
@@ -348,6 +362,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
|
||||
while(*a)
|
||||
if (*(a++) == '/')
|
||||
strlcatbuff(s, "../", ssize);
|
||||
//if (strlen(s)==0) strcatbuff(s,"/");
|
||||
|
||||
if (slash)
|
||||
strlcatbuff(s, "/", ssize); // keep it absolute!
|
||||
|
||||
@@ -639,7 +639,8 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
|
||||
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
|
||||
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
|
||||
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
// int ok=0; // OPTI
|
||||
ok = 0;
|
||||
switch (j) {
|
||||
case 0: // prioritaire
|
||||
|
||||
@@ -80,10 +80,6 @@ htspair_t hts_detect_embed[] = {
|
||||
{NULL, NULL}
|
||||
};
|
||||
|
||||
/* HTML5 media siblings of <img src>: same near-link treatment (#451) */
|
||||
static const htspair_t hts_detect_embed_html5[] = {
|
||||
{"source", "src"}, {"source", "srcset"}, {"track", "src"}, {NULL, NULL}};
|
||||
|
||||
/* Internal */
|
||||
static int hts_acceptlink_(httrackp * opt, int ptr, const char *adr,
|
||||
const char *fil, const char *tag,
|
||||
@@ -140,17 +136,6 @@ static int cmp_token(const char *tag, const char *cmp) {
|
||||
&& !isalnum((unsigned char) tag[p]));
|
||||
}
|
||||
|
||||
/* TRUE if (tag, attribute) matches an embedded-asset pair in the table */
|
||||
static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
|
||||
const char *attribute) {
|
||||
int i;
|
||||
for (i = 0; table[i].tag != NULL; i++) {
|
||||
if (cmp_token(tag, table[i].tag) && cmp_token(attribute, table[i].attr))
|
||||
return HTS_TRUE;
|
||||
}
|
||||
return HTS_FALSE;
|
||||
}
|
||||
|
||||
static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
const char *adr, const char *fil, const char *tag,
|
||||
const char *attribute, int *set_prio_to,
|
||||
@@ -178,9 +163,15 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
/* Built-in known tags (<img src=..>, ..) */
|
||||
if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) {
|
||||
if (is_embed_pair(hts_detect_embed, tag, attribute) ||
|
||||
is_embed_pair(hts_detect_embed_html5, tag, attribute)) {
|
||||
embedded_triggered = 1;
|
||||
int i;
|
||||
|
||||
for(i = 0; hts_detect_embed[i].tag != NULL; i++) {
|
||||
if (cmp_token(tag, hts_detect_embed[i].tag)
|
||||
&& cmp_token(attribute, hts_detect_embed[i].attr)
|
||||
) {
|
||||
embedded_triggered = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,10 +179,12 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
/* Doit-on traiter les non html? */
|
||||
if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas
|
||||
if (!ishtml(opt, fil)) { // non il ne faut pas
|
||||
if (!ishtml(opt, fil)) { // non il ne faut pas
|
||||
//adr[0]='\0'; // ne pas traiter ce lien, pas traiter
|
||||
forbidden_url = 1; // interdire récupération du lien
|
||||
hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr,
|
||||
fil);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,6 +343,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
|
||||
} else { // adresse différente, sortir?
|
||||
|
||||
//if (!opt->wizard) { // mode non wizard
|
||||
// doit-on traiter ce lien?.. vérifier droits de sortie
|
||||
switch ((opt->travel & HTS_TRAVEL_SCOPE_MASK)) {
|
||||
case HTS_TRAVEL_SAME_ADDRESS:
|
||||
@@ -377,6 +371,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
if ((i > 0) && (j > 0)) {
|
||||
if (!strfield2(adr + i, urladr() + j)) { // !=
|
||||
if (!opt->wizard) { // mode non wizard
|
||||
// printf("refused: %s\n",adr);
|
||||
forbidden_url = 1; // pas même domaine
|
||||
hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",
|
||||
adr, fil);
|
||||
@@ -404,6 +399,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
if ((i > 0) && (j > 0)) {
|
||||
if (!strfield2(adr + i, urladr() + j)) { // !-
|
||||
if (!opt->wizard) { // mode non wizard
|
||||
// printf("refused: %s\n",adr);
|
||||
forbidden_url = 1; // pas même .xx
|
||||
hts_log_print(opt, LOG_DEBUG,
|
||||
"foreign location link canceled: %s%s", adr, fil);
|
||||
@@ -434,6 +430,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
// récupérer les liens à côtés d'un lien (nearlink) (nvelle pos)
|
||||
if (forbidden_url != 0 && opt->nearlink) {
|
||||
if (!ishtml(opt, fil)) { // non html
|
||||
//printf("ok %s%s\n",ad,fil);
|
||||
forbidden_url = 0; // autoriser
|
||||
may_set_prio_to = 1 + 1; // set prio to 1 (parse but skip urls) if near is the winner
|
||||
hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil);
|
||||
@@ -881,6 +878,7 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
|
||||
}
|
||||
}
|
||||
}
|
||||
//adr[0]='\0'; // cancel
|
||||
}
|
||||
// -------------------- FINAL PHASE --------------------
|
||||
// Test if the "Near" test won
|
||||
|
||||
101
src/htszlib.c
101
src/htszlib.c
@@ -47,89 +47,48 @@ Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/*
|
||||
Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
|
||||
Unpack file into a new file
|
||||
Return value: size of the new file, or -1 if an error occurred
|
||||
*/
|
||||
/* Note: utf-8 */
|
||||
int hts_zunpack(char *filename, char *newfile) {
|
||||
int ret = -1;
|
||||
|
||||
if (filename != NULL && newfile != NULL && filename[0] && newfile[0]) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *const in = FOPEN(fconv(catbuff, sizeof(catbuff), filename), "rb");
|
||||
if (filename != NULL && newfile != NULL) {
|
||||
if (filename[0] && newfile[0]) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
FILE *const in = FOPEN(fconv(catbuff, sizeof(catbuff), filename), "rb");
|
||||
const int fd = in != NULL ? fileno(in) : -1;
|
||||
const int dup_fd = fd != -1 ? dup(fd) : -1;
|
||||
// Note: we must dup to be able to flose cleanly.
|
||||
const gzFile gz = dup_fd != -1 ? gzdopen(dup_fd, "rb") : NULL;
|
||||
|
||||
if (in != NULL) {
|
||||
unsigned char BIGSTK inbuf[8192];
|
||||
size_t navail = fread(inbuf, 1, sizeof(inbuf), in);
|
||||
/* gzip/zlib headers -> +32 windowBits; else raw deflate (RFC1951) */
|
||||
const hts_boolean wrapped =
|
||||
(navail >= 2 &&
|
||||
((inbuf[0] == 0x1f && inbuf[1] == 0x8b) ||
|
||||
((inbuf[0] & 0x0f) == Z_DEFLATED &&
|
||||
(((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
|
||||
int attempt;
|
||||
if (gz) {
|
||||
FILE *const fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
|
||||
int size = 0;
|
||||
|
||||
/* deflate is ambiguous; on failure retry with the other windowBits */
|
||||
for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
|
||||
const int windowBits =
|
||||
(attempt == 0 ? wrapped : !wrapped) ? (32 + MAX_WBITS) : -MAX_WBITS;
|
||||
FILE *fpout;
|
||||
z_stream strm;
|
||||
if (fpout) {
|
||||
int nr;
|
||||
|
||||
if (attempt > 0) {
|
||||
/* rewind input; reopening fpout "wb" discards the partial output */
|
||||
if (fseek(in, 0, SEEK_SET) != 0)
|
||||
break;
|
||||
navail = fread(inbuf, 1, sizeof(inbuf), in);
|
||||
}
|
||||
fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
|
||||
if (fpout == NULL)
|
||||
break;
|
||||
memset(&strm, 0, sizeof(strm));
|
||||
if (inflateInit2(&strm, windowBits) != Z_OK) {
|
||||
fclose(fpout);
|
||||
break;
|
||||
}
|
||||
{
|
||||
hts_boolean ok = HTS_TRUE;
|
||||
int size = 0;
|
||||
int zerr = Z_OK;
|
||||
|
||||
/* chunked inflate; first chunk in inbuf, single member */
|
||||
do {
|
||||
strm.next_in = inbuf;
|
||||
strm.avail_in = (uInt) navail;
|
||||
do {
|
||||
unsigned char BIGSTK outbuf[8192];
|
||||
size_t produced;
|
||||
char BIGSTK buff[1024];
|
||||
|
||||
strm.next_out = outbuf;
|
||||
strm.avail_out = sizeof(outbuf);
|
||||
zerr = inflate(&strm, Z_NO_FLUSH);
|
||||
if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
|
||||
zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
|
||||
ok = HTS_FALSE;
|
||||
break;
|
||||
}
|
||||
produced = sizeof(outbuf) - strm.avail_out;
|
||||
if (produced > 0 &&
|
||||
fwrite(outbuf, 1, produced, fpout) != produced) {
|
||||
ok = HTS_FALSE;
|
||||
break;
|
||||
}
|
||||
size += (int) produced;
|
||||
} while (strm.avail_out == 0);
|
||||
if (!ok || zerr == Z_STREAM_END)
|
||||
break;
|
||||
navail = fread(inbuf, 1, sizeof(inbuf), in);
|
||||
} while (navail > 0);
|
||||
if (ok && zerr == Z_STREAM_END)
|
||||
ret = size;
|
||||
}
|
||||
inflateEnd(&strm);
|
||||
fclose(fpout);
|
||||
nr = gzread(gz, buff, sizeof(buff));
|
||||
if (nr > 0) {
|
||||
size += nr;
|
||||
if (fwrite(buff, 1, nr, fpout) != nr)
|
||||
nr = size = -1;
|
||||
}
|
||||
} while(nr > 0);
|
||||
fclose(fpout);
|
||||
} else
|
||||
size = -1;
|
||||
gzclose(gz);
|
||||
ret = (int) size;
|
||||
}
|
||||
if (in != NULL) {
|
||||
fclose(in);
|
||||
}
|
||||
fclose(in);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
|
||||
@@ -36,6 +36,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
/* ZLib */
|
||||
#include "zlib.h"
|
||||
//#include "zutil.h"
|
||||
|
||||
/* MiniZip */
|
||||
#include "minizip/zip.h"
|
||||
|
||||
@@ -54,6 +54,7 @@ static int linput(FILE * fp, char *s, int max);
|
||||
#include "htswrap.h"
|
||||
|
||||
/* specific definitions */
|
||||
//#include "htsbase.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -453,7 +454,8 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b
|
||||
for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) { // no lien
|
||||
int i = (back_index + _i) % back_max; // commencer par le "premier" (l'actuel)
|
||||
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
if (back[i].status >= 0) { // signifie "lien actif"
|
||||
// int ok=0; // OPTI
|
||||
ok = 0;
|
||||
switch (j) {
|
||||
case 0: // prioritaire
|
||||
@@ -771,7 +773,28 @@ static void sig_finish(int code) { // finir et quitter
|
||||
fprintf(stderr, "\nExit requested to engine (signal %d)\n", code);
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
#ifdef _WIN32
|
||||
#if 0
|
||||
static void sig_ask(int code) { // demander
|
||||
char s[256];
|
||||
|
||||
signal(code, sig_term); // quitter si encore
|
||||
printf("\nQuit program/Interrupt/Cancel? (Q/I/C) ");
|
||||
fflush(stdout);
|
||||
scanf("%s", s);
|
||||
if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
|
||||
|| (s[0] == 'q') || (s[0] == 'Q'))
|
||||
exit(0); // quitter
|
||||
else if ((s[0] == 'i') || (s[0] == 'I')) {
|
||||
if (global_opt != NULL) {
|
||||
// ask for stop
|
||||
global_opt->state.stop = 1;
|
||||
}
|
||||
}
|
||||
signal(code, sig_ask); // remettre signal
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
static void sig_doback(int blind);
|
||||
static void sig_back(int code) { // ignorer et mettre en backing
|
||||
if (global_opt != NULL && !global_opt->background_on_suspend) {
|
||||
@@ -786,6 +809,36 @@ static void sig_back(int code) { // ignorer et mettre en backing
|
||||
}
|
||||
}
|
||||
|
||||
#if 0
|
||||
static void sig_ask(int code) { // demander
|
||||
char s[256];
|
||||
|
||||
signal(code, sig_term); // quitter si encore
|
||||
printf
|
||||
("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) ");
|
||||
fflush(stdout);
|
||||
scanf("%s", s);
|
||||
if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
|
||||
|| (s[0] == 'q') || (s[0] == 'Q'))
|
||||
exit(0); // quitter
|
||||
else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A'))
|
||||
sig_doback(0); // arrière plan
|
||||
else if ((s[0] == 'l') || (s[0] == 'L'))
|
||||
sig_doback(1); // arrière plan
|
||||
else if ((s[0] == 'i') || (s[0] == 'I')) {
|
||||
if (global_opt != NULL) {
|
||||
// ask for stop
|
||||
printf("finishing pending transfers.. please wait\n");
|
||||
global_opt->state.stop = 1;
|
||||
}
|
||||
signal(code, sig_ask); // remettre signal
|
||||
} else {
|
||||
printf("cancel..\n");
|
||||
signal(code, sig_ask); // remettre signal
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
static void sig_brpipe(int code) { // treat if necessary
|
||||
signal(code, sig_brpipe);
|
||||
}
|
||||
@@ -898,12 +951,23 @@ static void sig_leave(int code) {
|
||||
|
||||
static void signal_handlers(void) {
|
||||
#ifdef _WIN32
|
||||
#if 0 /* BUG366763 */
|
||||
signal(SIGINT, sig_ask); // ^C
|
||||
#else
|
||||
signal(SIGINT, sig_leave); // ^C
|
||||
#endif
|
||||
signal(SIGTERM, sig_finish); // kill <process>
|
||||
#else
|
||||
#if 0 /* BUG366763 */
|
||||
signal(SIGHUP, sig_back); // close window
|
||||
#endif
|
||||
signal(SIGTSTP, sig_back); // ^Z
|
||||
signal(SIGTERM, sig_finish); // kill <process>
|
||||
#if 0 /* BUG366763 */
|
||||
signal(SIGINT, sig_ask); // ^C
|
||||
#else
|
||||
signal(SIGINT, sig_leave); // ^C
|
||||
#endif
|
||||
signal(SIGPIPE, sig_brpipe); // broken pipe (write into non-opened socket)
|
||||
signal(SIGCHLD, sig_ignore); // child change status
|
||||
#endif
|
||||
|
||||
@@ -250,6 +250,13 @@ static int gethost(const char *hostname, SOCaddr * server) {
|
||||
struct addrinfo hints;
|
||||
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
#if 0
|
||||
if (IPV6_resolver == 1) // V4 only (for bogus V6 entries)
|
||||
hints.ai_family = PF_INET;
|
||||
else if (IPV6_resolver == 2) // V6 only (for testing V6 only)
|
||||
hints.ai_family = PF_INET6;
|
||||
else
|
||||
#endif
|
||||
hints.ai_family = PF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
hints.ai_protocol = IPPROTO_TCP;
|
||||
@@ -333,7 +340,12 @@ int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
|
||||
T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM);
|
||||
|
||||
if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) {
|
||||
//char url[HTS_URLMAXSIZE * 2];
|
||||
//char method[32];
|
||||
//char data[32768];
|
||||
|
||||
//url[0] = method[0] = data[0] = '\0';
|
||||
//
|
||||
printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort);
|
||||
printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort);
|
||||
#ifndef _WIN32
|
||||
@@ -485,12 +497,6 @@ static const char *GetHttpMessage(int statuscode) {
|
||||
case 417:
|
||||
return "Expectation Failed";
|
||||
break;
|
||||
case 429:
|
||||
return "Too Many Requests";
|
||||
break;
|
||||
case 451:
|
||||
return "Unavailable For Legal Reasons";
|
||||
break;
|
||||
case 500:
|
||||
return "Internal Server Error";
|
||||
break;
|
||||
@@ -938,6 +944,7 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
char *command;
|
||||
char *proto;
|
||||
char *surl;
|
||||
//int directHit = 0;
|
||||
int headRequest = 0;
|
||||
int listRequest = 0;
|
||||
|
||||
@@ -1079,8 +1086,10 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
|
||||
/* Post-process request */
|
||||
if (link_has_authority(surl)) {
|
||||
if (strncasecmp(surl, "http://proxytrack/",
|
||||
sizeof("http://proxytrack/") - 1) == 0) {
|
||||
if (strncasecmp
|
||||
(surl, "http://proxytrack/",
|
||||
sizeof("http://proxytrack/") - 1) == 0) {
|
||||
//directHit = 1; /* Another direct hit hack */
|
||||
}
|
||||
StringCopy(url, surl);
|
||||
} else {
|
||||
@@ -1101,6 +1110,7 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
toHit += 7;
|
||||
}
|
||||
/* Direct hit */
|
||||
//directHit = 1;
|
||||
StringCopy(url, "");
|
||||
if (!link_has_authority(toHit))
|
||||
StringCat(url, "http://");
|
||||
@@ -1111,6 +1121,7 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
const char *toHit = surl + sizeof("/proxytrack/") - 1;
|
||||
|
||||
/* Direct hit */
|
||||
//directHit = 1;
|
||||
StringCopy(url, "");
|
||||
if (!link_has_authority(toHit))
|
||||
StringCat(url, "http://");
|
||||
@@ -1165,15 +1176,11 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
if (element != NULL) {
|
||||
msgCode = element->statuscode;
|
||||
StringRoom(headers, 8192);
|
||||
sprintf(StringBuffRW(headers),
|
||||
"HTTP/1.1 %d %s\r\n"
|
||||
sprintf(StringBuffRW(headers), "HTTP/1.1 %d %s\r\n"
|
||||
#ifndef NO_WEBDAV
|
||||
"%s"
|
||||
#endif
|
||||
"Content-Type: %s%s%s%s\r\n"
|
||||
"%s%s%s"
|
||||
"%s%s%s"
|
||||
"%s%s%s",
|
||||
"Content-Type: %s%s%s%s\r\n" "%s%s%s" "%s%s%s" "%s%s%s",
|
||||
/* */
|
||||
msgCode, element->msg,
|
||||
#ifndef NO_WEBDAV
|
||||
@@ -1181,18 +1188,16 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
StringBuff(davHeaders),
|
||||
#endif
|
||||
/* Content-type: foo; [ charset=bar ] */
|
||||
hts_effective_mime(element->contenttype),
|
||||
element->contenttype,
|
||||
((element->charset[0]) ? "; charset=\"" : ""),
|
||||
element->charset, ((element->charset[0]) ? "\"" : ""),
|
||||
/* location */
|
||||
((element->location != NULL && element->location[0])
|
||||
? "Location: "
|
||||
: ""),
|
||||
((element->location != NULL && element->location[0])
|
||||
? element->location
|
||||
: ""),
|
||||
((element->location != NULL && element->location[0]) ? "\r\n"
|
||||
: ""),
|
||||
((element->location != NULL
|
||||
&& element->location[0]) ? "Location: " : ""),
|
||||
((element->location != NULL
|
||||
&& element->location[0]) ? element->location : ""),
|
||||
((element->location != NULL
|
||||
&& element->location[0]) ? "\r\n" : ""),
|
||||
/* last-modified */
|
||||
((element->lastmodified[0]) ? "Last-Modified: " : ""),
|
||||
((element->lastmodified[0]) ? element->lastmodified : ""),
|
||||
@@ -1200,7 +1205,8 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
|
||||
/* etag */
|
||||
((element->etag[0]) ? "ETag: " : ""),
|
||||
((element->etag[0]) ? element->etag : ""),
|
||||
((element->etag[0]) ? "\r\n" : ""));
|
||||
((element->etag[0]) ? "\r\n" : "")
|
||||
);
|
||||
StringLength(headers) = (int) strlen(StringBuff(headers));
|
||||
} else {
|
||||
/* No query string, no ending / : check the the <url>/ page */
|
||||
@@ -1594,7 +1600,10 @@ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) {
|
||||
unsigned char Opcode = buffer[0];
|
||||
unsigned char Version = buffer[1];
|
||||
unsigned short Message_Length = READ_NET16(&buffer[2]);
|
||||
unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
|
||||
unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
|
||||
//unsigned int Options = READ_NET32(&buffer[8]);
|
||||
//unsigned int Option_Data = READ_NET32(&buffer[12]); /* ICP_FLAG_SRC_RTT */
|
||||
//unsigned int Sender_Host_Address = READ_NET32(&buffer[16]); /* ignored */
|
||||
unsigned char *Payload = &buffer[20];
|
||||
|
||||
buffer[bufferSize] = '\0'; /* Ensure payload is NULL terminated */
|
||||
|
||||
@@ -52,7 +52,6 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htscore.h"
|
||||
#include "htsback.h"
|
||||
#include "htslib.h" /* hts_effective_mime */
|
||||
|
||||
#include "store.h"
|
||||
#include "proxystrings.h"
|
||||
@@ -1035,6 +1034,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
if (unzOpenCurrentFile(index->zFile) == Z_OK) {
|
||||
char headerBuff[8192 + 2];
|
||||
int readSizeHeader;
|
||||
//int totalHeader = 0;
|
||||
int dataincache = 0;
|
||||
|
||||
/* For BIG comments */
|
||||
@@ -1073,8 +1073,9 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified); // last-modified
|
||||
ZIP_READFIELD_STRING(line, value, "Etag", r->etag); // Etag
|
||||
ZIP_READFIELD_STRING(line, value, "Location", r->location); // 'location' pour moved
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition",
|
||||
r->cdispo); // Content-disposition
|
||||
ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo); // Content-disposition
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Addr", ..); // Original address
|
||||
//ZIP_READFIELD_STRING(line, value, "X-Fil", ..); // Original URI filename
|
||||
ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_); // Original save filename
|
||||
if (line[0] != '\0') {
|
||||
int len = r->headers ? ((int) strlen(r->headers)) : 0;
|
||||
@@ -1088,7 +1089,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
strcat(r->headers, "\r\n");
|
||||
}
|
||||
}
|
||||
} while (offset < readSizeHeader && !lineEof);
|
||||
} while(offset < readSizeHeader && !lineEof);
|
||||
//totalHeader = offset;
|
||||
|
||||
/* Previous entry */
|
||||
if (previous_save_[0] != '\0') {
|
||||
@@ -1199,6 +1201,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
strcpy(r->msg, "Cache Read Error : Read Data");
|
||||
} else
|
||||
*(r->adr + r->size) = '\0';
|
||||
//printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode);
|
||||
} else { // erreur
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpy(r->msg, "Cache Memory Error");
|
||||
@@ -1480,12 +1483,14 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
a += cache_brstr(a, firstline);
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
} else {
|
||||
// fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version);
|
||||
fclose(cache->dat);
|
||||
cache->dat = NULL;
|
||||
free(use);
|
||||
use = NULL;
|
||||
}
|
||||
} else { // non supporté
|
||||
} else { // non supporté
|
||||
// fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline);
|
||||
fclose(cache->dat);
|
||||
cache->dat = NULL;
|
||||
free(use);
|
||||
@@ -1494,6 +1499,7 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
/* */
|
||||
} else { // Vieille version du cache
|
||||
/* */
|
||||
// hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format");
|
||||
cache->version = 0; // cache 1.0
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
}
|
||||
@@ -2283,17 +2289,10 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
|
||||
int size_headers;
|
||||
|
||||
sprintf(st->headers,
|
||||
"HTTP/1.0 %d %s"
|
||||
"\r\n"
|
||||
"X-Server: ProxyTrack " PROXYTRACK_VERSION "\r\n"
|
||||
"Content-type: %s%s%s%s"
|
||||
"\r\n"
|
||||
"Last-modified: %s"
|
||||
"\r\n"
|
||||
"Content-length: %d"
|
||||
"\r\n",
|
||||
element->statuscode, element->msg,
|
||||
/**/ hts_effective_mime(element->contenttype),
|
||||
"HTTP/1.0 %d %s" "\r\n" "X-Server: ProxyTrack " PROXYTRACK_VERSION
|
||||
"\r\n" "Content-type: %s%s%s%s" "\r\n" "Last-modified: %s" "\r\n"
|
||||
"Content-length: %d" "\r\n", element->statuscode, element->msg,
|
||||
/**/ element->contenttype,
|
||||
(element->charset[0] ? "; charset=\"" : ""),
|
||||
(element->charset[0] ? element->charset : ""),
|
||||
(element->charset[0] ? "\"" : ""), /**/ element->lastmodified,
|
||||
@@ -2329,10 +2328,10 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
|
||||
/* args */
|
||||
(link_has_authority(url) ? "" : "http://"), url, "0.0.0.0",
|
||||
tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour,
|
||||
tm->tm_min, tm->tm_sec, hts_effective_mime(element->contenttype),
|
||||
element->statuscode, st->md5,
|
||||
(element->location ? element->location : "-"), (long int) ftell(fp),
|
||||
st->filename, (long int) (size_headers + element->size));
|
||||
tm->tm_min, tm->tm_sec, element->contenttype, element->statuscode,
|
||||
st->md5, (element->location ? element->location : "-"),
|
||||
(long int) ftell(fp), st->filename,
|
||||
(long int) (size_headers + element->size));
|
||||
/* network_doc */
|
||||
if (fwrite(st->headers, 1, size_headers, fp) != size_headers
|
||||
|| (element->size > 0
|
||||
|
||||
@@ -4,16 +4,16 @@
|
||||
# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
|
||||
# tool flags despite the #!/bin/bash above.
|
||||
|
||||
# Golden cache-format regression test (driven by 'httrack -#test=cache-golden <dir>').
|
||||
# Golden cache-format regression test (driven by 'httrack -#B <dir>').
|
||||
#
|
||||
# 01_zlib-cache.test writes the cache with the same build it reads back (a
|
||||
# 01_engine-cache.test writes the cache with the same build it reads back (a
|
||||
# round-trip), so it cannot catch a read-path or ZIP-format regression where
|
||||
# writer and reader drift together. This reads a *committed* cache frozen by an
|
||||
# earlier build and asserts a fixed set of entries still decodes field- and
|
||||
# byte-exact.
|
||||
#
|
||||
# Regenerate the fixture after a deliberate format change with
|
||||
# 'httrack -#test=cache-golden <dir> regen', then copy <dir>/hts-cache/new.zip over the
|
||||
# 'httrack -#B <dir> regen', then copy <dir>/hts-cache/new.zip over the
|
||||
# committed file.
|
||||
|
||||
set -eu
|
||||
@@ -37,11 +37,11 @@ trap 'rm -rf "$dir"' EXIT
|
||||
mkdir -p "$dir/hts-cache"
|
||||
cp "$fixture/hts-cache/new.zip" "$dir/hts-cache/new.zip"
|
||||
|
||||
out=$(httrack -#test=cache-golden "$dir")
|
||||
out=$(httrack -#B "$dir")
|
||||
|
||||
# Match the exact success line: the read must have found and verified every
|
||||
# entry, not merely failed to enter the mode (a renamed/removed test prints the
|
||||
# registry to stderr, which also exits non-zero but never prints this).
|
||||
# entry, not merely failed to enter the mode (a bad -#B falls through to the
|
||||
# usage screen, which also exits non-zero but never prints this).
|
||||
test "$out" = "cache-golden: OK" || {
|
||||
echo "expected 'cache-golden: OK', got: $out" >&2
|
||||
exit 1
|
||||
@@ -4,7 +4,7 @@
|
||||
# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
|
||||
# tool flags despite the #!/bin/bash above.
|
||||
|
||||
# Cache create/read/update logic (driven by 'httrack -#test=cache <dir>').
|
||||
# Cache create/read/update logic (driven by 'httrack -#A <dir>').
|
||||
#
|
||||
# The in-process self-test stores several hand-crafted edge entries (normal
|
||||
# HTML, an empty redirect with a near-limit location, a non-HTML body kept via
|
||||
@@ -20,13 +20,13 @@ set -eu
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# The working directory is a required argument; without it the test prints a
|
||||
# usage line to stderr and returns non-zero.
|
||||
out=$(httrack -#test=cache "$dir")
|
||||
# Like the other -# debug modes, a trailing token (the working directory) is
|
||||
# required; a bare '-#A' falls through to the usage screen.
|
||||
out=$(httrack -#A "$dir")
|
||||
|
||||
# Match the exact success line, so the test cannot pass for an unrelated reason
|
||||
# (e.g. the cache test being gone, which prints the registry to stderr but
|
||||
# never prints this line).
|
||||
# (e.g. the -#A mode being gone and falling through to the usage screen, which
|
||||
# also exits non-zero but never prints this).
|
||||
test "$out" = "cache-selftest: OK" || {
|
||||
echo "expected 'cache-selftest: OK', got: $out" >&2
|
||||
exit 1
|
||||
@@ -4,13 +4,13 @@
|
||||
set -euo pipefail
|
||||
|
||||
# charset -> UTF-8 conversion (hts_convertStringToUTF8).
|
||||
# -#test=charset <charset> <string> prints the string re-decoded from <charset> as UTF-8.
|
||||
# -#3 <charset> <string> prints the string re-decoded from <charset> as UTF-8.
|
||||
conv() {
|
||||
test "$(httrack -O /dev/null -#test=charset "$1" "$2")" == "$3" || exit 1
|
||||
test "$(httrack -O /dev/null -#3 "$1" "$2")" == "$3" || exit 1
|
||||
}
|
||||
# crash probe: malformed input must exit cleanly, not abort.
|
||||
runs() {
|
||||
httrack -O /dev/null -#test=charset "$1" "$2" >/dev/null 2>&1 || exit 1
|
||||
httrack -O /dev/null -#3 "$1" "$2" >/dev/null 2>&1 || exit 1
|
||||
}
|
||||
|
||||
# the source bytes below are UTF-8 (this file is UTF-8); "café" is 0x63 61 66 C3 A9.
|
||||
@@ -31,7 +31,7 @@ conv 'us-ascii' 'hello' 'hello'
|
||||
# unknown charset: ASCII passes through unchanged, but non-ASCII input cannot be
|
||||
# decoded and yields empty output (an error is printed to stderr).
|
||||
conv 'no-such-charset-xyz' 'abc' 'abc'
|
||||
test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev/null)" == "" || exit 1
|
||||
test "$(httrack -O /dev/null -#3 'no-such-charset-xyz' 'café' 2>/dev/null)" == "" || exit 1
|
||||
|
||||
# malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
|
||||
runs 'utf-8' $'\x80'
|
||||
|
||||
@@ -90,16 +90,4 @@ refused "dangling-quote argument not refused cleanly"
|
||||
run_only "$tmp/q-lone" '"'
|
||||
refused "lone-quote argument not refused cleanly"
|
||||
|
||||
# --pause (#185): valid MIN[:MAX] accepted; malformed, reversed, over-range and
|
||||
# non-finite values refused cleanly. NaN defeats naive `<`/`>` checks (it
|
||||
# compares false to everything), so it must not slip through to the int cast.
|
||||
run "$tmp/pause-ok" --pause 0.2:0.4
|
||||
accepted "$tmp/pause-ok" "#185: valid --pause range rejected"
|
||||
run "$tmp/pause-fix" --pause 0.2
|
||||
accepted "$tmp/pause-fix" "#185: valid fixed --pause rejected"
|
||||
for bad in nan nan:5 5:nan inf 10:5 99999; do
|
||||
run "$tmp/pause-bad" --pause "$bad"
|
||||
refused "#185: invalid --pause '$bad' not refused cleanly"
|
||||
done
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Issue #151 guard: the request Cookie header must be bare RFC 6265 name=value
|
||||
# pairs, no $Version/$Path attributes. Driven by the 'httrack -#test=cookies' selftest.
|
||||
# pairs, no $Version/$Path attributes. Driven by the 'httrack -#Q' selftest.
|
||||
|
||||
set -eu
|
||||
|
||||
# 'run' is an ignored placeholder argument.
|
||||
out=$(httrack -#test=cookies run)
|
||||
# A trailing token is required; a bare '-#Q' falls through to the usage screen.
|
||||
out=$(httrack -#Q run)
|
||||
|
||||
# Exact-match the success line so a renamed/removed test (it prints the registry
|
||||
# to stderr) can't pass.
|
||||
# Exact-match the success line so a fall-through to usage can't pass the test.
|
||||
test "$out" = "cookie-header: OK" || {
|
||||
echo "expected 'cookie-header: OK', got: $out" >&2
|
||||
exit 1
|
||||
|
||||
@@ -2,16 +2,15 @@
|
||||
#
|
||||
# Regression guard for the unsigned-enum sentinel trap: copy_htsopt's
|
||||
# `if (from->X > -1)` guard is always false for unsigned hts_boolean fields, so
|
||||
# they silently stop being copied. Driven by the in-process 'httrack -#test=copyopt' test.
|
||||
# they silently stop being copied. Driven by the in-process 'httrack -#9' test.
|
||||
# Keep POSIX-portable (harness runs it via $(BASH), a plain /bin/sh on macOS).
|
||||
|
||||
set -eu
|
||||
|
||||
# 'run' is an ignored placeholder argument.
|
||||
out=$(httrack -#test=copyopt run)
|
||||
# A trailing token is required; a bare '-#9' falls through to the usage screen.
|
||||
out=$(httrack -#9 run)
|
||||
|
||||
# Exact-match the success line so a renamed/removed test (it prints the registry
|
||||
# to stderr) can't pass.
|
||||
# Exact-match the success line so a fall-through to usage can't pass the test.
|
||||
test "$out" = "copy-htsopt: OK" || {
|
||||
echo "expected 'copy-htsopt: OK', got: $out" >&2
|
||||
exit 1
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# DNS resolver/cache self-test: a mock getaddrinfo (no network) checks address
|
||||
# family, single-address selection, the -@i4/-@i6 family filter, and cache reuse.
|
||||
# 'run' is an ignored placeholder argument.
|
||||
out=$(httrack -#test=dns run)
|
||||
|
||||
test "$out" = "dns-selftest: OK" || {
|
||||
echo "expected 'dns-selftest: OK', got: $out" >&2
|
||||
exit 1
|
||||
}
|
||||
@@ -4,13 +4,13 @@
|
||||
set -euo pipefail
|
||||
|
||||
# HTML entity unescaping (hts_unescapeEntitiesWithCharset).
|
||||
# -#test=entities <string> prints the string with entities decoded (UTF-8 output).
|
||||
# -#6 <string> prints the string with entities decoded (UTF-8 output).
|
||||
ent() {
|
||||
test "$(httrack -O /dev/null -#test=entities "$1")" == "$2" || exit 1
|
||||
test "$(httrack -O /dev/null -#6 "$1")" == "$2" || exit 1
|
||||
}
|
||||
# crash probe: malformed input must exit cleanly, not abort.
|
||||
runs() {
|
||||
httrack -O /dev/null -#test=entities "$1" >/dev/null 2>&1 || exit 1
|
||||
httrack -O /dev/null -#6 "$1" >/dev/null 2>&1 || exit 1
|
||||
}
|
||||
|
||||
# named entities
|
||||
@@ -18,21 +18,6 @@ ent '&' '&'
|
||||
ent '<>' '<>'
|
||||
ent 'é' 'é'
|
||||
|
||||
# HTML5 names from the WHATWG set
|
||||
ent '…' '…'
|
||||
ent '⋃' '⋃'
|
||||
# longest name (31 chars) exercises the name-length cap
|
||||
ent '∳' '∳'
|
||||
# astral codepoint -> 4-byte UTF-8
|
||||
ent '𝔸' '𝔸'
|
||||
# multi-codepoint refs are skipped at generation, so left verbatim
|
||||
ent 'fj' 'fj'
|
||||
|
||||
# common HTML4 names still decode (regression guard against accidental drops)
|
||||
ent '©®™' '©®™'
|
||||
ent '—–' '—–'
|
||||
ent 'αβ' 'αβ'
|
||||
|
||||
# numeric: decimal and hex
|
||||
ent 'AB' 'AB'
|
||||
ent 'A' 'A'
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# HT_ADD_HTMLESCAPED* must reserve the escaper's worst case (6 for _full).
|
||||
httrack -O /dev/null -#test=escape-room run | grep -q "escape-room self-test OK"
|
||||
@@ -1,65 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# -%L URL-list loading (#49): a readable list is honored; an unusable one fails
|
||||
# with the reason (errno / not-a-regular-file), not a bare "Could not include
|
||||
# URL list". Offline: file:// fixture, no server. Asserts on httrack's own
|
||||
# strings and the message shape, so it is locale-independent.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_filelist.XXXXXX") || exit 1
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
echo '<html><body>hi</body></html>' >"$tmp/index.html"
|
||||
|
||||
# run httrack with the given -%L target; structured log lands in $out/hts-log.txt
|
||||
run() {
|
||||
local out="$1" list="$2"
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
httrack -O "$out" --quiet -n "-%L" "$list" >"$out/.stdout" 2>&1 || true
|
||||
LOG="$out/hts-log.txt"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1"
|
||||
cat "$LOG"
|
||||
exit 1
|
||||
}
|
||||
loghas() {
|
||||
grep -Eq "$1" "$LOG" || fail "expected /$1/ in $LOG"
|
||||
}
|
||||
lognot() {
|
||||
if grep -Eq "$1" "$LOG"; then fail "unexpected /$1/ in $LOG"; fi
|
||||
}
|
||||
|
||||
# readable list: its one URL is loaded and counted (count must be non-zero)
|
||||
printf 'file://%s/index.html\n' "$tmp" >"$tmp/urls.txt"
|
||||
run "$tmp/ok" "$tmp/urls.txt"
|
||||
loghas '[1-9][0-9]* links added from'
|
||||
|
||||
# missing file: quoted name + a non-empty reason, never the old reasonless
|
||||
# "Could not include URL list: <name>". The reason is the stat() errno, not the
|
||||
# directory fallback literal (guards against dropping the errno lookup).
|
||||
run "$tmp/miss" "$tmp/nope.txt"
|
||||
loghas 'Could not include URL list "[^"]+": .+'
|
||||
lognot 'Could not include URL list: '
|
||||
lognot 'not a regular file'
|
||||
|
||||
# a directory is rejected with our own reason (locale-independent)
|
||||
mkdir -p "$tmp/adir"
|
||||
run "$tmp/dir" "$tmp/adir"
|
||||
loghas 'Could not include URL list "[^"]+": not a regular file'
|
||||
|
||||
# unreadable regular file: the fopen() errno arm fires, distinct from the
|
||||
# directory branch. Root bypasses mode 000, so skip it there.
|
||||
if test "$(id -u)" -ne 0; then
|
||||
: >"$tmp/noperm.txt"
|
||||
chmod 000 "$tmp/noperm.txt"
|
||||
run "$tmp/perm" "$tmp/noperm.txt"
|
||||
chmod 644 "$tmp/noperm.txt"
|
||||
loghas 'Could not include URL list "[^"]+": .+'
|
||||
lognot 'not a regular file'
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -4,13 +4,13 @@
|
||||
set -euo pipefail
|
||||
|
||||
# wildcard filter engine (strjoker), the core of +/- include/exclude rules.
|
||||
# -#test=filter <filter> <string> prints "<string> does match <filter>" or "... does NOT match ...".
|
||||
# -#0 <filter> <string> prints "<string> does match <filter>" or "... does NOT match ...".
|
||||
|
||||
match() {
|
||||
test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does match $1" || exit 1
|
||||
test "$(httrack -O /dev/null -#0 "$1" "$2")" == "$2 does match $1" || exit 1
|
||||
}
|
||||
nomatch() {
|
||||
test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
||||
test "$(httrack -O /dev/null -#0 "$1" "$2")" == "$2 does NOT match $1" || exit 1
|
||||
}
|
||||
|
||||
# bare star matches everything
|
||||
@@ -50,75 +50,24 @@ match '*foo*bar' 'foozbar'
|
||||
# '?' is the query-string marker, not a single-char wildcard
|
||||
nomatch 'a?c' 'abc'
|
||||
|
||||
# Inside a class, backslash escapes the next char as a literal member (#148):
|
||||
# '\X' matches X only (not '\'), and an escaped ']' is a member, not the terminator.
|
||||
# backslash escapes a metacharacter inside a class so it is matched literally.
|
||||
# Quirk: the decoder also adds the backslash itself to the set, so '\X' matches
|
||||
# both X and '\'. These assertions pin that behavior.
|
||||
match '*[\*]' '*'
|
||||
nomatch '*[\*]' "\\"
|
||||
match '*[\*]' "\\"
|
||||
nomatch '*[\*]' 'a'
|
||||
match '*[\\]' "\\"
|
||||
nomatch '*[\\]' '*'
|
||||
nomatch '*[\\]' 'a'
|
||||
match '*[\[]' '['
|
||||
nomatch '*[\[]' "\\"
|
||||
match '*[\]]' ']'
|
||||
nomatch '*[\]]' "\\"
|
||||
match '*[\[]' "\\"
|
||||
nomatch '*[\[]' 'a'
|
||||
|
||||
# '*[\[\]]' is "the [ or ] character", as the filter guide documents.
|
||||
match '*[\[\]]' '['
|
||||
match '*[\[\]]' ']'
|
||||
nomatch '*[\[\]]' 'a'
|
||||
match '*[\[,\]]' '[' # comma between members is optional
|
||||
match '*[\[,\]]' ']'
|
||||
match '*[a,\[]' 'a' # an escaped member no longer eats the preceding one
|
||||
match '*[a,\[]' '['
|
||||
|
||||
# Escape is decoded before the range/separator/size checks, so '\-' '\,' '\<'
|
||||
# are literal members, not operators.
|
||||
match '*[a\-z]' 'a'
|
||||
match '*[a\-z]' 'z'
|
||||
nomatch '*[a\-z]' 'b' # not the a..z range
|
||||
match '*[\,]' ','
|
||||
nomatch '*[\,]' "\\" # the escape must not leak '\' into the class
|
||||
match '*[\<]' '<'
|
||||
nomatch '*[\<]' "\\"
|
||||
match '*[\[,\],a]' '['
|
||||
match '*[\[,\],a]' ']'
|
||||
match '*[\[,\],a]' 'a'
|
||||
|
||||
# A truncated range '*[a-' is the literal members {a,-}; the parser must not
|
||||
# read past the end decoding it (was a 1-byte heap over-read in the range arm).
|
||||
match '*[a-' 'a'
|
||||
nomatch '*[a-' 'b'
|
||||
|
||||
# *(...) matches exactly one char from the class; *[...] matches a run.
|
||||
match '*(a,b)' 'a'
|
||||
nomatch '*(a,b)' 'aa'
|
||||
nomatch '*(a,b)' 'c'
|
||||
|
||||
# documented composite filters (filters.html)
|
||||
match 'www.*[path].com/*[path].zip' 'www.foo.com/a/b.zip'
|
||||
nomatch 'www.*[path].com/*[path].zip' 'www.foo.com/a/b.tar'
|
||||
match '*.html*[]' 'page.html'
|
||||
nomatch '*.html*[]' 'page.html?x=1' # *[] forbids the trailing query
|
||||
|
||||
# Size-based rules (-#test=filtersize <size> <string> <filter...>): a negative size
|
||||
# means the size is still unknown (scan time). A size exclusion must stay neutral
|
||||
# then, so the file is fetched and only cancelled once its size is known (#143).
|
||||
fsize() {
|
||||
local want="$1"
|
||||
shift
|
||||
test "$(httrack -O /dev/null -#test=filtersize "$@")" == "$want" || exit 1
|
||||
}
|
||||
fsize 'verdict=allowed size_flag=0' -1 foo.jpg -* '+*.jpg' '-*.jpg*[<10]' # scan time: keep
|
||||
fsize 'verdict=forbidden size_flag=1' 5 foo.jpg -* '+*.jpg' '-*.jpg*[<10]' # <10KB: cancel
|
||||
fsize 'verdict=allowed size_flag=1' 20 foo.jpg -* '+*.jpg' '-*.jpg*[<10]' # >=10KB: keep
|
||||
fsize 'verdict=forbidden size_flag=0' -1 foo.txt -* '+*.jpg' '-*.jpg*[<10]' # not a jpg
|
||||
# the '>' operator is just as neutral at scan time, and fires once size is known
|
||||
fsize 'verdict=allowed size_flag=0' -1 foo.jpg -* '+*.jpg' '-*.jpg*[>10]' # scan time: keep
|
||||
fsize 'verdict=forbidden size_flag=1' 20 foo.jpg -* '+*.jpg' '-*.jpg*[>10]' # >10KB: cancel
|
||||
|
||||
# [name]/[file]/[path] never span '?' mid-string; a trailing query is still
|
||||
# tolerated by the global '?' rule (same as plain *.aspx), not the class (#144).
|
||||
nomatch '*[path]/end' 'a?b/end'
|
||||
nomatch '*[file]end' 'foo?xend'
|
||||
nomatch '*[name]X' 'abc?X'
|
||||
match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
|
||||
match '*.aspx' 'page.aspx?y=2'
|
||||
# A literal ']' cannot be a class member: the class parser stops at the first
|
||||
# ']', escaped or not. So '*[\[\]]' does NOT mean "the [ or ] character" as the
|
||||
# filter guide claims (GitHub #148); it parses as the class {'[','\'} followed
|
||||
# by a trailing literal ']'. These assertions document the current (buggy)
|
||||
# behavior so any future matcher fix is a deliberate, visible change.
|
||||
nomatch '*[\[\]]' '[' # not matched, despite the docs
|
||||
match '*[\[\]]' ']' # only via the empty class-match + trailing ']'
|
||||
match '*[\[\]]' '[]' # one of {'[','\'} then the trailing ']'
|
||||
nomatch '*[\[\]]' '[]x'
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# get_ftp_line bounds a hostile CRLF-less FTP reply into its 1024-byte buffer.
|
||||
httrack -O /dev/null -#test=ftp-line run | grep -q "ftp-line self-test OK"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ftp_split_userpass bounds an over-long user:pass@ from a hostile ftp:// URL.
|
||||
httrack -O /dev/null -#test=ftp-userpass run | grep -q "ftp-userpass self-test OK"
|
||||
@@ -3,7 +3,5 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# httrack internal hashtable autotest on 100K keys. Assert the success line (on
|
||||
# stderr) so a misrouted registry entry can't pass on exit code alone.
|
||||
out=$(httrack -#test=hashtable 100000 2>&1)
|
||||
printf '%s\n' "$out" | grep -q "all hashtable tests were successful!" || exit 1
|
||||
# httrack internal hashtable autotest on 100K keys
|
||||
httrack -#7 100000
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Response header-line parsing (treathead via -#test=header <raw-line> ...).
|
||||
# Isolates the wire layer from url_savename, which strips traversal on its own.
|
||||
|
||||
hdr() {
|
||||
local want="$1"
|
||||
shift
|
||||
out="$(httrack -O /dev/null -#test=header "$@" | grep '^contenttype=')"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: $* -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
hdr 'contenttype=application/pdf cdispo=' 'Content-Type: application/pdf'
|
||||
|
||||
# filename= is honored quoted or bare.
|
||||
hdr 'contenttype= cdispo=report.pdf' \
|
||||
'Content-Disposition: attachment; filename="report.pdf"'
|
||||
hdr 'contenttype= cdispo=report.pdf' \
|
||||
'Content-Disposition: attachment; filename=report.pdf'
|
||||
|
||||
# Path components in the filename are dropped on the wire (RFC 2616).
|
||||
hdr 'contenttype= cdispo=evil.pdf' \
|
||||
'Content-Disposition: attachment; filename="../../evil.pdf"'
|
||||
@@ -3,13 +3,13 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# IDNA / punycode encode (-#test=idna-encode) and decode (-#test=idna-decode). This code has a CVE history,
|
||||
# IDNA / punycode encode (-#4) and decode (-#5). This code has a CVE history,
|
||||
# so the edge cases below cover passthrough, round-trips, and malformed input.
|
||||
|
||||
enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
|
||||
dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
|
||||
enc() { test "$(httrack -O /dev/null -#4 "$1")" == "$2" || exit 1; }
|
||||
dec() { test "$(httrack -O /dev/null -#5 "$1")" == "$2" || exit 1; }
|
||||
# crash probe: malformed ACE input must exit cleanly, not abort.
|
||||
runs() { httrack -O /dev/null -#test=idna-decode "$1" >/dev/null 2>&1 || exit 1; }
|
||||
runs() { httrack -O /dev/null -#5 "$1" >/dev/null 2>&1 || exit 1; }
|
||||
|
||||
# encode
|
||||
enc 'www.café.com' 'www.xn--caf-dma.com'
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# inplace_escape_*() must match escape_*() on a copy: guards the shared helper.
|
||||
httrack -O /dev/null -#test=inplace-escape run | grep -q "inplace-escape self-test OK"
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# .class constant-pool count is capped to the file size (calloc DoS).
|
||||
httrack -O /dev/null -#test=java run | grep -q "java constant-pool cap self-test OK"
|
||||
@@ -1,12 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# hts_finish_makeindex writes the footer and gates the refresh meta on a single
|
||||
# first link (guards the macro->function extraction).
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
httrack -O /dev/null -#test=makeindex "$dir" run |
|
||||
grep -q "makeindex self-test OK"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user