mirror of
https://github.com/telemt/telemt.git
synced 2026-07-21 16:19:52 +03:00
Compare commits
152 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d851200e47 | ||
|
|
a5216d77fb | ||
|
|
5b5cd952c8 | ||
|
|
a40578b278 | ||
|
|
bd99493622 | ||
|
|
d862deebb2 | ||
|
|
0d869d716c | ||
|
|
1e85b91ad3 | ||
|
|
4679bdcfd5 | ||
|
|
7291c3192c | ||
|
|
4a5dc0b21b | ||
|
|
fabd98ce89 | ||
|
|
7df3dab5e8 | ||
|
|
c6f40e3717 | ||
|
|
91e05265be | ||
|
|
991d5b2c38 | ||
|
|
1f9c82c924 | ||
|
|
73afeccae1 | ||
|
|
feb51cbf57 | ||
|
|
8c65cd868c | ||
|
|
ea296bbdc8 | ||
|
|
fb042f826e | ||
|
|
96425f15c8 | ||
|
|
d4c4980e5a | ||
|
|
893ce0cf36 | ||
|
|
2ac93c6d49 | ||
|
|
a51e58009b | ||
|
|
d523406c0a | ||
|
|
5b3ad0096b | ||
|
|
b587fdbf94 | ||
|
|
77a45e509a | ||
|
|
b8be805aed | ||
|
|
a1ebd44cee | ||
|
|
25d02a8e0e | ||
|
|
3375017460 | ||
|
|
25e0abae8a | ||
|
|
50538d234e | ||
|
|
e3a7be6786 | ||
|
|
3fc2877205 | ||
|
|
cd1dc2f4c9 | ||
|
|
451227da60 | ||
|
|
f55d8479e3 | ||
|
|
81ae483201 | ||
|
|
ed1895d6df | ||
|
|
88d161a5e9 | ||
|
|
a0ac108807 | ||
|
|
809352fac5 | ||
|
|
22627b498d | ||
|
|
b9c5c71dbc | ||
|
|
7aee991416 | ||
|
|
9a9fd3f55d | ||
|
|
3a5fe31262 | ||
|
|
82f63d0d8a | ||
|
|
fce75163b0 | ||
|
|
fe56621a83 | ||
|
|
1f2910f5bc | ||
|
|
d67e7c5a6f | ||
|
|
558f352a57 | ||
|
|
1ee9a234d7 | ||
|
|
2e13f89f6d | ||
|
|
5eaccee68f | ||
|
|
f56895feac | ||
|
|
87c82c2a63 | ||
|
|
7e5a1841b1 | ||
|
|
e994ddea00 | ||
|
|
5e5c82a0ad | ||
|
|
840713a359 | ||
|
|
50b67a93d6 | ||
|
|
72800e4aa7 | ||
|
|
49742d38a7 | ||
|
|
869d8517a0 | ||
|
|
e82ce634d6 | ||
|
|
f1f46fac42 | ||
|
|
37d0184a0b | ||
|
|
d81d7dba62 | ||
|
|
04b8d8365c | ||
|
|
2e26bfb86e | ||
|
|
d414c73c9b | ||
|
|
d1a97fe10f | ||
|
|
b153782597 | ||
|
|
9dc67727b0 | ||
|
|
2d02fbe548 | ||
|
|
2675779915 | ||
|
|
c4954f745f | ||
|
|
f33abfb09e | ||
|
|
9904da737a | ||
|
|
9a3ff726b2 | ||
|
|
942882f9de | ||
|
|
eeff16c3fd | ||
|
|
c86dc2f65e | ||
|
|
1cbde70a14 | ||
|
|
b95956d141 | ||
|
|
26cd4734de | ||
|
|
52a1b66ad7 | ||
|
|
9ff48c2028 | ||
|
|
b43c683615 | ||
|
|
e41470fb4c | ||
|
|
09dc0cb76c | ||
|
|
c36eb81808 | ||
|
|
0f8aca56d9 | ||
|
|
4e66933a35 | ||
|
|
7cf00db242 | ||
|
|
8bc1ac06d6 | ||
|
|
59cfcf05d3 | ||
|
|
fcbedf66ea | ||
|
|
f5c402d9fc | ||
|
|
118d53239a | ||
|
|
607f5442ad | ||
|
|
1edd63bfb1 | ||
|
|
a808dc2815 | ||
|
|
6dc9f8c27a | ||
|
|
409b0ef5ee | ||
|
|
3d0560d583 | ||
|
|
62af515504 | ||
|
|
eba55e755d | ||
|
|
c4b58ad374 | ||
|
|
db7ff8737c | ||
|
|
cd2bb9c8cd | ||
|
|
8d3f8a8215 | ||
|
|
ff7a12d5f8 | ||
|
|
27ee634f4a | ||
|
|
d7e16f5b26 | ||
|
|
e39aaeb5c5 | ||
|
|
1628a7d822 | ||
|
|
e9c62b6d8d | ||
|
|
36cf3b035c | ||
|
|
8491f5183c | ||
|
|
357852cc59 | ||
|
|
504cafb129 | ||
|
|
1096e38854 | ||
|
|
9bbdf796d8 | ||
|
|
27a5f5a4ec | ||
|
|
a8adc9fe54 | ||
|
|
44be585ee3 | ||
|
|
cb89d3f4fe | ||
|
|
c4e522a16d | ||
|
|
8e5f73a86b | ||
|
|
7d543aeb67 | ||
|
|
89a885c25f | ||
|
|
54e40fd073 | ||
|
|
1934c1279c | ||
|
|
0bc99b9f74 | ||
|
|
1d8e8890a4 | ||
|
|
d1680a7a80 | ||
|
|
b027608282 | ||
|
|
2f2c9b336c | ||
|
|
b9ebfdcd7b | ||
|
|
34b48325fd | ||
|
|
5c573a926b | ||
|
|
462215b53c | ||
|
|
2264980926 | ||
|
|
3d0d575b94 |
49
.github/workflows/coverage.yml
vendored
Normal file
49
.github/workflows/coverage.yml
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
name: Coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ "*" ]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
coverage:
|
||||
name: LLVM coverage report
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
components: llvm-tools-preview
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-llvm-cov-
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Install cargo-llvm-cov
|
||||
run: cargo install --locked cargo-llvm-cov || true
|
||||
|
||||
- name: Generate LCOV report
|
||||
run: cargo llvm-cov --locked --lcov --output-path lcov.info
|
||||
|
||||
- name: Upload LCOV report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: telemt-lcov
|
||||
path: lcov.info
|
||||
@@ -1,18 +1,20 @@
|
||||
# Issues
|
||||
## Warnung
|
||||
### Warnung
|
||||
|
||||
Before opening Issue, if it is more question than problem or bug - ask about that [in our chat](https://t.me/telemtrs)
|
||||
|
||||
## What it is not
|
||||
- NOT Question and Answer
|
||||
- NOT Helpdesk
|
||||
|
||||
***Each of your Issues triggers attempts to reproduce problems and analyze them, which are done manually by people***
|
||||
|
||||
Issues is **NOT** about:
|
||||
- Question and Answer
|
||||
- Helpdesk
|
||||
- Configuration or Intergraion Support
|
||||
|
||||
---
|
||||
|
||||
# Pull Requests
|
||||
|
||||
## General
|
||||
### General
|
||||
- ONLY signed and verified commits
|
||||
- ONLY from your name
|
||||
- DO NOT commit with `codex`, `claude`, or other AI tools as author/committer
|
||||
@@ -20,7 +22,7 @@ Before opening Issue, if it is more question than problem or bug - ask about tha
|
||||
|
||||
---
|
||||
|
||||
## Definition of Ready (MANDATORY)
|
||||
### Definition of Ready (MANDATORY)
|
||||
|
||||
A Pull Request WILL be ignored or closed if:
|
||||
|
||||
@@ -32,14 +34,14 @@ A Pull Request WILL be ignored or closed if:
|
||||
|
||||
---
|
||||
|
||||
## Blessed Principles
|
||||
### Blessed Principles
|
||||
- PR must build
|
||||
- PR must pass tests
|
||||
- PR must be understood by author
|
||||
|
||||
---
|
||||
|
||||
## AI Usage Policy
|
||||
### AI Usage Policy
|
||||
|
||||
AI tools (Claude, ChatGPT, Codex, DeepSeek, etc.) are allowed as **assistants**, NOT as decision-makers.
|
||||
|
||||
@@ -60,7 +62,7 @@ PRs that look like unverified AI dumps WILL be closed
|
||||
|
||||
---
|
||||
|
||||
## Maintainer Policy
|
||||
### Maintainer Policy
|
||||
|
||||
Maintainers reserve the right to:
|
||||
|
||||
@@ -72,7 +74,7 @@ Respect the reviewers time
|
||||
|
||||
---
|
||||
|
||||
## Enforcement
|
||||
### Enforcement
|
||||
|
||||
Pull Requests that violate project standards may be closed without review.
|
||||
|
||||
|
||||
542
Cargo.lock
generated
542
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
116
Cargo.toml
116
Cargo.toml
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "telemt"
|
||||
version = "3.4.13"
|
||||
version = "3.4.25"
|
||||
edition = "2024"
|
||||
|
||||
[features]
|
||||
@@ -8,88 +8,90 @@ redteam_offline_expected_fail = []
|
||||
|
||||
[dependencies]
|
||||
# C
|
||||
libc = "0.2"
|
||||
libc = "0.2.186"
|
||||
|
||||
# Async runtime
|
||||
tokio = { version = "1.42", features = ["full", "tracing"] }
|
||||
tokio-util = { version = "0.7", features = ["full"] }
|
||||
tokio = { version = "1.52.3", features = ["full", "tracing"] }
|
||||
tokio-util = { version = "0.7.18", features = ["full"] }
|
||||
|
||||
# Crypto
|
||||
aes = "0.8"
|
||||
ctr = "0.9"
|
||||
cbc = "0.1"
|
||||
sha2 = "0.10"
|
||||
sha1 = "0.10"
|
||||
md-5 = "0.10"
|
||||
hmac = "0.12"
|
||||
crc32fast = "1.4"
|
||||
crc32c = "0.6"
|
||||
zeroize = { version = "1.8", features = ["derive"] }
|
||||
subtle = "2.6"
|
||||
static_assertions = "1.1"
|
||||
aes = { version = "0.8.4", features = ["zeroize"] }
|
||||
ctr = { version = "0.9.2", features = ["zeroize"] }
|
||||
cbc = "0.1.2"
|
||||
sha2 = "0.10.9"
|
||||
sha1 = "0.10.6"
|
||||
md-5 = "0.10.6"
|
||||
hmac = "0.12.1"
|
||||
crc32fast = "1.5.0"
|
||||
crc32c = "0.6.8"
|
||||
zeroize = { version = "1.9.0", features = ["derive"] }
|
||||
subtle = "2.6.1"
|
||||
static_assertions = "1.1.0"
|
||||
ml-kem = { version = "0.3.2", default-features = false, features = ["alloc", "zeroize"] }
|
||||
|
||||
# Network
|
||||
socket2 = { version = "0.6", features = ["all"] }
|
||||
nix = { version = "0.31", default-features = false, features = [
|
||||
socket2 = { version = "0.6.4", features = ["all"] }
|
||||
nix = { version = "0.31.3", default-features = false, features = [
|
||||
"net",
|
||||
"user",
|
||||
"process",
|
||||
"fs",
|
||||
"signal",
|
||||
] }
|
||||
shadowsocks = { version = "1.24", features = ["aead-cipher-2022"] }
|
||||
shadowsocks = { version = "1.24.0", features = ["aead-cipher-2022"] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
toml = "1.0"
|
||||
x509-parser = "0.18"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = "1.0.150"
|
||||
toml = "1.1"
|
||||
x509-parser = "0.18.1"
|
||||
|
||||
# Utils
|
||||
bytes = "1.9"
|
||||
thiserror = "2.0"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tracing-appender = "0.2"
|
||||
parking_lot = "0.12"
|
||||
dashmap = "6.1"
|
||||
arc-swap = "1.7"
|
||||
lru = "0.16"
|
||||
rand = "0.10"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
hex = "0.4"
|
||||
base64 = "0.22"
|
||||
url = "2.5"
|
||||
regex = "1.11"
|
||||
crossbeam-queue = "0.3"
|
||||
num-bigint = "0.4"
|
||||
num-traits = "0.2"
|
||||
x25519-dalek = "2"
|
||||
anyhow = "1.0"
|
||||
bytes = "1.12.0"
|
||||
thiserror = "2.0.18"
|
||||
tracing = "0.1.44"
|
||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
|
||||
tracing-appender = "0.2.5"
|
||||
parking_lot = "0.12.5"
|
||||
dashmap = "6.2.1"
|
||||
arc-swap = "1.9.1"
|
||||
lru = "0.16.4"
|
||||
rand = "0.10.1"
|
||||
chrono = { version = "0.4.45", features = ["serde"] }
|
||||
hex = "0.4.3"
|
||||
base64 = "0.22.1"
|
||||
url = "2.5.8"
|
||||
regex = "1.12.4"
|
||||
crossbeam-queue = "0.3.12"
|
||||
num-bigint = "0.4.6"
|
||||
num-traits = "0.2.19"
|
||||
x25519-dalek = "2.0.1"
|
||||
anyhow = "1.0.102"
|
||||
|
||||
# HTTP
|
||||
reqwest = { version = "0.13", features = ["rustls"], default-features = false }
|
||||
notify = "8.2"
|
||||
ipnetwork = { version = "0.21", features = ["serde"] }
|
||||
hyper = { version = "1", features = ["server", "http1"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "server-auto"] }
|
||||
http-body-util = "0.1"
|
||||
httpdate = "1.0"
|
||||
tokio-rustls = { version = "0.26", default-features = false, features = [
|
||||
reqwest = { version = "0.13.4", features = ["rustls"], default-features = false }
|
||||
notify = "8.2.0"
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
hyper = { version = "1.10.1", features = ["server", "http1"] }
|
||||
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto"] }
|
||||
http-body-util = "0.1.3"
|
||||
httpdate = "1.0.3"
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
||||
"tls12",
|
||||
] }
|
||||
rustls = { version = "0.23", default-features = false, features = [
|
||||
rustls = { version = "0.23.41", default-features = false, features = [
|
||||
"std",
|
||||
"tls12",
|
||||
"ring",
|
||||
] }
|
||||
webpki-roots = "1.0"
|
||||
webpki-roots = "1.0.8"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
criterion = "0.8"
|
||||
proptest = "1.4"
|
||||
futures = "0.3"
|
||||
tokio-test = "0.4.5"
|
||||
criterion = "0.8.2"
|
||||
proptest = "1.11.0"
|
||||
futures = "0.3.32"
|
||||
tempfile = "3.27.0"
|
||||
|
||||
[[bench]]
|
||||
name = "crypto_bench"
|
||||
|
||||
16
Dockerfile
16
Dockerfile
@@ -55,6 +55,16 @@ RUN set -eux; \
|
||||
strip --strip-unneeded /telemt || true; \
|
||||
rm -f "/tmp/${ASSET}" "/tmp/${ASSET}.sha256" /tmp/telemt
|
||||
|
||||
RUN --mount=type=bind,target=/tmp \
|
||||
mkdir -p /app && \
|
||||
if [ -f /tmp/config.toml ]; then \
|
||||
cp /tmp/config.toml /app/config.toml; \
|
||||
elif [ -f /tmp/config/config.toml ]; then \
|
||||
cp /tmp/config/config.toml /app/config.toml; \
|
||||
else \
|
||||
echo "Config file not found" && exit 1; \
|
||||
fi
|
||||
|
||||
# ==========================
|
||||
# Debug Image
|
||||
# ==========================
|
||||
@@ -73,7 +83,7 @@ RUN set -eux; \
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=minimal /telemt /app/telemt
|
||||
COPY config.toml /app/config.toml
|
||||
COPY ./config/config.toml /app/config.toml
|
||||
|
||||
EXPOSE 443 9090 9091
|
||||
|
||||
@@ -99,7 +109,7 @@ RUN set -eux; \
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=minimal /telemt /app/telemt
|
||||
COPY config.toml /app/config.toml
|
||||
COPY --from=minimal /app/config.toml /app/config.toml
|
||||
|
||||
EXPOSE 443 9090 9091
|
||||
|
||||
@@ -116,7 +126,7 @@ FROM gcr.io/distroless/static-debian12 AS prod
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=minimal /telemt /app/telemt
|
||||
COPY config.toml /app/config.toml
|
||||
COPY --from=minimal /app/config.toml /app/config.toml
|
||||
|
||||
USER nonroot:nonroot
|
||||
|
||||
|
||||
@@ -6,7 +6,9 @@
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> Telegram Clients TLS ClientHello has been banned by JA3 Fingerprint: we are already looking for ways to solve this problem
|
||||
> From June 5th, 2026: we are already analyzing the causes of a new wave of "malfunctions"
|
||||
>
|
||||
> Telegram Clients TLS ClientHello has been banned by JA4/JA4+ Fingerprint: we are already looking for ways to solve this problem
|
||||
>
|
||||
> You can try build your client with our Telegram Devlibrary - [tdlib-obf](https://github.com/telemt/tdlib-obf)
|
||||
|
||||
|
||||
66
README.ru.md
66
README.ru.md
@@ -1,57 +1,52 @@
|
||||
# Telemt — MTProxy на Rust + Tokio
|
||||
|
||||
[](https://github.com/telemt/telemt/releases/latest) [](https://github.com/telemt/telemt/stargazers) [](https://github.com/telemt/telemt/network/members) [](https://t.me/telemtrs)
|
||||
|
||||
***Решает проблемы раньше, чем другие узнают об их существовании***
|
||||
[](https://github.com/telemt/telemt/releases/latest) [](https://github.com/telemt/telemt/stargazers) [](https://github.com/telemt/telemt/network/members)
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> Исправленный TLS ClientHello доступен в Telegram для настольных ПК, Android и iOS.
|
||||
> Клиенты Telegram подвергаются блокировке по JA3-отпечатку; мы ищем варианты решения этой проблемы
|
||||
>
|
||||
> Пожалуйста, обновите клиентское приложение для работы с EE-MTProxy.
|
||||
> Вы можете попробовать собрать свой клиент с нашей Telegram Devlibrary — [tdlib-obf](https://github.com/telemt/tdlib-obf)
|
||||
|
||||
<p align="center">
|
||||
<a href="https://t.me/telemtrs">
|
||||
<img src="/docs/assets/telegram_button.svg" width="150"/>
|
||||
<img src="https://github.com/user-attachments/assets/30b7e7b9-974a-4e3d-aab6-b58a85de4507" width="240"/>
|
||||
</a>
|
||||
</p>
|
||||
|
||||
**Telemt** — это быстрый, безопасный и функциональный сервер, написанный на Rust. Он полностью реализует официальный алгоритм прокси Telegram и добавляет множество улучшений для продакшена:
|
||||
**Telemt** — это быстрый, безопасный и функциональный сервер, написанный на Rust: он полностью реализует официальный алгоритм Telegram прокси и добавляет множество различных улучшений
|
||||
|
||||
## Установка и обновление одной командой
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/telemt/telemt/main/install.sh | sh
|
||||
```
|
||||
|
||||
- [Инструкция по быстрому запуску](docs/Quick_start/QUICK_START_GUIDE.ru.md)
|
||||
- [Quick Start Guide](docs/Quick_start/QUICK_START_GUIDE.en.md)
|
||||
|
||||
Реализация **TLS-fronting** максимально приближена к поведению реального HTTPS-трафика (подробнее - [FAQ](docs/FAQ.ru.md#распознаваемость-для-dpi-и-сканеров)).
|
||||
## Функционал
|
||||
Наша реализация **TLS-fronting** одна из наиболее глубоко отлаженных, продвинутых и почти поведенчески неотличима от настоящего: мы уверены, что сделали это правильно - [см. доказательства в нашей проверке](docs/FAQ.ru.md#распознаваемость-для-dpi-и-сканеров).
|
||||
|
||||
***Middle-End Pool*** оптимизирован для высокой производительности.
|
||||
Наша архитектура ***Middle-End Pool*** в стандартных сценариях самая производительная, по сравнению с другими реализациями подключения к Middle-End прокси: не кардинально, но достаточно
|
||||
|
||||
- Поддержка всех режимов MTProto proxy:
|
||||
- Полная поддержа всех официальных режимов MTProto proxy:
|
||||
- Classic;
|
||||
- Secure (префикс `dd`);
|
||||
- Fake TLS (префикс `ee` + SNI fronting);
|
||||
- Secure — с префиксом `dd`;
|
||||
- Fake TLS — с префиксом `ee` + SNI fronting;
|
||||
- Защита от replay-атак;
|
||||
- Маскировка трафика (перенаправление неизвестных подключений на реальные сайты);
|
||||
- Настраиваемые keepalive, таймауты, IPv6 и «быстрый режим»;
|
||||
- Опциональная маскировка трафика: перенаправление неизвестных подключений на реальные сайты;
|
||||
- Настраиваемые keepalive, таймауты, IPv6 и "быстрый режим";
|
||||
- Корректное завершение работы (Ctrl+C);
|
||||
- Подробное логирование через `trace` и `debug`.
|
||||
- Подробное логирование через `trace` и `debug` с помощью `RUST_LOG`.
|
||||
|
||||
# Подробнее о Telemt
|
||||
- [FAQ](#faq)
|
||||
- [Архитектура](docs/Architecture)
|
||||
- [Параметры конфигурационного файла](docs/Config_params)
|
||||
- [Сборка](#build)
|
||||
- [Установка на BSD](#%D1%83%D1%81%D1%82%D0%B0%D0%BD%D0%BE%D0%B2%D0%BA%D0%B0-%D0%BD%D0%B0-bsd)
|
||||
- [Почему Rust?](#why-rust)
|
||||
## ЧаВо
|
||||
- [Часто задаваемые вопросы](docs/FAQ.ru.md)
|
||||
|
||||
## FAQ
|
||||
- [FAQ RU](docs/FAQ.ru.md)
|
||||
- [FAQ EN](docs/FAQ.en.md)
|
||||
# Узнайте больше о Telemt
|
||||
- [Наша архитектура](docs/Architecture)
|
||||
- [Все конфигурационные параметры](docs/Config_params)
|
||||
- [Как собрать Telemt самостоятельно?](#сборка)
|
||||
- [Установка на BSD](docs/Quick_start/OPENBSD_QUICK_START_GUIDE.en.md)
|
||||
- [Почему Rust?](#почему-rust)
|
||||
|
||||
## Сборка
|
||||
```bash
|
||||
@@ -63,7 +58,7 @@ cd telemt
|
||||
cargo build --release
|
||||
|
||||
# В текущем release-профиле используется lto = "fat" для максимальной оптимизации (см. Cargo.toml).
|
||||
# На системах с малым объёмом RAM (~1 ГБ) можно переопределить это значение на "thin".
|
||||
# На системах с малым объёмом ОЗУ (~1 ГБ) можно переопределить это значение на "thin".
|
||||
|
||||
# Перейдите в каталог /bin
|
||||
mv ./target/release/telemt /bin
|
||||
@@ -73,24 +68,19 @@ chmod +x /bin/telemt
|
||||
telemt config.toml
|
||||
```
|
||||
|
||||
## Установка на BSD
|
||||
- Руководство по сборке и настройке на английском языке [OpenBSD Guide (EN)](docs/Quick_start/OPENBSD_QUICK_START_GUIDE.en.md);
|
||||
- Пример rc.d скрипта: [contrib/openbsd/telemt.rcd](contrib/openbsd/telemt.rcd);
|
||||
- Поддержка sandbox с `pledge(2)` и `unveil(2)` пока не реализована.
|
||||
|
||||
## Почему Rust?
|
||||
- Надёжность для долгоживущих процессов;
|
||||
- Детерминированное управление ресурсами (RAII);
|
||||
- Надёжность при длительной работе и идемпотентное поведение;
|
||||
- Детерминированное управление ресурсами — RAII;
|
||||
- Отсутствие сборщика мусора;
|
||||
- Безопасность памяти;
|
||||
- Безопасность памяти и меньше поверхность атаки;
|
||||
- Асинхронная архитектура Tokio.
|
||||
|
||||
## Поддержать Telemt
|
||||
|
||||
Telemt — это бесплатное программное обеспечение с открытым исходным кодом, разработанное в свободное время.
|
||||
Telemt — это бесплатное программное обеспечение с открытым исходным кодом, разрабатываемое в свободное время.
|
||||
Если оно оказалось вам полезным, вы можете поддержать дальнейшую разработку.
|
||||
|
||||
Принимаемые криптовалюты (BTC, ETH, USDT, 350+ и другие):
|
||||
Любая криптовалюта (BTC, ETH, USDT и 350+ других):
|
||||
|
||||
<p align="center">
|
||||
<a href="https://nowpayments.io/donation?api_key=2bf1afd2-abc2-49f9-a012-f1e715b37223" target="_blank" rel="noreferrer noopener">
|
||||
|
||||
@@ -1,12 +1,24 @@
|
||||
// Cryptobench
|
||||
use criterion::{Criterion, black_box, criterion_group};
|
||||
use criterion::{Criterion, criterion_group, criterion_main};
|
||||
use std::hint::black_box;
|
||||
|
||||
#[allow(unused_imports)]
|
||||
#[path = "../src/crypto/aes.rs"]
|
||||
mod aes_impl;
|
||||
#[allow(unused_imports)]
|
||||
#[path = "../src/error.rs"]
|
||||
mod error;
|
||||
|
||||
use aes_impl::AesCtr;
|
||||
|
||||
fn bench_aes_ctr(c: &mut Criterion) {
|
||||
c.bench_function("aes_ctr_encrypt_64kb", |b| {
|
||||
let data = vec![0u8; 65536];
|
||||
b.iter(|| {
|
||||
let mut enc = AesCtr::new(&[0u8; 32], 0);
|
||||
black_box(enc.encrypt(&data))
|
||||
black_box(enc.encrypt(black_box(data.as_slice())))
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_aes_ctr);
|
||||
criterion_main!(benches);
|
||||
|
||||
@@ -4,7 +4,6 @@ services:
|
||||
context: .
|
||||
target: prod-netfilter
|
||||
network_mode: host
|
||||
ports: []
|
||||
ports: !reset []
|
||||
cap_add:
|
||||
- NET_BIND_SERVICE
|
||||
- NET_ADMIN
|
||||
- NET_ADMIN
|
||||
@@ -86,6 +86,9 @@ Die unten angegebenen `Default`-Werte sind Code-Defaults (bei fehlendem Schlüss
|
||||
| `[[upstreams]].weight` | alle Upstreams | `u16` | nein | `1` | Basisgewicht für weighted-random Auswahl. |
|
||||
| `[[upstreams]].enabled` | alle Upstreams | `bool` | nein | `true` | Deaktivierte Einträge werden beim Start ignoriert. |
|
||||
| `[[upstreams]].scopes` | alle Upstreams | `String` | nein | `""` | Komma-separierte Scope-Tags für Request-Routing. |
|
||||
| `[[upstreams]].ipv4` | alle Upstreams | `Option<bool>` | nein | `auto` | Erlaubt IPv4-DC-Ziele für diesen Upstream. |
|
||||
| `[[upstreams]].ipv6` | alle Upstreams | `Option<bool>` | nein | `auto` | Erlaubt IPv6-DC-Ziele für diesen Upstream, inklusive Proxy-Egress unabhängig vom Host-IPv6. |
|
||||
| `[[upstreams]].prefer` | alle Upstreams | `Option<4 \| 6>` | nein | effective `[network].prefer` | Pro-Upstream-Präferenz für die DC-Ziel-Adressfamilie. |
|
||||
| `interface` | `direct` | `Option<String>` | nein | `null` | Interface-Name (z. B. `eth0`) oder lokale Literal-IP. |
|
||||
| `bind_addresses` | `direct` | `Option<Vec<IpAddr>>` | nein | `null` | Explizite Source-IP-Kandidaten (strikter Vorrang vor `interface`). |
|
||||
| `address` | `socks4` | `String` | ja | n/a | SOCKS4-Server (`ip:port` oder `host:port`). |
|
||||
|
||||
@@ -86,6 +86,9 @@ Defaults below are code defaults (used when a key is omitted), not necessarily v
|
||||
| `[[upstreams]].weight` | all upstreams | `u16` | no | `1` | Base weight for weighted-random selection. |
|
||||
| `[[upstreams]].enabled` | all upstreams | `bool` | no | `true` | Disabled entries are ignored at startup. |
|
||||
| `[[upstreams]].scopes` | all upstreams | `String` | no | `""` | Comma-separated scope tags for request-level routing. |
|
||||
| `[[upstreams]].ipv4` | all upstreams | `Option<bool>` | no | `auto` | Allow IPv4 DC targets for this upstream. |
|
||||
| `[[upstreams]].ipv6` | all upstreams | `Option<bool>` | no | `auto` | Allow IPv6 DC targets for this upstream, including proxy egress independent of host IPv6. |
|
||||
| `[[upstreams]].prefer` | all upstreams | `Option<4 \| 6>` | no | effective `[network].prefer` | Per-upstream DC target family preference. |
|
||||
| `interface` | `direct` | `Option<String>` | no | `null` | Interface name (e.g. `eth0`) or literal local IP for bind selection. |
|
||||
| `bind_addresses` | `direct` | `Option<Vec<IpAddr>>` | no | `null` | Explicit local source IP candidates (strict priority over `interface`). |
|
||||
| `address` | `socks4` | `String` | yes | n/a | SOCKS4 server endpoint (`ip:port` or `host:port`). |
|
||||
|
||||
@@ -86,6 +86,9 @@
|
||||
| `[[upstreams]].weight` | все upstream | `u16` | нет | `1` | Базовый вес в weighted-random выборе. |
|
||||
| `[[upstreams]].enabled` | все upstream | `bool` | нет | `true` | Выключенные записи игнорируются на старте. |
|
||||
| `[[upstreams]].scopes` | все upstream | `String` | нет | `""` | Список scope-токенов через запятую для маршрутизации. |
|
||||
| `[[upstreams]].ipv4` | все upstream | `Option<bool>` | нет | `auto` | Разрешает IPv4 DC-targets для этого upstream. |
|
||||
| `[[upstreams]].ipv6` | все upstream | `Option<bool>` | нет | `auto` | Разрешает IPv6 DC-targets для этого upstream, включая proxy egress независимо от IPv6 на хосте. |
|
||||
| `[[upstreams]].prefer` | все upstream | `Option<4 \| 6>` | нет | эффективный `[network].prefer` | Предпочтительное семейство DC-target для конкретного upstream. |
|
||||
| `interface` | `direct` | `Option<String>` | нет | `null` | Имя интерфейса (например `eth0`) или literal локальный IP. |
|
||||
| `bind_addresses` | `direct` | `Option<Vec<IpAddr>>` | нет | `null` | Явные кандидаты source IP (имеют приоритет над `interface`). |
|
||||
| `address` | `socks4` | `String` | да | n/a | Адрес SOCKS4 сервера (`ip:port` или `host:port`). |
|
||||
|
||||
@@ -103,14 +103,21 @@ Notes:
|
||||
| `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` |
|
||||
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
||||
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
||||
| `GET` | `/v1/runtime/tls-fingerprints` | optional `limit=1..1000` | `200` | `RuntimeEdgeTlsFingerprintsData` |
|
||||
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
|
||||
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
||||
| `GET` | `/v1/config` | none | `200` | `ConfigData` |
|
||||
| `PATCH` | `/v1/config` | sparse JSON object; optional reload query | `200` or `202` | `PatchConfigResponse` |
|
||||
| `POST` | `/v1/system/reload` | `ReloadRequest` or empty body | `202` | `ReloadAccepted` |
|
||||
| `GET` | `/v1/system/reload/{id}` | none | `200` | `ReloadStatus` |
|
||||
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
|
||||
| `POST` | `/v1/users` | `CreateUserRequest` | `201` or `202` | `CreateUserResponse` |
|
||||
| `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` |
|
||||
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` or `202` | `UserInfo` |
|
||||
| `DELETE` | `/v1/users/{username}` | none | `200` or `202` | `DeleteUserResponse` |
|
||||
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` or `202` | `CreateUserResponse` |
|
||||
| `POST` | `/v1/users/{username}/enable` | empty body | `200` or `202` | `UserInfo` |
|
||||
| `POST` | `/v1/users/{username}/disable` | empty body | `200` or `202` | `UserInfo` |
|
||||
| `POST` | `/v1/users/{username}/reset-quota` | empty body | `200` | `ResetUserQuotaResponse` |
|
||||
|
||||
## Endpoint Behavior
|
||||
@@ -140,12 +147,18 @@ Notes:
|
||||
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
|
||||
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
|
||||
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
|
||||
| `GET /v1/config` | Returns the current editable config sections as JSON (no `access.*`) plus the revision. |
|
||||
| `PATCH /v1/config` | Applies a sparse patch and optionally submits an in-process runtime reload to Maestro. |
|
||||
| `POST /v1/system/reload` | Loads and validates the current on-disk config, then asks Maestro to prepare and activate a new runtime generation. |
|
||||
| `GET /v1/system/reload/{id}` | Returns one retained reload status; the coordinator retains the most recent 32 operations. |
|
||||
| `GET /v1/users` | Returns disk-first user views sorted by username. |
|
||||
| `POST /v1/users` | Creates a user and returns the effective user view plus secret. |
|
||||
| `GET /v1/users/{username}` | Returns one disk-first user view or `404` when absent. |
|
||||
| `PATCH /v1/users/{username}` | Updates selected per-user fields with JSON Merge Patch semantics. |
|
||||
| `DELETE /v1/users/{username}` | Deletes one user and related per-user access-map entries. |
|
||||
| `POST /v1/users/{username}/rotate-secret` | Rotates one user's secret and returns the effective secret. |
|
||||
| `POST /v1/users/{username}/enable` | Enables one user, removing any disabled override from config. |
|
||||
| `POST /v1/users/{username}/disable` | Disables one user and closes active runtime sessions for that user. |
|
||||
| `POST /v1/users/{username}/reset-quota` | Resets one user's runtime quota counter and persists quota state. |
|
||||
|
||||
## Common Error Codes
|
||||
@@ -153,17 +166,21 @@ Notes:
|
||||
| HTTP | `error.code` | Trigger |
|
||||
| --- | --- | --- |
|
||||
| `400` | `bad_request` | Invalid JSON, validation failures, malformed request body. |
|
||||
| `400` | `access_not_editable` | `PATCH /v1/config` body contains an `access` key (managed via users API). |
|
||||
| `400` | `section_not_editable` | `PATCH /v1/config` body contains `server`, `network`, or an unknown top-level key. |
|
||||
| `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. |
|
||||
| `403` | `forbidden` | Source IP is not allowed by whitelist. |
|
||||
| `403` | `read_only` | Mutating endpoint called while `read_only=true`. |
|
||||
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route. |
|
||||
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
|
||||
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
|
||||
| `409` | `reload_in_progress` | Another reload operation is non-terminal. |
|
||||
| `409` | `user_exists` | User already exists on create. |
|
||||
| `409` | `last_user_forbidden` | Attempt to delete last configured user. |
|
||||
| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. |
|
||||
| `500` | `internal_error` | Internal error (I/O, serialization, config load/save). |
|
||||
| `503` | `api_disabled` | API disabled in config. |
|
||||
| `503` | `maestro_unavailable` | Maestro's reload command channel is unavailable. |
|
||||
|
||||
## Routing and Method Edge Cases
|
||||
|
||||
@@ -172,9 +189,12 @@ Notes:
|
||||
| Path matching | Exact match on `req.uri().path()`. Query string does not affect route matching. |
|
||||
| Trailing slash | Trimmed for route matching when path length is greater than 1. Example: `/v1/users/` matches `/v1/users`. |
|
||||
| Username route with extra slash | `/v1/users/{username}/...` is not treated as user route and returns `404`. |
|
||||
| `DELETE /v1/config` (or any method not in `GET`, `PATCH`) | `405 method_not_allowed` with `Allow: GET, PATCH`. |
|
||||
| `PUT /v1/users/{username}` | `405 method_not_allowed`. |
|
||||
| `POST /v1/users/{username}` | `404 not_found`. |
|
||||
| `POST /v1/users/{username}/rotate-secret/` | Trailing slash is trimmed and the route matches `rotate-secret`. |
|
||||
| `POST /v1/users/{username}/enable/` | Trailing slash is trimmed and the route matches `enable`. |
|
||||
| `POST /v1/users/{username}/disable/` | Trailing slash is trimmed and the route matches `disable`. |
|
||||
| `POST /v1/users/{username}/reset-quota/` | Trailing slash is trimmed and the route matches `reset-quota`. |
|
||||
|
||||
## Body and JSON Semantics
|
||||
@@ -205,9 +225,10 @@ Notes:
|
||||
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
|
||||
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
|
||||
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
|
||||
| `rate_limit_up_bps` | `u64` | no | Per-user upload rate limit in bytes per second. |
|
||||
| `rate_limit_down_bps` | `u64` | no | Per-user download rate limit in bytes per second. |
|
||||
| `rate_limit_up_bps` | `u64` | no | Per-user upload rate limit in bits per second. |
|
||||
| `rate_limit_down_bps` | `u64` | no | Per-user download rate limit in bits per second. |
|
||||
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
|
||||
| `enabled` | `bool` | no | User enable flag. Missing means enabled. `false` persists a disabled override. |
|
||||
|
||||
### `PatchUserRequest`
|
||||
| Field | Type | Required | Description |
|
||||
@@ -217,9 +238,10 @@ Notes:
|
||||
| `max_tcp_conns` | `usize|null` | no | Per-user concurrent TCP limit; `null` removes the per-user override. |
|
||||
| `expiration_rfc3339` | `string|null` | no | RFC3339 expiration timestamp; `null` removes the expiration. |
|
||||
| `data_quota_bytes` | `u64|null` | no | Per-user traffic quota; `null` removes the per-user quota. |
|
||||
| `rate_limit_up_bps` | `u64|null` | no | Per-user upload rate limit in bytes per second; `null` removes the upload direction limit. |
|
||||
| `rate_limit_down_bps` | `u64|null` | no | Per-user download rate limit in bytes per second; `null` removes the download direction limit. |
|
||||
| `rate_limit_up_bps` | `u64|null` | no | Per-user upload rate limit in bits per second; `null` removes the upload direction limit. |
|
||||
| `rate_limit_down_bps` | `u64|null` | no | Per-user download rate limit in bits per second; `null` removes the download direction limit. |
|
||||
| `max_unique_ips` | `usize|null` | no | Per-user unique source IP limit; `null` removes the per-user override. |
|
||||
| `enabled` | `bool|null` | no | `false` disables the user. `true` or `null` removes the disabled override, so the user is enabled. |
|
||||
|
||||
### `access.user_source_deny` via API
|
||||
- In current API surface, per-user deny-list is **not** exposed as a dedicated field in `CreateUserRequest` / `PatchUserRequest`.
|
||||
@@ -236,6 +258,20 @@ alice = ["203.0.113.0/24", "2001:db8:abcd::/48"]
|
||||
bob = ["198.51.100.42/32"]
|
||||
```
|
||||
|
||||
### `PatchConfigRequest`
|
||||
|
||||
A sparse JSON object containing only the top-level config sections to modify. Each key must be one of the editable sections (`general`, `timeouts`, `censorship`, `upstreams`, `show_link`, `dc_overrides`). Tables within a section are deep-merged field-by-field into the existing config; arrays and scalar values replace the existing value wholesale. Untouched sections and file comments are preserved.
|
||||
|
||||
**Rejected keys:**
|
||||
- `access` → `400 access_not_editable` (users/secrets are managed via `POST/PATCH /v1/users`).
|
||||
- `server`, `network`, or any unknown top-level key → `400 section_not_editable`.
|
||||
- An object with no editable keys → `400 bad_request` (empty patch).
|
||||
|
||||
Example — patch only the SNI domain:
|
||||
```json
|
||||
{"censorship": {"tls_domain": "front.example.com"}}
|
||||
```
|
||||
|
||||
### `RotateSecretRequest`
|
||||
| Field | Type | Required | Description |
|
||||
| --- | --- | --- | --- |
|
||||
@@ -245,6 +281,35 @@ An empty request body is accepted and generates a new secret automatically.
|
||||
|
||||
## Response Data Contracts
|
||||
|
||||
### `ConfigData`
|
||||
|
||||
Returned by `GET /v1/config` as the envelope `data`. The fields are exactly the editable TOML sections. The current revision is returned in the envelope `revision` field (same value as `config_hash` in `SystemInfoData`), **not** inside `data`.
|
||||
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `general` | `object?` | `[general]` section, if present in config. |
|
||||
| `timeouts` | `object?` | `[timeouts]` section, if present. |
|
||||
| `censorship` | `object?` | `[censorship]` section, if present. |
|
||||
| `upstreams` | `object?` | `[upstreams]` section, if present. |
|
||||
| `show_link` | `object?` | `[show_link]` section, if present. |
|
||||
| `dc_overrides` | `object?` | `[dc_overrides]` section, if present. |
|
||||
|
||||
Sections absent from the config file are absent from the response (not `null`). Only the editable sections above are returned; `access` (users/secrets), `server` (carries the API `auth_header` and per-node identity), and `network` (per-node addresses) are always excluded.
|
||||
|
||||
### `PatchConfigResponse`
|
||||
|
||||
Returned by `PATCH /v1/config` on success (`200`, or `202` when a reload was accepted).
|
||||
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `revision` | `string` | SHA-256 hex of the config file after the patch was written. |
|
||||
| `restart_required` | `bool` | Legacy classifier result: `true` when the old file watcher alone cannot apply every changed field. Use `runtime_reload_required` and `process_restart_required` for new integrations. |
|
||||
| `runtime_reload_required` | `bool` | `true` when full effect requires a Maestro runtime-generation reload rather than the legacy hot-field overlay. |
|
||||
| `process_restart_required` | `bool` | `true` when process-owned sockets or paths changed and remain deferred after an in-process reload. |
|
||||
| `deferred_process_fields` | `string[]` | Process-owned fields that the active process cannot rebind during generation activation. |
|
||||
| `changed` | `string[]` | Top-level section names that differed between the old and new config (e.g. `["censorship"]`). |
|
||||
| `reload` | `ReloadAccepted?` | Present only when the patch included a valid reload query and Maestro accepted the operation. |
|
||||
|
||||
### `HealthData`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
@@ -269,6 +334,7 @@ An empty request body is accepted and generates a new secret automatically.
|
||||
| `connections_bad_total` | `u64` | Failed/invalid client connections. |
|
||||
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
|
||||
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
|
||||
| `handshake_failures_by_stage` | `StageCount[]` | Handshake failures grouped by state-machine stage. |
|
||||
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
|
||||
| `configured_users` | `usize` | Number of configured users in config. |
|
||||
|
||||
@@ -278,6 +344,38 @@ An empty request body is accepted and generates a new secret automatically.
|
||||
| `class` | `string` | Failure class label. |
|
||||
| `total` | `u64` | Counter value for this class. |
|
||||
|
||||
#### `StageCount`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `stage` | `string` | State-machine stage label. |
|
||||
| `total` | `u64` | Counter value for this stage. |
|
||||
|
||||
#### Handshake failure stage diagnostics
|
||||
|
||||
`handshake_failures_by_class` and `telemt_handshake_failures_by_class_total` describe the error kind. `handshake_failures_by_stage` and `telemt_handshake_failures_by_stage_total` describe where the same failure happened in the handshake state machine.
|
||||
|
||||
This does not add a DPI verdict or any protocol decision. The stage is derived from the existing Telemt handshake control flow and is counted only when the existing handshake failure or timeout accounting path is reached.
|
||||
|
||||
Fixed stage labels:
|
||||
|
||||
| Stage | Meaning |
|
||||
| --- | --- |
|
||||
| `first_packet_prelude` | Reading the first 5 bytes before selecting the TLS or direct branch. |
|
||||
| `tls_clienthello_body` | Reading the TLS ClientHello body after the TLS record header. |
|
||||
| `tls_core` | Running the TLS-F handshake/auth flow. |
|
||||
| `tls_post_serverhello_mtproto` | Waiting for the 64-byte MTProto handshake after TLS ServerHello. |
|
||||
| `direct_mtproto` | Reading the direct classic/secure 64-byte MTProto handshake. |
|
||||
|
||||
Example:
|
||||
|
||||
```text
|
||||
telemt_handshake_failures_by_class_total{class="expected_64_got_0_unexpected_eof"} 3
|
||||
telemt_handshake_failures_by_stage_total{stage="direct_mtproto"} 1
|
||||
telemt_handshake_failures_by_stage_total{stage="tls_post_serverhello_mtproto"} 2
|
||||
```
|
||||
|
||||
This means the same EOF-while-reading-64-bytes failure happened once in the direct MTProto path and twice after TLS ServerHello.
|
||||
|
||||
### `SystemInfoData`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
@@ -807,6 +905,43 @@ An empty request body is accepted and generates a new secret automatically.
|
||||
| `event_type` | `string` | Event kind identifier. |
|
||||
| `context` | `string` | Context text (truncated to implementation-defined max length). |
|
||||
|
||||
### `RuntimeEdgeTlsFingerprintsData`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `enabled` | `bool` | Endpoint availability under `runtime_edge_enabled`. |
|
||||
| `reason` | `string?` | `feature_disabled` when endpoint is disabled. |
|
||||
| `generated_at_epoch_secs` | `u64` | Snapshot generation timestamp. |
|
||||
| `data` | `RuntimeEdgeTlsFingerprintsPayload?` | Null when unavailable. |
|
||||
|
||||
#### `RuntimeEdgeTlsFingerprintsPayload`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `limit` | `usize` | Effective Top-N row count. |
|
||||
| `retention_secs` | `u64` | In-memory retention window, derived from `general.beobachten_minutes`. |
|
||||
| `capacity` | `usize` | Maximum retained fingerprint buckets. |
|
||||
| `dropped_total` | `u64` | Buckets dropped because the collector was full. |
|
||||
| `parse_error_total` | `u64` | Complete ClientHello records that could not be fingerprinted. |
|
||||
| `by_fingerprint` | `RuntimeEdgeTlsFingerprintRow[]` | Global JA3/JA4 leaderboard. |
|
||||
| `by_ip` | `RuntimeEdgeTlsFingerprintRow[]` | Source-IP scoped leaderboard. |
|
||||
| `by_cidr` | `RuntimeEdgeTlsFingerprintRow[]` | Source CIDR scoped leaderboard (`/24` for IPv4, `/56` for IPv6). |
|
||||
| `by_user` | `RuntimeEdgeTlsFingerprintRow[]` | Authenticated user scoped leaderboard. |
|
||||
|
||||
#### `RuntimeEdgeTlsFingerprintRow`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `scope` | `string?` | IP, CIDR, or username; absent in `by_fingerprint`. |
|
||||
| `ja3` | `string` | JA3 MD5 hash. |
|
||||
| `ja3_raw` | `string` | Raw JA3 field string. |
|
||||
| `ja4` | `string` | JA4 TLS client fingerprint. |
|
||||
| `ja4_raw` | `string` | Raw JA4 material used for the hashed parts. |
|
||||
| `total` | `u64` | Complete ClientHello observations for this bucket. |
|
||||
| `auth_success` | `u64` | TLS-authenticated observations for this bucket. |
|
||||
| `bad_or_probe` | `u64` | Complete ClientHello observations later classified as bad/probe. |
|
||||
| `first_seen_epoch_secs` | `u64` | First observation timestamp. |
|
||||
| `last_seen_epoch_secs` | `u64` | Last observation timestamp. |
|
||||
|
||||
JA3 follows the Salesforce ClientHello field order. JA4 follows the FoxIO TLS-client `a_b_c` format; GREASE values are excluded and no high-cardinality Prometheus labels are emitted for fingerprints.
|
||||
|
||||
### `ZeroAllData`
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
@@ -825,6 +960,7 @@ An empty request body is accepted and generates a new secret automatically.
|
||||
| `connections_bad_total` | `u64` | Failed/invalid connections. |
|
||||
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
|
||||
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
|
||||
| `handshake_failures_by_stage` | `StageCount[]` | Handshake failures grouped by state-machine stage. |
|
||||
| `handshake_timeouts_total` | `u64` | Handshake timeouts. |
|
||||
| `accept_permit_timeout_total` | `u64` | Listener admission permit acquisition timeouts. |
|
||||
| `configured_users` | `usize` | Configured user count. |
|
||||
@@ -1165,13 +1301,14 @@ An empty request body is accepted and generates a new secret automatically.
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `username` | `string` | Username. |
|
||||
| `enabled` | `bool` | Effective user enable flag. Missing config entry is reported as `true`. |
|
||||
| `in_runtime` | `bool` | Whether current runtime config already contains this user. |
|
||||
| `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). |
|
||||
| `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. |
|
||||
| `expiration_rfc3339` | `string?` | Optional expiration timestamp. |
|
||||
| `data_quota_bytes` | `u64?` | Optional data quota. |
|
||||
| `rate_limit_up_bps` | `u64?` | Optional upload rate limit in bytes per second. |
|
||||
| `rate_limit_down_bps` | `u64?` | Optional download rate limit in bytes per second. |
|
||||
| `rate_limit_up_bps` | `u64?` | Optional upload rate limit in bits per second. |
|
||||
| `rate_limit_down_bps` | `u64?` | Optional download rate limit in bits per second. |
|
||||
| `max_unique_ips` | `usize?` | Optional unique IP limit. |
|
||||
| `current_connections` | `u64` | Current live connections. |
|
||||
| `active_unique_ips` | `usize` | Current active unique source IPs. |
|
||||
@@ -1232,13 +1369,159 @@ Link generation uses active config and enabled modes:
|
||||
| `used_bytes` | `u64` | Current used bytes after reset; always `0` on success. |
|
||||
| `last_reset_epoch_secs` | `u64` | Unix timestamp of the reset operation. |
|
||||
|
||||
## Config Endpoints
|
||||
|
||||
### `GET /v1/config`
|
||||
|
||||
Returns the current editable config sections as TOML-shaped JSON, plus the current revision. The `access` section (users and secrets) is always stripped and never appears in the response.
|
||||
|
||||
**Auth:** requires `Authorization` header when `auth_header` is configured (same as all other endpoints).
|
||||
|
||||
**Success `200` response body** (`data` field of the standard envelope):
|
||||
```json
|
||||
{
|
||||
"revision": "<sha256-hex>",
|
||||
"censorship": {"tls_domain": "front.example.com"},
|
||||
"general": {"log_level": "normal"}
|
||||
}
|
||||
```
|
||||
|
||||
Top-level sections absent from the config file are absent from the response. Only `GET` and `PATCH` are accepted; any other method returns `405 Method Not Allowed` with `Allow: GET, PATCH`.
|
||||
|
||||
---
|
||||
|
||||
### `PATCH /v1/config`
|
||||
|
||||
Applies a sparse patch to the editable config sections. The merged config is fully validated before writing; if validation fails the file is not modified.
|
||||
|
||||
**Auth:** requires `Authorization` header when `auth_header` is configured.
|
||||
|
||||
**Headers:**
|
||||
|
||||
| Header | Required | Description |
|
||||
| --- | --- | --- |
|
||||
| `Authorization` | when configured | Same token as all other endpoints. |
|
||||
| `Content-Type: application/json` | recommended | Not enforced, but body must be valid JSON. |
|
||||
| `If-Match: <revision>` | no | Optimistic concurrency. `<revision>` is the `revision` value from `GET /v1/config` or `config_hash` from `GET /v1/system/info`. If supplied and it does not match the current on-disk revision, returns `409 revision_conflict`. If omitted, the patch applies unconditionally. |
|
||||
|
||||
**Editable sections:** `general`, `timeouts`, `censorship`, `upstreams`, `show_link`, `dc_overrides`.
|
||||
|
||||
**Rejected keys and their error codes:**
|
||||
|
||||
| Key | HTTP | `error.code` |
|
||||
| --- | --- | --- |
|
||||
| `access` | `400` | `access_not_editable` |
|
||||
| `server`, `network`, or any unknown key | `400` | `section_not_editable` |
|
||||
| Object with no editable key | `400` | `bad_request` |
|
||||
|
||||
**Merge semantics:** tables are deep-merged field-by-field; arrays and scalar values replace the existing value wholesale. File comments and untouched sections are preserved.
|
||||
|
||||
**Validation:** the merged config is deserialized into the full `ProxyConfig` type and validated before writing. Failures return `400` with a descriptive message; the file is not modified.
|
||||
|
||||
**Read-only mode:** returns `403 read_only` when the API runs with `read_only = true`.
|
||||
|
||||
**Optional in-process reload query:**
|
||||
|
||||
| Query | Required | Description |
|
||||
| --- | --- | --- |
|
||||
| `reload=instant` | no | Activates a new generation and cancels sessions owned by the previous generation. |
|
||||
| `reload=drain` | no | Activates a new generation and lets old sessions finish until `timeout_secs`. |
|
||||
| `timeout_secs=1..3600` | for `reload=drain` | Bounded old-generation drain interval. Invalid with `reload=instant`. |
|
||||
| `failure_policy=keep_new\|rollback` | no | Defaults to `keep_new`. `rollback` applies only through the activation barrier, before old-generation teardown. |
|
||||
|
||||
Without a `reload` query parameter, the endpoint preserves the legacy behavior: it writes the patch and the file watcher applies only supported hot fields.
|
||||
|
||||
**Success `200` or `202` response body** (`data` field of the standard envelope):
|
||||
```json
|
||||
{
|
||||
"revision": "<new-sha256-hex>",
|
||||
"restart_required": true,
|
||||
"runtime_reload_required": true,
|
||||
"process_restart_required": false,
|
||||
"deferred_process_fields": [],
|
||||
"changed": ["censorship"],
|
||||
"reload": {
|
||||
"reload_id": 7,
|
||||
"target_generation": 2,
|
||||
"config_revision": "<new-sha256-hex>",
|
||||
"state": "accepted",
|
||||
"mode": "instant",
|
||||
"failure_policy": "keep_new"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- `revision` — SHA-256 hex of the config file after the write.
|
||||
- `restart_required` — legacy file-watcher classification retained for compatibility.
|
||||
- `runtime_reload_required` — reports whether a full Maestro generation reload is needed for runtime effect.
|
||||
- `process_restart_required` and `deferred_process_fields` — report process-owned sockets or paths that remain unchanged by an in-process reload.
|
||||
- `changed` — list of top-level section names that differed.
|
||||
- `reload` — accepted operation metadata; omitted when no reload query was supplied.
|
||||
|
||||
**Status codes:**
|
||||
|
||||
| HTTP | `error.code` | Condition |
|
||||
| --- | --- | --- |
|
||||
| `200` | — | Patch applied successfully. |
|
||||
| `202` | — | Patch applied and runtime reload accepted. |
|
||||
| `400` | `bad_request` | Invalid JSON, empty patch, or config validation/deserialization failure. |
|
||||
| `400` | `access_not_editable` | Patch contains an `access` key. |
|
||||
| `400` | `section_not_editable` | Patch contains `server`, `network`, or an unknown top-level key. |
|
||||
| `401` | `unauthorized` | Missing or invalid `Authorization` header. |
|
||||
| `403` | `read_only` | API is in read-only mode. |
|
||||
| `405` | `method_not_allowed` | Method other than `GET` or `PATCH` used on `/v1/config`. |
|
||||
| `409` | `revision_conflict` | `If-Match` header supplied but does not match current revision. |
|
||||
| `409` | `reload_in_progress` | Another runtime reload is active; the patch is not written. |
|
||||
| `500` | `internal_error` | I/O or serialization failure. |
|
||||
|
||||
**curl example:**
|
||||
```bash
|
||||
# get current revision
|
||||
curl -s -H "Authorization: <token>" http://127.0.0.1:<api>/v1/system/info | jq -r .config_hash
|
||||
|
||||
# patch the SNI domain with optimistic concurrency
|
||||
curl -s -X PATCH -H "Authorization: <token>" -H "If-Match: <revision>" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"censorship":{"tls_domain":"front.example.com"}}' \
|
||||
'http://127.0.0.1:<api>/v1/config?reload=instant'
|
||||
```
|
||||
|
||||
## Runtime Reload Endpoints
|
||||
|
||||
### `POST /v1/system/reload`
|
||||
|
||||
Loads the current on-disk config under the API mutation lock and submits an immutable config snapshot to Maestro. `If-Match` is optional and uses the same revision contract as `PATCH /v1/config`. An empty body defaults to `{"mode":"instant","failure_policy":"keep_new"}`.
|
||||
|
||||
```json
|
||||
{
|
||||
"mode": "drain",
|
||||
"timeout_secs": 30,
|
||||
"failure_policy": "rollback"
|
||||
}
|
||||
```
|
||||
|
||||
The endpoint returns `202` with `ReloadAccepted`. A concurrent non-terminal reload returns `409 reload_in_progress`. Config parsing or validation failure is reported before a command is submitted.
|
||||
|
||||
### `GET /v1/system/reload/{id}`
|
||||
|
||||
Returns `ReloadStatus` with `state` equal to `accepted`, `preparing`, `activating`, `draining`, `succeeded`, `rolled_back`, or `failed`. Terminal statuses include `finished_at_epoch_secs`; failures include `error`. Successful activation may include `warnings` for old-generation cleanup failures and `deferred_process_fields` for process-owned settings.
|
||||
|
||||
Runtime generation activation rebuilds statistics, upstream routing, replay and buffer state, TLS-front cache, IP tracking, admission/route state, and Middle-End orchestration. Per-user quota accounting is process-scoped and remains continuous across generations. API, metrics, client TCP/Unix listeners, PID ownership, and logging remain process-scoped; changed bind/path fields are reported as deferred and do not cause Maestro to invoke systemd, containerd, or another process supervisor.
|
||||
|
||||
Reload preparation requires every configured TLS-front domain to have a non-default cached profile and requires a ready Middle-End pool when direct fallback is disabled. A candidate that does not satisfy either readiness condition fails without replacing the active generation.
|
||||
|
||||
The revision is verified again after preparation. With `failure_policy=rollback`, a changed revision or revision read failure rolls the candidate back; with `failure_policy=keep_new`, the condition is reported in `warnings` and activation continues.
|
||||
|
||||
## Mutation Semantics
|
||||
|
||||
| Endpoint | Notes |
|
||||
| --- | --- |
|
||||
| `PATCH /v1/config` | Deep-merges and validates the patch, writes touched sections via atomic `tmp + rename`, and optionally submits the exact written revision for an in-process Maestro reload. |
|
||||
| `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). |
|
||||
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged; explicit `null` removes optional per-user entries. The write path updates only affected `access.*` TOML tables. |
|
||||
| `POST /v1/users/{username}/rotate-secret` | Replaces the user's secret with a provided valid 32-hex value or a generated value, then returns the effective secret in `CreateUserResponse`. |
|
||||
| `POST /v1/users/{username}/enable` | Enables the user idempotently by removing the `access.user_enabled[username]` override and updating the runtime admission state immediately. |
|
||||
| `POST /v1/users/{username}/disable` | Disables the user idempotently by writing `access.user_enabled[username] = false`, updating runtime admission immediately, and cancelling active sessions for that username. |
|
||||
| `POST /v1/users/{username}/reset-quota` | Resets the runtime quota counter for the route username, persists quota state to `general.quota_state_path`, and does not modify user config. |
|
||||
| `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. |
|
||||
|
||||
@@ -1282,6 +1565,7 @@ Additional runtime endpoint behavior:
|
||||
| `/v1/runtime/me-selftest` | No | ME pool unavailable => `enabled=false`, `reason=source_unavailable` | `enabled=true`, full payload |
|
||||
| `/v1/runtime/connections/summary` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Recompute lock contention with no cache entry => `enabled=true`, `reason=source_unavailable` | `enabled=true`, full payload |
|
||||
| `/v1/runtime/events/recent` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload |
|
||||
| `/v1/runtime/tls-fingerprints` | `runtime_edge_enabled=false` => `enabled=false`, `reason=feature_disabled` | Not used in current implementation | `enabled=true`, full payload |
|
||||
|
||||
## ME Fallback Behavior Exposed Via API
|
||||
|
||||
|
||||
507
docs/Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md
Normal file
507
docs/Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md
Normal file
@@ -0,0 +1,507 @@
|
||||
# JA3 и JA4 анализ в Telemt
|
||||
|
||||
Этот документ описывает, как использовать JA3/JA4 telemetry в Telemt для диагностики блокировок, которые происходят на основе TLS ClientHello, особенно JA4 TLS client fingerprint.
|
||||
|
||||
Цель документа практическая: помочь оператору понять, какой клиентский TLS-отпечаток реально доходит до Telemt, как он распределён по IP/CIDR/пользователям, и как отделить JA4-based фильтрацию от блокировки по IP, SNI, домену, server flight или активному сканированию.
|
||||
|
||||
## Коротко
|
||||
|
||||
JA3 и JA4 описывают форму TLS ClientHello. ClientHello отправляет клиент, поэтому JA3/JA4 в этом контексте являются fingerprint'ами клиентской TLS-реализации, а не Telemt как сервера.
|
||||
|
||||
Telemt собирает JA3/JA4 только из уже прочитанного полного ClientHello:
|
||||
|
||||
- без packet capture;
|
||||
- без MITM;
|
||||
- без расшифровки TLS;
|
||||
- без дополнительных сетевых чтений;
|
||||
- без Prometheus labels с высокой кардинальностью;
|
||||
- с ограниченным in-memory TTL/cap collector.
|
||||
|
||||
Собранные данные доступны:
|
||||
|
||||
- через API: `GET /v1/runtime/tls-fingerprints`;
|
||||
- через `/beobachten`, если `general.beobachten=true`.
|
||||
|
||||
Основная польза:
|
||||
|
||||
- увидеть, какие JA4 реально используют клиенты;
|
||||
- понять, один ли fingerprint страдает у всех пользователей;
|
||||
- отделить проблему клиента от проблемы IP/ASN/домена;
|
||||
- увидеть, доходят ли проблемные соединения до Telemt вообще;
|
||||
- сравнить successful TLS-auth и bad/probe поток для одного fingerprint;
|
||||
- собрать evidence для последующего изменения клиента, маршрута или deployment-профиля.
|
||||
|
||||
## Что такое JA3
|
||||
|
||||
JA3 - старый и широко совместимый способ получить hash от TLS ClientHello.
|
||||
|
||||
JA3 строится из ClientHello fields:
|
||||
|
||||
```text
|
||||
SSLVersion,Cipher,SSLExtension,EllipticCurve,EllipticCurvePointFormat
|
||||
```
|
||||
|
||||
Значения внутри полей записываются в порядке, в котором они пришли в ClientHello. GREASE values исключаются. Итоговая строка хэшируется MD5, поэтому в API есть два поля:
|
||||
|
||||
- `ja3` - MD5 hash;
|
||||
- `ja3_raw` - исходная строка, из которой получен hash.
|
||||
|
||||
Практическое значение JA3 в 2026 году ограничено тем, что современные TLS-клиенты и браузерные стеки могут менять порядок extensions. Поэтому JA3 полезен как совместимый исторический сигнал, но для диагностики современных блокировок обычно важнее JA4.
|
||||
|
||||
## Что такое JA4
|
||||
|
||||
JA4 TLS client fingerprint - более структурированный fingerprint ClientHello.
|
||||
|
||||
JA4 в Telemt считается для TLS-over-TCP ClientHello и имеет форму:
|
||||
|
||||
```text
|
||||
t<version><sni_marker><cipher_count><extension_count><alpn_marker>_<cipher_hash>_<extension_hash>
|
||||
```
|
||||
|
||||
Пример:
|
||||
|
||||
```text
|
||||
t13d1516h2_8daaf6152771_e5627efa2ab1
|
||||
```
|
||||
|
||||
Части JA4:
|
||||
|
||||
| Часть | Смысл |
|
||||
| --- | --- |
|
||||
| `t` | TLS over TCP. Telemt сейчас не считает JA4 для QUIC/DTLS. |
|
||||
| `13`, `12`, `11`, `10` | TLS version, предпочтительно из `supported_versions`. |
|
||||
| `d` / `i` | Есть SNI domain (`d`) или SNI отсутствует (`i`). |
|
||||
| `15` | Количество cipher suites без GREASE, capped до `99`. |
|
||||
| `16` | Количество extensions без GREASE, capped до `99`. |
|
||||
| `h2`, `h1`, `00` | ALPN marker: первый и последний символ первого ALPN value или `00`. |
|
||||
| `cipher_hash` | SHA256 от отсортированного списка ciphers, первые 12 hex chars. |
|
||||
| `extension_hash` | SHA256 от отсортированных extensions плюс signature algorithms, первые 12 hex chars. |
|
||||
|
||||
Важное отличие JA4 от JA3: JA4 нормализует часть полей, поэтому он устойчивее к простому изменению порядка extensions. Это делает JA4 удобным для фильтров и одновременно полезным для диагностики таких фильтров.
|
||||
|
||||
## Где Telemt видит ClientHello
|
||||
|
||||
В TLS/FakeTLS режиме Telemt получает первые bytes соединения и определяет, похоже ли оно на TLS handshake. Если record является полным ClientHello и проходит bounds checks, Telemt один раз парсит его для JA3/JA4.
|
||||
|
||||
Дальше возможны три исхода:
|
||||
|
||||
1. **Успешный MTProxy/FakeTLS клиент**
|
||||
- Telemt принимает TLS-auth;
|
||||
- fingerprint записывается в global/IP/CIDR scopes;
|
||||
- после успешной TLS-auth Telemt добавляет user scope.
|
||||
|
||||
2. **Bad client или probe**
|
||||
- ClientHello полный, но auth не проходит;
|
||||
- fingerprint записывается в global/IP/CIDR scopes;
|
||||
- user scope не записывается;
|
||||
- `bad_or_probe` увеличивается.
|
||||
|
||||
3. **Неполный или обрезанный ClientHello**
|
||||
- fingerprint не считается;
|
||||
- такие случаи остаются в существующих bad-class counters.
|
||||
|
||||
Если фильтр режет трафик до того, как TCP connection или ClientHello дошли до процесса Telemt, Telemt не увидит этот fingerprint. Это важнейшее диагностическое отличие: отсутствие fingerprint'а во время жалобы пользователя часто означает блокировку до приложения, а не проблему внутри Telemt.
|
||||
|
||||
## Включение сбора
|
||||
|
||||
Collector включается, когда включён хотя бы один потребитель:
|
||||
|
||||
```toml
|
||||
[general]
|
||||
beobachten = true
|
||||
beobachten_minutes = 10
|
||||
```
|
||||
|
||||
или:
|
||||
|
||||
```toml
|
||||
[server.api]
|
||||
runtime_edge_enabled = true
|
||||
runtime_edge_top_n = 50
|
||||
```
|
||||
|
||||
Практически:
|
||||
|
||||
- для файлового/metrics endpoint анализа достаточно `general.beobachten=true`;
|
||||
- для API snapshot нужен `server.api.runtime_edge_enabled=true`;
|
||||
- `general.beobachten_minutes` задаёт retention window для fingerprint buckets;
|
||||
- `server.api.runtime_edge_top_n` задаёт default Top-N размер API snapshot.
|
||||
|
||||
## API snapshot
|
||||
|
||||
Endpoint:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints
|
||||
```
|
||||
|
||||
С явным лимитом:
|
||||
|
||||
```bash
|
||||
curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100'
|
||||
```
|
||||
|
||||
Если API защищён header'ом:
|
||||
|
||||
```bash
|
||||
curl -s \
|
||||
-H 'Authorization: Bearer YOUR_TOKEN' \
|
||||
'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100'
|
||||
```
|
||||
|
||||
Если `runtime_edge_enabled=false`, endpoint возвращает payload с:
|
||||
|
||||
```json
|
||||
{
|
||||
"enabled": false,
|
||||
"reason": "feature_disabled"
|
||||
}
|
||||
```
|
||||
|
||||
### Структура payload
|
||||
|
||||
Основные поля:
|
||||
|
||||
| Поле | Смысл |
|
||||
| --- | --- |
|
||||
| `retention_secs` | Текущее TTL окно collector'а. |
|
||||
| `capacity` | Максимум retained buckets. |
|
||||
| `dropped_total` | Сколько новых buckets отброшено из-за cap. |
|
||||
| `parse_error_total` | Сколько полных ClientHello не удалось распарсить. |
|
||||
| `by_fingerprint` | Top fingerprints глобально. |
|
||||
| `by_ip` | Top fingerprints по exact source IP. |
|
||||
| `by_cidr` | Top fingerprints по source prefix: IPv4 `/24`, IPv6 `/56`. |
|
||||
| `by_user` | Top fingerprints по authenticated user. |
|
||||
|
||||
Строка snapshot:
|
||||
|
||||
| Поле | Смысл |
|
||||
| --- | --- |
|
||||
| `scope` | IP, CIDR или username. В `by_fingerprint` отсутствует. |
|
||||
| `ja3` | JA3 hash. |
|
||||
| `ja3_raw` | Raw JA3 string. |
|
||||
| `ja4` | JA4 TLS client fingerprint. |
|
||||
| `ja4_raw` | Raw JA4 material. |
|
||||
| `total` | Сколько полных ClientHello попало в этот bucket. |
|
||||
| `auth_success` | Сколько из них успешно прошли TLS-auth. |
|
||||
| `bad_or_probe` | Сколько были bad/probe после полного ClientHello. |
|
||||
| `first_seen_epoch_secs` | Первый timestamp bucket'а. |
|
||||
| `last_seen_epoch_secs` | Последний timestamp bucket'а. |
|
||||
|
||||
### Быстрый просмотр через jq
|
||||
|
||||
Top JA4 глобально:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints \
|
||||
| jq -r '.data.data.by_fingerprint[] | [.ja4, .total, .auth_success, .bad_or_probe] | @tsv'
|
||||
```
|
||||
|
||||
Top JA4 по пользователям:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100 \
|
||||
| jq -r '.data.data.by_user[] | [.scope, .ja4, .total, .auth_success] | @tsv'
|
||||
```
|
||||
|
||||
Top JA4 по CIDR:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=100 \
|
||||
| jq -r '.data.data.by_cidr[] | [.scope, .ja4, .total, .auth_success, .bad_or_probe] | @tsv'
|
||||
```
|
||||
|
||||
Ошибки парсинга и drops:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9091/v1/runtime/tls-fingerprints \
|
||||
| jq '.data.data | {retention_secs, capacity, dropped_total, parse_error_total}'
|
||||
```
|
||||
|
||||
## Beobachten output
|
||||
|
||||
Если включён endpoint metrics, `/beobachten` содержит обычные forensic buckets и, когда есть данные, append-only секцию TLS fingerprints:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9090/beobachten
|
||||
```
|
||||
|
||||
Фрагмент:
|
||||
|
||||
```text
|
||||
[tls_fingerprints]
|
||||
retention_secs=600 capacity=65536 dropped_total=0 parse_error_total=0
|
||||
[tls_fingerprints.by_fingerprint]
|
||||
ja4=t13d1516h2_8daaf6152771_e5627efa2ab1 ja3=... total=42 auth_success=41 bad_or_probe=1 first_seen=... last_seen=...
|
||||
[tls_fingerprints.by_cidr]
|
||||
scope=203.0.113.0/24 ja4=t13d1516h2_8daaf6152771_e5627efa2ab1 ja3=... total=10 auth_success=10 bad_or_probe=0 first_seen=... last_seen=...
|
||||
```
|
||||
|
||||
`/beobachten` удобен для быстрой операторской диагностики без API client. API удобнее для автоматической корреляции.
|
||||
|
||||
## Как анализировать JA4-based блокировку
|
||||
|
||||
### 1. Зафиксировать симптом
|
||||
|
||||
Перед анализом нужно записать:
|
||||
|
||||
- какие пользователи жалуются;
|
||||
- какая версия Telegram client используется;
|
||||
- какая платформа: Desktop, Android, iOS;
|
||||
- какой источник сети: mobile ISP, home ISP, corporate network, country/region;
|
||||
- работает ли тот же пользователь через другой network path;
|
||||
- работает ли другой пользователь с того же IP/CIDR;
|
||||
- видит ли Telemt новые ClientHello от проблемного пользователя в момент попытки.
|
||||
|
||||
JA4 без контекста почти всегда недостаточен. Фильтры часто используют сочетание:
|
||||
|
||||
- JA4;
|
||||
- destination IP;
|
||||
- SNI;
|
||||
- порт;
|
||||
- ASN/source network;
|
||||
- rate или connection pattern;
|
||||
- reputation домена/IP;
|
||||
- active probing result.
|
||||
|
||||
### 2. Проверить, доходит ли ClientHello до Telemt
|
||||
|
||||
Во время попытки подключения проблемного пользователя смотрите:
|
||||
|
||||
```bash
|
||||
curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=200' \
|
||||
| jq '.data.data.by_user, .data.data.by_ip, .data.data.by_cidr'
|
||||
```
|
||||
|
||||
Интерпретация:
|
||||
|
||||
| Наблюдение | Вероятный вывод |
|
||||
| --- | --- |
|
||||
| Нет новых rows для IP/CIDR пользователя | Блокировка до Telemt: routing, firewall, ISP/DPI drop, IP block, SYN/TCP reset, UDP/TCP path issue. |
|
||||
| Есть `by_ip`/`by_cidr`, но нет `by_user` | ClientHello дошёл, но TLS-auth/MTProxy layer не дошёл до успешного пользователя. Возможны bad key, probe, wrong client, active scanner, обрыв после ClientHello. |
|
||||
| Есть `by_user.auth_success` | Клиентский JA4 дошёл и был принят Telemt. Если пользователь всё равно видит проблему, искать нужно дальше: relay path, Telegram upstream, quota, route mode, session cancellation, ME/direct routing. |
|
||||
| Резко растёт `bad_or_probe` для одного JA4 | Вероятны сканеры или неправильные клиенты с тем же fingerprint family. |
|
||||
|
||||
### 3. Сравнить working и blocked случаи
|
||||
|
||||
Снимите snapshot во время working case и blocked case:
|
||||
|
||||
```bash
|
||||
curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=500' > tls-fp-working.json
|
||||
curl -s 'http://127.0.0.1:9091/v1/runtime/tls-fingerprints?limit=500' > tls-fp-blocked.json
|
||||
```
|
||||
|
||||
Сравните:
|
||||
|
||||
- появился ли тот же `ja4` в blocked сети;
|
||||
- меняется ли `ja4` между версиями клиента;
|
||||
- меняется ли только IP/CIDR при том же `ja4`;
|
||||
- есть ли `auth_success` для того же `ja4` из других сетей;
|
||||
- отличается ли `bad_or_probe` между сетями.
|
||||
|
||||
Ключевая матрица:
|
||||
|
||||
| Working JA4 | Blocked JA4 | Вывод |
|
||||
| --- | --- | --- |
|
||||
| Same | Same, но blocked network не доходит до Telemt | Вероятна фильтрация по JA4 + destination/IP/SNI/network до приложения. |
|
||||
| Same | Same, доходит и `auth_success>0` | JA4 ClientHello не является точкой отказа; искать post-auth проблему. |
|
||||
| Different | Blocked только один JA4 | Вероятен client-version/platform-specific fingerprint block. |
|
||||
| Same | `bad_or_probe` растёт, `auth_success=0` | Возможно, доходит не тот клиент/secret или фильтр/прокси ломает поток после ClientHello. |
|
||||
|
||||
### 4. Разделить client JA4 и server fingerprint
|
||||
|
||||
JA4 ClientHello - это клиентская сторона. Настройки Telemt вроде TLS-front server flight, `mask_host`, ticket-tail или CCS replay не меняют ClientHello, который отправляет Telegram client.
|
||||
|
||||
Если фильтр принимает решение строго после ClientHello, то серверные улучшения могут не помочь. В этом случае полезные действия:
|
||||
|
||||
- проверить обновление Telegram client;
|
||||
- сравнить платформы и версии клиента;
|
||||
- проверить, меняется ли JA4 на другой версии;
|
||||
- проверить, блокируется ли тот же JA4 к другому destination;
|
||||
- проверить, блокируется ли другой JA4 к тому же Telemt IP/SNI;
|
||||
- собрать evidence для client-side fingerprint fix.
|
||||
|
||||
Если ClientHello проходит, а блокировка возникает после server response, тогда уже важны:
|
||||
|
||||
- форма FakeTLS server flight;
|
||||
- TLS front profile fidelity;
|
||||
- `mask_host` поведение для non-auth clients;
|
||||
- certificate/provenance fallback для сканеров;
|
||||
- TCP relay behavior;
|
||||
- upstream route к Telegram.
|
||||
|
||||
### 5. Коррелировать с packet capture
|
||||
|
||||
Telemt collector показывает только то, что процесс увидел. Для подтверждения фильтрации до Telemt нужен внешний capture.
|
||||
|
||||
На сервере:
|
||||
|
||||
```bash
|
||||
sudo tcpdump -i any -w telemt-clienthello.pcap host CLIENT_IP and port 443
|
||||
```
|
||||
|
||||
Быстрый tshark вывод ClientHello fields:
|
||||
|
||||
```bash
|
||||
tshark -r telemt-clienthello.pcap -Y "tls.handshake.type == 1" -T fields \
|
||||
-e frame.time_epoch \
|
||||
-e ip.src \
|
||||
-e ip.dst \
|
||||
-e tcp.srcport \
|
||||
-e tcp.dstport \
|
||||
-e tls.handshake.extensions_server_name \
|
||||
-e tls.handshake.extensions_alpn_str
|
||||
```
|
||||
|
||||
Если на клиентской стороне capture видит ClientHello, а серверный capture не видит, проблема в сети между клиентом и сервером. Если серверный capture видит ClientHello, но Telemt API не видит fingerprint, проверьте порт, listener, PROXY protocol, TLS record fragmentation и bounds/errors.
|
||||
|
||||
## Практические сценарии
|
||||
|
||||
### Сценарий A: один JA4 перестал работать у многих пользователей
|
||||
|
||||
Признаки:
|
||||
|
||||
- один `ja4` доминирует в жалобах;
|
||||
- у разных source CIDR нет `auth_success`;
|
||||
- working пользователи используют другой JA4;
|
||||
- обновление клиента меняет поведение.
|
||||
|
||||
Вероятный вывод: фильтр на стороне сети научился распознавать конкретный ClientHello family.
|
||||
|
||||
Действия:
|
||||
|
||||
- сравнить Telegram client versions;
|
||||
- проверить, не используют ли пользователи старые клиенты;
|
||||
- собрать `ja4`, `ja4_raw`, platform/version, source network;
|
||||
- проверить тот же client через другую сеть;
|
||||
- проверить другой client version через ту же сеть.
|
||||
|
||||
### Сценарий B: один CIDR не работает, JA4 обычный
|
||||
|
||||
Признаки:
|
||||
|
||||
- тот же `ja4` успешно работает из других сетей;
|
||||
- проблемный `/24` или `/56` не доходит до Telemt или не получает `auth_success`;
|
||||
- нет общей корреляции по версии клиента.
|
||||
|
||||
Вероятный вывод: проблема не в JA4 alone, а в source network policy или destination reputation.
|
||||
|
||||
Действия:
|
||||
|
||||
- сменить route/VPS/IP;
|
||||
- проверить port;
|
||||
- проверить SNI/domain reputation;
|
||||
- сравнить с другим Telemt endpoint;
|
||||
- смотреть server-side packet capture.
|
||||
|
||||
### Сценарий C: много `bad_or_probe` на одном JA4
|
||||
|
||||
Признаки:
|
||||
|
||||
- `bad_or_probe` высокий;
|
||||
- `by_user` пустой или слабый;
|
||||
- source IP/CIDR разнообразные;
|
||||
- попытки не соответствуют реальным пользователям.
|
||||
|
||||
Вероятный вывод: активное сканирование или нерелевантный TLS traffic с похожим ClientHello.
|
||||
|
||||
Действия:
|
||||
|
||||
- смотреть `/beobachten` по IP classes;
|
||||
- проверить `unknown_tls_sni` и bad-client counters;
|
||||
- убедиться, что fallback `mask_host` отвечает правдоподобно;
|
||||
- не делать вывод о блокировке пользователей только по global `bad_or_probe`.
|
||||
|
||||
### Сценарий D: `auth_success` есть, но пользователь жалуется
|
||||
|
||||
Признаки:
|
||||
|
||||
- fingerprint присутствует в `by_user`;
|
||||
- `auth_success` растёт;
|
||||
- соединение проходит TLS-auth.
|
||||
|
||||
Вероятный вывод: JA4 ClientHello не является причиной отказа в этом случае.
|
||||
|
||||
Действия:
|
||||
|
||||
- проверить user enabled/disabled status;
|
||||
- проверить quota;
|
||||
- проверить direct/ME route;
|
||||
- проверить upstream health;
|
||||
- проверить runtime events;
|
||||
- смотреть relay/session logs.
|
||||
|
||||
## Что нельзя вывести из JA3/JA4
|
||||
|
||||
JA3/JA4 не говорят:
|
||||
|
||||
- почему сеть приняла решение о блокировке;
|
||||
- какой именно vendor DPI используется;
|
||||
- был ли block только по JA4 или по связке JA4+IP+SNI;
|
||||
- что произошло с соединением после TLS-auth;
|
||||
- как выглядит server-side TLS fingerprint;
|
||||
- как ведёт себя HTTP layer после TLS.
|
||||
|
||||
JA3/JA4 также не являются уникальной идентичностью человека. Это fingerprint клиентской TLS-реализации и её настроек. Один fingerprint может быть у большого числа пользователей.
|
||||
|
||||
## Ограничения collector'а Telemt
|
||||
|
||||
- Считается только TLS ClientHello, который полностью дошёл до Telemt.
|
||||
- QUIC/DTLS/HTTP JA4 variants не собираются.
|
||||
- Truncated ClientHello не fingerprint'ится.
|
||||
- User scope появляется только после успешной TLS-auth.
|
||||
- `by_ip` и `by_cidr` отражают source address после нормализации/PROXY protocol path, если он используется.
|
||||
- Collector bounded: при большом количестве уникальных buckets возможен рост `dropped_total`.
|
||||
- Retention зависит от `general.beobachten_minutes`.
|
||||
- Данные runtime in-memory; это snapshot для диагностики, а не долговременное хранилище.
|
||||
|
||||
## Рекомендованный workflow расследования
|
||||
|
||||
1. Включить `runtime_edge_enabled=true` и разумный `runtime_edge_top_n`, например `100`.
|
||||
2. Зафиксировать baseline в период нормальной работы.
|
||||
3. Во время жалобы снять API snapshot и `/beobachten`.
|
||||
4. Сравнить `by_user`, `by_ip`, `by_cidr`, `by_fingerprint`.
|
||||
5. Проверить, появляется ли problematic source в Telemt вообще.
|
||||
6. Если не появляется, снять packet capture на сервере и клиенте.
|
||||
7. Если появляется без `auth_success`, проверить secret/client/proxy link и bad/probe counters.
|
||||
8. Если появляется с `auth_success`, исключить JA4 ClientHello как primary cause и перейти к relay/upstream/runtime диагностике.
|
||||
9. Если один JA4 стабильно коррелирует с block, собрать client version/platform evidence.
|
||||
10. Проверить, меняет ли обновление клиента JA4 и результат подключения.
|
||||
|
||||
## Минимальный incident report
|
||||
|
||||
Для полезного отчёта по JA4-based блокировке соберите:
|
||||
|
||||
```text
|
||||
time_window:
|
||||
telemt_version:
|
||||
server_ip:
|
||||
server_port:
|
||||
tls_domain:
|
||||
mask_host:
|
||||
client_platform:
|
||||
client_version:
|
||||
source_network:
|
||||
source_ip_or_cidr:
|
||||
ja4:
|
||||
ja4_raw:
|
||||
ja3:
|
||||
total:
|
||||
auth_success:
|
||||
bad_or_probe:
|
||||
seen_in_by_user: yes/no
|
||||
seen_in_by_ip: yes/no
|
||||
seen_in_by_cidr: yes/no
|
||||
server_tcpdump_seen_clienthello: yes/no
|
||||
client_tcpdump_sent_clienthello: yes/no
|
||||
works_from_other_network: yes/no
|
||||
works_with_other_client_version: yes/no
|
||||
```
|
||||
|
||||
Этот набор обычно достаточен, чтобы отличить client fingerprint block от IP/SNI/reputation block и от post-auth проблем Telemt.
|
||||
|
||||
## Источники форматов
|
||||
|
||||
- JA3 reference: https://github.com/salesforce/ja3
|
||||
- JA4 technical details: https://github.com/FoxIO-LLC/ja4/blob/main/technical_details/JA4.md
|
||||
|
||||
3574
docs/Config_params/CONFIG_PARAMS.de.md
Normal file
3574
docs/Config_params/CONFIG_PARAMS.de.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -14,6 +14,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
|
||||
# Table of contents
|
||||
- [Top-level keys](#top-level-keys)
|
||||
- [logging](#logging)
|
||||
- [general](#general)
|
||||
- [general.modes](#generalmodes)
|
||||
- [general.links](#generallinks)
|
||||
@@ -35,6 +36,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
| --- | ---- | ------- | ---------- |
|
||||
| [`include`](#include) | `String` (special directive) | — | `✔` |
|
||||
| [`show_link`](#show_link) | `"*"` or `String[]` | `[]` (`ShowLink::None`) | `✘` |
|
||||
| [`logging`](#logging) | Table | default values | `✘` |
|
||||
| [`dc_overrides`](#dc_overrides) | `Map<String, String or String[]>` | `{}` | `✘` |
|
||||
| [`default_dc`](#default_dc) | `u8` | — (effective fallback: `2` in ME routing) | `✘` |
|
||||
| [`beobachten`](#beobachten) | `bool` | `true` | `✘` |
|
||||
@@ -83,6 +85,84 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
default_dc = 2
|
||||
```
|
||||
|
||||
# [logging]
|
||||
|
||||
| Key | Type | Default | Hot-Reload |
|
||||
| --- | ---- | ------- | ---------- |
|
||||
| [`destination`](#loggingdestination) | `"stderr"` / `"syslog"` / `"file"` | `"stderr"` | `✘` |
|
||||
| [`path`](#loggingpath) | `String` | — | `✘` |
|
||||
| [`rotation`](#loggingrotation) | `"never"` / `"minutely"` / `"hourly"` / `"daily"` / `"weekly"` | `"never"` | `✘` |
|
||||
| [`max_size_bytes`](#loggingmax_size_bytes) | `u64` | `0` | `✘` |
|
||||
| [`max_files`](#loggingmax_files) | `usize` | `0` | `✘` |
|
||||
| [`max_age_secs`](#loggingmax_age_secs) | `u64` | `0` | `✘` |
|
||||
|
||||
## logging.destination
|
||||
- **Constraints / validation**: Must be `stderr`, `syslog`, or `file`. `syslog` is supported only on Unix platforms. `file` requires `logging.path`.
|
||||
- **Description**: Selects the runtime log destination. CLI flags override this value.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[logging]
|
||||
destination = "file"
|
||||
path = "/var/log/telemt.log"
|
||||
```
|
||||
## logging.path
|
||||
- **Constraints / validation**: Required when `logging.destination = "file"`; must not be empty.
|
||||
- **Description**: File path used for file logging. With time rotation, the file name is used as the rolling prefix.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[logging]
|
||||
destination = "file"
|
||||
path = "/var/log/telemt.log"
|
||||
```
|
||||
## logging.rotation
|
||||
- **Constraints / validation**: Must be `never`, `minutely`, `hourly`, `daily`, or `weekly`.
|
||||
- **Description**: Time-based file rotation interval. `weekly` rotates at the Sunday UTC boundary. `never` writes to the exact `logging.path` unless size rotation is enabled.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[logging]
|
||||
destination = "file"
|
||||
path = "/var/log/telemt.log"
|
||||
rotation = "daily"
|
||||
```
|
||||
## logging.max_size_bytes
|
||||
- **Constraints / validation**: `0` disables size rotation.
|
||||
- **Description**: Rotates file logs before writing the next record when the active file is non-empty and that record would exceed this byte limit. Records are written whole and are not split.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[logging]
|
||||
destination = "file"
|
||||
path = "/var/log/telemt.log"
|
||||
max_size_bytes = 104857600
|
||||
```
|
||||
## logging.max_files
|
||||
- **Constraints / validation**: `0` disables count-based retention.
|
||||
- **Description**: Keeps at most this many matching file logs, counting the active file and rotated archives. The active file is never deleted by retention cleanup.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[logging]
|
||||
destination = "file"
|
||||
path = "/var/log/telemt.log"
|
||||
rotation = "daily"
|
||||
max_files = 14
|
||||
```
|
||||
## logging.max_age_secs
|
||||
- **Constraints / validation**: `0` disables age-based retention.
|
||||
- **Description**: Removes rotated file logs older than this many seconds based on file modification time. The active file is never deleted by retention cleanup.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[logging]
|
||||
destination = "file"
|
||||
path = "/var/log/telemt.log"
|
||||
rotation = "daily"
|
||||
max_age_secs = 1209600
|
||||
```
|
||||
|
||||
# [general]
|
||||
|
||||
|
||||
@@ -632,7 +712,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
```
|
||||
## beobachten
|
||||
- **Constraints / validation**: `bool`.
|
||||
- **Description**: Enables per-IP forensic observation buckets.
|
||||
- **Description**: Enables per-IP forensic observation buckets and appends TLS JA3/JA4 fingerprint snapshots to Beobachten output when available.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
@@ -641,7 +721,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
```
|
||||
## beobachten_minutes
|
||||
- **Constraints / validation**: Must be `> 0` (minutes).
|
||||
- **Description**: Retention window (minutes) for per-IP observation buckets.
|
||||
- **Description**: Retention window (minutes) for per-IP observation buckets and in-memory TLS fingerprint buckets.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
@@ -1805,6 +1885,8 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
| [`listen_unix_sock`](#listen_unix_sock) | `String` | — | `✘` |
|
||||
| [`listen_unix_sock_perm`](#listen_unix_sock_perm) | `String` | — | `✘` |
|
||||
| [`listen_tcp`](#listen_tcp) | `bool` | — (auto) | `✘` |
|
||||
| [`client_mss`](#client_mss) | `String` | `""` | `✘` |
|
||||
| [`client_mss_bulk`](#client_mss_bulk) | `String` | `""` | `✘` |
|
||||
| [`proxy_protocol`](#proxy_protocol) | `bool` | `false` | `✘` |
|
||||
| [`proxy_protocol_header_timeout_ms`](#proxy_protocol_header_timeout_ms) | `u64` | `500` | `✘` |
|
||||
| [`proxy_protocol_trusted_cidrs`](#proxy_protocol_trusted_cidrs) | `IpNetwork[]` | `[]` | `✘` |
|
||||
@@ -1887,6 +1969,27 @@ This document lists all configuration keys accepted by `config.toml`.
|
||||
listen_unix_sock = "/run/telemt.sock"
|
||||
listen_tcp = true
|
||||
```
|
||||
## client_mss
|
||||
- **Constraints / validation**: `String`. Empty or omitted means do not change kernel MSS. Presets: `"extreme-low"` = `88`, `"tspu"` = `92`, `"2in8"` = `256`. Custom decimal strings must be within `88..=4096`.
|
||||
- **Description**: Client-facing TCP MSS applied to TCP listener sockets before `listen(2)`, so Linux can announce it in SYN/ACK. This affects only proxy client TCP listeners, not API, metrics, Unix sockets, Telegram upstreams, ME sockets, or mask backend connections. Changes require listener restart/rebind.
|
||||
- **Operator note**: The two-tier `synlimit` profile does not require Telemt to disable MSS automatically. Operators that follow external host-tuning recipes should decide explicitly whether to leave MSS shaping enabled for handshake fragmentation or disable it for higher media throughput.
|
||||
- **Performance note**: Low MSS increases packet count predictably. Approximate segment multiplier is `ceil(1460 / client_mss)`.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[server]
|
||||
client_mss = "tspu"
|
||||
```
|
||||
## client_mss_bulk
|
||||
- **Constraints / validation**: `String`. Same grammar as [`client_mss`](#client_mss) (empty/omitted, presets `"extreme-low"`/`"tspu"`/`"2in8"`, or a decimal in `88..=4096`).
|
||||
- **Description**: Optional bulk-phase MSS. When set, the low `client_mss` is applied only while the TLS handshake (including the DPI-inspected ServerHello) is sent; once the connection transitions to relaying, the client socket MSS is raised to `client_mss_bulk` for the bulk data phase. This keeps the anti-DPI handshake fragmentation but restores normal-size packets for payload, cutting outgoing packets-per-second by roughly the `client_mss` segment multiplier (e.g. ~10x with `"tspu"`). Useful on hosts whose abuse detection counts packets-per-second rather than bandwidth. When empty/omitted, the handshake MSS is kept for the whole connection (previous behavior). Linux only; a no-op elsewhere.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[server]
|
||||
client_mss = "tspu"
|
||||
client_mss_bulk = "1400"
|
||||
```
|
||||
## proxy_protocol
|
||||
- **Constraints / validation**: `bool`.
|
||||
- **Description**: Enables HAProxy PROXY protocol parsing on incoming connections (PROXY v1/v2). When enabled, client source address is taken from the PROXY header.
|
||||
@@ -2173,7 +2276,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
||||
```
|
||||
## runtime_edge_top_n
|
||||
- **Constraints / validation**: `1..=1000`.
|
||||
- **Description**: Top-N size for edge connection leaderboard.
|
||||
- **Description**: Top-N size for edge connection and TLS fingerprint leaderboard snapshots.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
@@ -2207,6 +2310,16 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
||||
| --- | ---- | ------- | ---------- |
|
||||
| [`ip`](#ip) | `IpAddr` | — | `✘` |
|
||||
| [`port`](#port-serverlisteners) | `u16` | `server.port` | `✘` |
|
||||
| [`client_mss`](#client_mss-serverlisteners) | `String` | `[server].client_mss` | `✘` |
|
||||
| [`synlimit`](#synlimit-serverlisteners) | `false`, `"iptables"`, or `"nftables"` | `false` | `✔` |
|
||||
| [`synlimit_seconds`](#synlimit_seconds-serverlisteners) | `u32` | `60` | `✔` |
|
||||
| [`synlimit_hitcount`](#synlimit_hitcount-serverlisteners) | `u32` | `48` | `✔` |
|
||||
| [`synlimit_burst`](#synlimit_burst-serverlisteners) | `u32` | `1` | `✔` |
|
||||
| [`synlimit_ios_seconds`](#synlimit_ios_seconds-serverlisteners) | `u32` | `1` | `✔` |
|
||||
| [`synlimit_ios_hitcount`](#synlimit_ios_hitcount-serverlisteners) | `u32` | `12` | `✔` |
|
||||
| [`synlimit_ios_burst`](#synlimit_ios_burst-serverlisteners) | `u32` | `24` | `✔` |
|
||||
| [`synlimit_hashlimit_expire_ms`](#synlimit_hashlimit_expire_ms-serverlisteners) | `u32` | `60000` | `✔` |
|
||||
| [`synlimit_hashlimit_size`](#synlimit_hashlimit_size-serverlisteners) | `u32` | `32768` | `✔` |
|
||||
| [`announce`](#announce) | `String` | — | `✘` |
|
||||
| [`announce_ip`](#announce_ip) | `IpAddr` | — | `✘` |
|
||||
| [`proxy_protocol`](#proxy_protocol) | `bool` | — | `✘` |
|
||||
@@ -2231,6 +2344,130 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
```
|
||||
## client_mss (server.listeners)
|
||||
- **Constraints / validation**: `String` (optional). Same values as `[server].client_mss`.
|
||||
- **Description**: Per-listener MSS override. When omitted, inherits `[server].client_mss`; when set to an empty string, disables MSS shaping for this listener even if the global value is set. Changes require listener restart/rebind.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
client_mss = "256"
|
||||
```
|
||||
## synlimit (server.listeners)
|
||||
- **Constraints / validation**: `false`, `"iptables"`, or `"nftables"`. Omitted or `false` disables SYN limiting for this listener.
|
||||
- **Description**: Installs per-listener Linux netfilter two-tier SYN-fix rules for the listener port. `"iptables"` uses `iptables`/`ip6tables` filter rules with the `hashlimit`, `length`, and TTL/hop-limit matches. `"nftables"` uses Telemt-owned tables with per-source `meter` rules and equivalent IPv4/IPv6 classifiers. Rules are inserted early in `INPUT`, accept under-limit SYN packets, and reject over-limit SYN packets with TCP RST so clients retry promptly instead of waiting for a silent DROP timeout. The generic bucket is controlled by `synlimit_seconds`, `synlimit_hitcount`, and `synlimit_burst`; the iOS-like TTL/length bucket is controlled by `synlimit_ios_*`. Rules are reconciled at runtime and removed during graceful Telemt shutdown; `SIGKILL` cannot be cleaned up by the process. Requires CAP_NET_ADMIN. `synlimit*` changes hot-reload for existing listener endpoints; changing listener `ip` or `port` still requires restart/rebind.
|
||||
- **Operator note**: Telemt does not persist rules with `iptables-persistent`, write `/etc/sysctl.d`, edit systemd limits, or modify `client_mss`. Apply host-level tuning manually if your deployment policy requires it.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
|
||||
[[server.listeners]]
|
||||
ip = "::"
|
||||
port = 443
|
||||
synlimit = "nftables"
|
||||
```
|
||||
## synlimit_seconds (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `60`.
|
||||
- **Description**: Generic SYN-fix token-bucket interval. The rate is `synlimit_hitcount / synlimit_seconds` and is rendered to native netfilter rate units (`second`, `minute`, `hour`, or `day`). This bucket handles SYN packets that do not match the iOS-like TTL/length classifier.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_seconds = 60
|
||||
```
|
||||
## synlimit_hitcount (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `48`.
|
||||
- **Description**: Generic SYN-fix token-bucket rate amount. Together with `synlimit_seconds`, it defines the allowed source-IP SYN rate before excess SYN packets receive TCP RST.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_hitcount = 48
|
||||
```
|
||||
## synlimit_burst (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `1`.
|
||||
- **Description**: Generic SYN-fix token-bucket burst size. Higher values allow short connection bursts from the same source IP before the steady-state `synlimit_hitcount / synlimit_seconds` rate is enforced.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_burst = 1
|
||||
```
|
||||
## synlimit_ios_seconds (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `1`.
|
||||
- **Description**: Token-bucket interval for SYN packets matching the iOS-like classifier. IPv4 matches packet length `64` and TTL `< 65`; IPv6 matches packet length `84` and hop limit `< 65`.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_ios_seconds = 1
|
||||
```
|
||||
## synlimit_ios_hitcount (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `12`.
|
||||
- **Description**: Token-bucket rate amount for the iOS-like SYN classifier.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_ios_hitcount = 12
|
||||
```
|
||||
## synlimit_ios_burst (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `24`.
|
||||
- **Description**: Token-bucket burst size for the iOS-like SYN classifier.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_ios_burst = 24
|
||||
```
|
||||
## synlimit_hashlimit_expire_ms (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `60000`.
|
||||
- **Description**: Entry expiration in milliseconds for iptables/ip6tables hashlimit buckets. nftables meters use kernel-managed state and do not expose this exact knob.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_hashlimit_expire_ms = 60000
|
||||
```
|
||||
## synlimit_hashlimit_size (server.listeners)
|
||||
- **Constraints / validation**: `u32`, must be `> 0`. Default is `32768`.
|
||||
- **Description**: Hash table size for iptables/ip6tables hashlimit buckets. nftables meters use kernel-managed state and do not expose this exact knob.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_hashlimit_size = 32768
|
||||
```
|
||||
## announce
|
||||
- **Constraints / validation**: `String` (optional). Must not be empty when set.
|
||||
- **Description**: Public IP/domain announced in proxy links for this listener. Takes precedence over `announce_ip`.
|
||||
@@ -2934,6 +3171,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
| Key | Type | Default | Hot-Reload |
|
||||
| --- | ---- | ------- | ---------- |
|
||||
| [`users`](#users) | `Map<String, String>` | `{"default": "000…000"}` | `✔` |
|
||||
| [`user_enabled`](#user_enabled-1) | `Map<String, bool>` | `{}` | `✔` |
|
||||
| [`user_ad_tags`](#user_ad_tags) | `Map<String, String>` | `{}` | `✔` |
|
||||
| [`user_max_tcp_conns`](#user_max_tcp_conns) | `Map<String, usize>` | `{}` | `✔` |
|
||||
| [`user_max_tcp_conns_global_each`](#user_max_tcp_conns_global_each) | `usize` | `0` | `✔` |
|
||||
@@ -2948,7 +3186,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` | `✘` |
|
||||
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` | `✘` |
|
||||
| [`user_rate_limits`](#user_rate_limits) | `Map<String, RateLimitBps>` | `{}` | `✔` |
|
||||
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<IpNetwork, RateLimitBps>` | `{}` | `✔` |
|
||||
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<CidrRateLimitKey, RateLimitBps>` | `{}` | `✔` |
|
||||
|
||||
## users
|
||||
- **Constraints / validation**: Must not be empty (at least one user must exist). Each value must be **exactly 32 hex characters**.
|
||||
@@ -2960,6 +3198,16 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
alice = "00112233445566778899aabbccddeeff"
|
||||
bob = "0123456789abcdef0123456789abcdef"
|
||||
```
|
||||
## user_enabled
|
||||
- **Constraints / validation**: `Map<String, bool>`.
|
||||
- **Description**: Optional per-user enable overrides. Missing users are enabled by default. A value of `false` disables new sessions for that user; setting the value to `true` is accepted but equivalent to removing the override. API enable operations remove the override, while disable operations write `false`.
|
||||
- **Runtime behavior**: Hot reload applies this map immediately. Users disabled through API or config reload are rejected after successful authentication and active runtime sessions for that username are cancelled.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[access.user_enabled]
|
||||
alice = false
|
||||
```
|
||||
## user_ad_tags
|
||||
- **Constraints / validation**: Each value must be **exactly 32 hex characters** (same format as `general.ad_tag`). An all-zero tag is allowed but logs a warning.
|
||||
- **Description**: Per-user sponsored-channel ad tag override. When a user has an entry here, it takes precedence over `general.ad_tag`.
|
||||
@@ -3093,7 +3341,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
|
||||
## user_rate_limits
|
||||
- **Constraints / validation**: Table `username -> { up_bps, down_bps }`. At least one direction must be non-zero.
|
||||
- **Description**: Per-user bandwidth caps in bytes/sec for upload (`up_bps`) and download (`down_bps`).
|
||||
- **Description**: Per-user bandwidth caps in bits/sec for upload (`up_bps`) and download (`down_bps`).
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
@@ -3101,13 +3349,15 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
alice = { up_bps = 1048576, down_bps = 2097152 }
|
||||
```
|
||||
## cidr_rate_limits
|
||||
- **Constraints / validation**: Table `CIDR -> { up_bps, down_bps }`. CIDR must parse as `IpNetwork`; at least one direction must be non-zero.
|
||||
- **Description**: Source-subnet bandwidth caps applied alongside per-user limits.
|
||||
- **Constraints / validation**: Table `CIDR or auto-template -> { up_bps, down_bps }`. Explicit CIDR keys must parse as `IpNetwork`; auto-template keys must be `*4/N` (`N=0..32`), `*6/N` (`N=0..128`), or `*/N` (`N=0..32`). At least one direction must be non-zero. Duplicate normalized auto-templates are rejected.
|
||||
- **Description**: Source-subnet bandwidth caps applied alongside per-user limits. Explicit CIDR rules use longest-prefix-wins and take priority over auto-templates. Auto-templates create buckets lazily per matched source subnet: `*4/N` for IPv4, `*6/N` for IPv6, and `*/N` as a dual-stack shorthand where IPv4 uses `/N` and IPv6 uses `/(N * 4)`.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[access.cidr_rate_limits]
|
||||
"203.0.113.0/24" = { up_bps = 0, down_bps = 1048576 }
|
||||
"*4/32" = { up_bps = 262144, down_bps = 1048576 }
|
||||
"*6/64" = { up_bps = 262144, down_bps = 1048576 }
|
||||
```
|
||||
# [[upstreams]]
|
||||
|
||||
@@ -3120,6 +3370,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
| [`scopes`](#scopes) | `String` | `""` | `✘` |
|
||||
| [`ipv4`](#ipv4-upstreams) | `bool` | — (auto) | `✘` |
|
||||
| [`ipv6`](#ipv6-upstreams) | `bool` | — (auto) | `✘` |
|
||||
| [`prefer`](#prefer-upstreams) | `4` or `6` | effective `[network].prefer` | `✘` |
|
||||
| [`interface`](#interface) | `String` | — | `✘` |
|
||||
| [`bind_addresses`](#bind_addresses) | `String[]` | — | `✘` |
|
||||
| [`bindtodevice`](#bindtodevice) | `String` | — | `✘` |
|
||||
@@ -3191,7 +3442,7 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
```
|
||||
## ipv6 (upstreams)
|
||||
- **Constraints / validation**: `bool` (optional).
|
||||
- **Description**: Allows IPv6 DC targets for this upstream. When omitted, Telemt auto-detects support from runtime connectivity state.
|
||||
- **Description**: Allows IPv6 DC targets for this upstream. When omitted, Telemt auto-detects support from runtime connectivity state. Set this to `true` when the upstream proxy is reachable from the local host over IPv4 but the proxy itself can connect to Telegram DCs over IPv6.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
@@ -3199,6 +3450,18 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
||||
type = "direct"
|
||||
ipv6 = false
|
||||
```
|
||||
## prefer (upstreams)
|
||||
- **Constraints / validation**: Optional integer. Must be `4` or `6`.
|
||||
- **Description**: Overrides the IP family preference for Telegram DC targets selected through this upstream. When omitted, the upstream inherits the effective global `[network].prefer` decision. Use `prefer = 6` together with `ipv6 = true` for a SOCKS or Shadowsocks upstream that can egress over IPv6 even when the local Telemt host is IPv4-only.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[[upstreams]]
|
||||
type = "socks5"
|
||||
address = "192.0.2.10:1080"
|
||||
ipv6 = true
|
||||
prefer = 6
|
||||
```
|
||||
## interface
|
||||
- **Constraints / validation**: `String` (optional).
|
||||
- For `"direct"`: may be an IP address (used as explicit local bind) or an OS interface name (resolved to an IP at runtime; Unix only).
|
||||
|
||||
@@ -632,7 +632,7 @@
|
||||
```
|
||||
## beobachten
|
||||
- **Ограничения / валидация**: `bool`.
|
||||
- **Описание**: Включает "криминалистическое" наблюдения для каждого IP-адреса. Анализирует поведение всех подключений и записывает возможные типы клиентов, которые посылают active-probing запросы.
|
||||
- **Описание**: Включает "криминалистическое" наблюдения для каждого IP-адреса. Анализирует поведение всех подключений, записывает возможные типы клиентов, которые посылают active-probing запросы, и добавляет snapshot’ы TLS JA3/JA4 fingerprint’ов в Beobachten output, когда есть данные.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
@@ -641,7 +641,7 @@
|
||||
```
|
||||
## beobachten_minutes
|
||||
- **Ограничения / валидация**: Должно быть `> 0` (минут).
|
||||
- **Описание**: Время хранения (минуты) для сегментов наблюдения по каждому IP-адресу.
|
||||
- **Описание**: Время хранения (минуты) для сегментов наблюдения по каждому IP-адресу и in-memory bucket’ов TLS fingerprint’ов.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
@@ -1807,6 +1807,8 @@
|
||||
| [`listen_unix_sock`](#listen_unix_sock) | `String` | — | `✘` |
|
||||
| [`listen_unix_sock_perm`](#listen_unix_sock_perm) | `String` | — | `✘` |
|
||||
| [`listen_tcp`](#listen_tcp) | `bool` | — (auto) | `✘` |
|
||||
| [`client_mss`](#client_mss) | `String` | `""` | `✘` |
|
||||
| [`client_mss_bulk`](#client_mss_bulk) | `String` | `""` | `✘` |
|
||||
| [`proxy_protocol`](#proxy_protocol) | `bool` | `false` | `✘` |
|
||||
| [`proxy_protocol_header_timeout_ms`](#proxy_protocol_header_timeout_ms) | `u64` | `500` | `✘` |
|
||||
| [`proxy_protocol_trusted_cidrs`](#proxy_protocol_trusted_cidrs) | `IpNetwork[]` | `[]` | `✘` |
|
||||
@@ -1889,6 +1891,27 @@
|
||||
listen_unix_sock = "/run/telemt.sock"
|
||||
listen_tcp = true
|
||||
```
|
||||
## client_mss
|
||||
- **Ограничения / валидация**: `String`. Пустое значение или отсутствие параметра означает, что Telemt не изменяет MSS, выбранный ядром. Поддерживаемые presets: `"extreme-low"` = `88`, `"tspu"` = `92`, `"2in8"` = `256`. Пользовательское десятичное значение должно быть строкой в диапазоне `88..=4096`.
|
||||
- **Описание**: MSS для входящих TCP-соединений клиентов. Значение применяется к TCP listener-сокетам до `listen(2)`, чтобы Linux мог объявить его в SYN/ACK. Параметр влияет только на proxy client TCP listeners и не применяется к API, metrics, Unix sockets, Telegram upstreams, ME sockets или mask backend connections. Изменение требует restart/rebind listener’ов.
|
||||
- **Operator note**: Two-tier `synlimit` profile больше не требует автоматического отключения MSS внутри Telemt. Оператор должен сам решить, оставлять MSS shaping для handshake fragmentation или отключать его ради более высокой скорости media.
|
||||
- **Performance note**: Низкий MSS предсказуемо увеличивает количество TCP-сегментов. Приблизительный multiplier: `ceil(1460 / client_mss)`.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[server]
|
||||
client_mss = "tspu"
|
||||
```
|
||||
## client_mss_bulk
|
||||
- **Ограничения / валидация**: `String`. Грамматика та же, что у [`client_mss`](#client_mss) (пусто/не задано, пресеты `"extreme-low"`/`"tspu"`/`"2in8"` либо десятичное число в диапазоне `88..=4096`).
|
||||
- **Описание**: Необязательный MSS для bulk-фазы. Если задан, низкий `client_mss` применяется только на время TLS-handshake (включая инспектируемый DPI ServerHello); как только соединение переходит в фазу relay, MSS клиентского сокета поднимается до `client_mss_bulk` для передачи полезной нагрузки. Так сохраняется anti-DPI фрагментация handshake, но для данных возвращаются пакеты нормального размера — это снижает исходящий packets-per-second примерно во столько раз, каков segment multiplier у `client_mss` (например, ~10x для `"tspu"`). Полезно на хостингах, где abuse-детекция считает packets-per-second, а не полосу. Если пусто/не задано — MSS handshake сохраняется на всё соединение (прежнее поведение). Только Linux; на прочих платформах — no-op.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[server]
|
||||
client_mss = "tspu"
|
||||
client_mss_bulk = "1400"
|
||||
```
|
||||
## proxy_protocol
|
||||
- **Ограничения / валидация**: `bool`.
|
||||
- **Описание**: Включает поддержку разбора PROXY protocol от HAProxy (v1/v2) на входящих соединениях. При включении исходный IP клиента берётся из PROXY-заголовка.
|
||||
@@ -2179,7 +2202,7 @@
|
||||
```
|
||||
## runtime_edge_top_n
|
||||
- **Ограничения / валидация**: `1..=1000`.
|
||||
- **Описание**: Размер выборки Top-N для рейтинга (leaderboard) edge-соединений.
|
||||
- **Описание**: Размер выборки Top-N для snapshot’ов рейтинга edge-соединений и TLS fingerprint’ов.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
@@ -2213,6 +2236,16 @@
|
||||
| --- | ---- | ------- | ---------- |
|
||||
| [`ip`](#ip) | `IpAddr` | — | `✘` |
|
||||
| [`port`](#port-serverlisteners) | `u16` | `server.port` | `✘` |
|
||||
| [`client_mss`](#client_mss-serverlisteners) | `String` | `[server].client_mss` | `✘` |
|
||||
| [`synlimit`](#synlimit-serverlisteners) | `false`, `"iptables"` или `"nftables"` | `false` | `✔` |
|
||||
| [`synlimit_seconds`](#synlimit_seconds-serverlisteners) | `u32` | `60` | `✔` |
|
||||
| [`synlimit_hitcount`](#synlimit_hitcount-serverlisteners) | `u32` | `48` | `✔` |
|
||||
| [`synlimit_burst`](#synlimit_burst-serverlisteners) | `u32` | `1` | `✔` |
|
||||
| [`synlimit_ios_seconds`](#synlimit_ios_seconds-serverlisteners) | `u32` | `1` | `✔` |
|
||||
| [`synlimit_ios_hitcount`](#synlimit_ios_hitcount-serverlisteners) | `u32` | `12` | `✔` |
|
||||
| [`synlimit_ios_burst`](#synlimit_ios_burst-serverlisteners) | `u32` | `24` | `✔` |
|
||||
| [`synlimit_hashlimit_expire_ms`](#synlimit_hashlimit_expire_ms-serverlisteners) | `u32` | `60000` | `✔` |
|
||||
| [`synlimit_hashlimit_size`](#synlimit_hashlimit_size-serverlisteners) | `u32` | `32768` | `✔` |
|
||||
| [`announce`](#announce) | `String` | — | `✘` |
|
||||
| [`announce_ip`](#announce_ip) | `IpAddr` | — | `✘` |
|
||||
| [`proxy_protocol`](#proxy_protocol) | `bool` | — | `✘` |
|
||||
@@ -2237,6 +2270,130 @@
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
```
|
||||
## client_mss (server.listeners)
|
||||
- **Ограничения / валидация**: `String` (необязательный параметр). Допустимые значения совпадают с `[server].client_mss`.
|
||||
- **Описание**: Per-listener override для MSS. Если параметр не задан, listener наследует `[server].client_mss`; если задана пустая строка, MSS shaping отключается только для этого listener’а, даже когда глобальный параметр задан. Изменение требует restart/rebind listener’а.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
client_mss = "256"
|
||||
```
|
||||
## synlimit (server.listeners)
|
||||
- **Ограничения / валидация**: `false`, `"iptables"` или `"nftables"`. Если параметр не задан или задан как `false`, SYN limiter для этого listener’а выключен.
|
||||
- **Описание**: Устанавливает per-listener Linux netfilter two-tier SYN-fix rules для порта listener’а. `"iptables"` использует `iptables`/`ip6tables` filter rules с `hashlimit`, `length` и TTL/hop-limit matches. `"nftables"` использует Telemt-owned tables с per-source `meter` rules и эквивалентными IPv4/IPv6 classifiers. Rules вставляются рано в `INPUT`, принимают under-limit SYN packets и отвечают TCP RST на over-limit SYN packets, чтобы клиент быстро переподключался вместо ожидания silent DROP timeout. Generic bucket управляется `synlimit_seconds`, `synlimit_hitcount` и `synlimit_burst`; iOS-like TTL/length bucket управляется `synlimit_ios_*`. Rules reconciled at runtime и удаляются при graceful shutdown Telemt; `SIGKILL` процессом не очищается. Требует CAP_NET_ADMIN. Изменения `synlimit*` hot-reload’ятся для существующих listener endpoints; изменение listener `ip` или `port` по-прежнему требует restart/rebind.
|
||||
- **Operator note**: Telemt не сохраняет rules через `iptables-persistent`, не пишет `/etc/sysctl.d`, не меняет systemd limits и не модифицирует `client_mss`. Host-level tuning применяется оператором вручную.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
|
||||
[[server.listeners]]
|
||||
ip = "::"
|
||||
port = 443
|
||||
synlimit = "nftables"
|
||||
```
|
||||
## synlimit_seconds (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `60`.
|
||||
- **Описание**: Generic SYN-fix token-bucket interval. Rate равен `synlimit_hitcount / synlimit_seconds` и рендерится в native netfilter rate units (`second`, `minute`, `hour` или `day`). Этот bucket обрабатывает SYN packets, которые не совпали с iOS-like TTL/length classifier.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_seconds = 60
|
||||
```
|
||||
## synlimit_hitcount (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `48`.
|
||||
- **Описание**: Generic SYN-fix token-bucket rate amount. Вместе с `synlimit_seconds` задает разрешенный source-IP SYN rate до того, как excess SYN packets получат TCP RST.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_hitcount = 48
|
||||
```
|
||||
## synlimit_burst (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `1`.
|
||||
- **Описание**: Generic SYN-fix token-bucket burst size. Более высокие значения разрешают short connection bursts с одного source IP перед применением steady-state rate `synlimit_hitcount / synlimit_seconds`.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_burst = 1
|
||||
```
|
||||
## synlimit_ios_seconds (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `1`.
|
||||
- **Описание**: Token-bucket interval для SYN packets, совпавших с iOS-like classifier. IPv4 match: packet length `64` и TTL `< 65`; IPv6 match: packet length `84` и hop limit `< 65`.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_ios_seconds = 1
|
||||
```
|
||||
## synlimit_ios_hitcount (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `12`.
|
||||
- **Описание**: Token-bucket rate amount для iOS-like SYN classifier.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_ios_hitcount = 12
|
||||
```
|
||||
## synlimit_ios_burst (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `24`.
|
||||
- **Описание**: Token-bucket burst size для iOS-like SYN classifier.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_ios_burst = 24
|
||||
```
|
||||
## synlimit_hashlimit_expire_ms (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `60000`.
|
||||
- **Описание**: Entry expiration в миллисекундах для iptables/ip6tables hashlimit buckets. nftables meters используют kernel-managed state и не имеют точного аналога этого knob.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_hashlimit_expire_ms = 60000
|
||||
```
|
||||
## synlimit_hashlimit_size (server.listeners)
|
||||
- **Ограничения / валидация**: `u32`, должно быть `> 0`. Значение по умолчанию: `32768`.
|
||||
- **Описание**: Hash table size для iptables/ip6tables hashlimit buckets. nftables meters используют kernel-managed state и не имеют точного аналога этого knob.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[server.listeners]]
|
||||
ip = "0.0.0.0"
|
||||
port = 443
|
||||
synlimit = "iptables"
|
||||
synlimit_hashlimit_size = 32768
|
||||
```
|
||||
## announce
|
||||
- **Ограничения / валидация**: `String` (необязательный параметр). Не должен быть пустым, если задан.
|
||||
- **Описание**: Публичный IP-адрес или домен, объявляемый в proxy-ссылках для данного listener’а. Имеет приоритет над `announce_ip`.
|
||||
@@ -2955,7 +3112,7 @@
|
||||
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` | `✘` |
|
||||
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` | `✘` |
|
||||
| [`user_rate_limits`](#user_rate_limits) | `Map<String, RateLimitBps>` | `{}` | `✔` |
|
||||
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<IpNetwork, RateLimitBps>` | `{}` | `✔` |
|
||||
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<CidrRateLimitKey, RateLimitBps>` | `{}` | `✔` |
|
||||
|
||||
## users
|
||||
- **Ограничения / валидация**: Не должно быть пустым (должен существовать хотя бы один пользователь). Каждое значение должно состоять **ровно из 32 шестнадцатеричных символов**.
|
||||
@@ -3100,7 +3257,7 @@
|
||||
|
||||
## user_rate_limits
|
||||
- **Ограничения / валидация**: Таблица `username -> { up_bps, down_bps }`. Должно быть ненулевое значение хотя бы в одном направлении.
|
||||
- **Описание**: Персональные лимиты скорости по пользователям в байтах/сек для отправки (`up_bps`) и получения (`down_bps`).
|
||||
- **Описание**: Персональные лимиты скорости по пользователям в битах/сек для отправки (`up_bps`) и получения (`down_bps`).
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
@@ -3108,13 +3265,15 @@
|
||||
alice = { up_bps = 1048576, down_bps = 2097152 }
|
||||
```
|
||||
## cidr_rate_limits
|
||||
- **Ограничения / валидация**: Таблица `CIDR -> { up_bps, down_bps }`. CIDR должен корректно разбираться как `IpNetwork`; хотя бы одно направление должно быть ненулевым.
|
||||
- **Описание**: Лимиты скорости для подсетей источников, применяются поверх пользовательских ограничений.
|
||||
- **Ограничения / валидация**: Таблица `CIDR или auto-template -> { up_bps, down_bps }`. Explicit CIDR-ключи должны корректно разбираться как `IpNetwork`; auto-template ключи должны иметь вид `*4/N` (`N=0..32`), `*6/N` (`N=0..128`) или `*/N` (`N=0..32`). Хотя бы одно направление должно быть ненулевым. Дублирующиеся нормализованные auto-template отклоняются.
|
||||
- **Описание**: Лимиты скорости для подсетей источников, применяются поверх пользовательских ограничений. Explicit CIDR-правила используют longest-prefix-wins и имеют приоритет над auto-template. Auto-template создают bucket’ы лениво по matched source subnet: `*4/N` для IPv4, `*6/N` для IPv6, а `*/N` является dual-stack shorthand, где IPv4 использует `/N`, а IPv6 — `/(N * 4)`.
|
||||
- **Example**:
|
||||
|
||||
```toml
|
||||
[access.cidr_rate_limits]
|
||||
"203.0.113.0/24" = { up_bps = 0, down_bps = 1048576 }
|
||||
"*4/32" = { up_bps = 262144, down_bps = 1048576 }
|
||||
"*6/64" = { up_bps = 262144, down_bps = 1048576 }
|
||||
```
|
||||
# [[upstreams]]
|
||||
|
||||
@@ -3127,6 +3286,7 @@
|
||||
| [`scopes`](#scopes) | `String` | `""` | `✘` |
|
||||
| [`ipv4`](#ipv4-upstreams) | `bool` | — (auto) | `✘` |
|
||||
| [`ipv6`](#ipv6-upstreams) | `bool` | — (auto) | `✘` |
|
||||
| [`prefer`](#prefer-upstreams) | `4` или `6` | эффективный `[network].prefer` | `✘` |
|
||||
| [`interface`](#interface) | `String` | — | `✘` |
|
||||
| [`bind_addresses`](#bind_addresses) | `String[]` | — | `✘` |
|
||||
| [`bindtodevice`](#bindtodevice) | `String` | — | `✘` |
|
||||
@@ -3198,7 +3358,7 @@
|
||||
```
|
||||
## ipv6 (upstreams)
|
||||
- **Ограничения / валидация**: `bool` (необязательный параметр).
|
||||
- **Описание**: Разрешает IPv6 DC-targets для этого upstream. Если не задан, Telemt определяет поддержку автоматически по runtime-состоянию connectivity.
|
||||
- **Описание**: Разрешает IPv6 DC-targets для этого upstream. Если не задан, Telemt определяет поддержку автоматически по runtime-состоянию connectivity. Установите `true`, если upstream proxy доступен с локального хоста по IPv4, но сам proxy умеет подключаться к Telegram DC по IPv6.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
@@ -3206,6 +3366,18 @@
|
||||
type = "direct"
|
||||
ipv6 = false
|
||||
```
|
||||
## prefer (upstreams)
|
||||
- **Ограничения / валидация**: Необязательное число. Должно быть `4` или `6`.
|
||||
- **Описание**: Переопределяет предпочтительное IP-семейство для Telegram DC-targets, выбранных через этот upstream. Если параметр не задан, upstream наследует эффективное глобальное решение `[network].prefer`. Используйте `prefer = 6` вместе с `ipv6 = true` для SOCKS или Shadowsocks upstream, который умеет выходить в IPv6, даже если локальный хост с Telemt работает только по IPv4.
|
||||
- **Пример**:
|
||||
|
||||
```toml
|
||||
[[upstreams]]
|
||||
type = "socks5"
|
||||
address = "192.0.2.10:1080"
|
||||
ipv6 = true
|
||||
prefer = 6
|
||||
```
|
||||
## interface
|
||||
- **Ограничения / валидация**: `String` (необязательный параметр).
|
||||
- для `"direct"`: может быть IP-адресом (используется как явный local bind) или именем сетевого интерфейса ОС (резолвится в IP во время выполнения; только Unix).
|
||||
|
||||
@@ -40,6 +40,8 @@ hello2 = "ad_tag2"
|
||||
> Проблема с TLS отпечатком исправлена в последних версиях клиентов Telegram для Desktop / Android / iOS.
|
||||
> Обновите свой клиент для корректной работы с MTProxy Fake-TLS!
|
||||
|
||||
- Для расследования блокировок на базе JA4 ClientHello используйте отдельную инструкцию: [`JA3 и JA4 анализ в Telemt`](Architecture/Fronting-splitting/TLS_JA3_JA4_ANALYSIS.ru.md).
|
||||
|
||||
- Мы считаем это прорывом, которому на сегодняшний день нет стабильных аналогов;
|
||||
- Исходя из этого: если `telemt` настроен правильно, **режим TLS полностью идентичен реальному «рукопожатию» + обмену данными** с указанным хостом;
|
||||
- Вот наши доказательства:
|
||||
|
||||
@@ -206,7 +206,7 @@ File content:
|
||||
"publicKey": "<SERVER_B_PUBLIC_KEY>",
|
||||
"shortId": "<SHORT_ID>",
|
||||
"spiderX": "/",
|
||||
"fingerprint": "chrome"
|
||||
"fingerprint": "firefox"
|
||||
},
|
||||
"xhttpSettings": {
|
||||
"path": "/<YOUR_RANDOM_PATH>"
|
||||
|
||||
@@ -206,7 +206,7 @@ nano /usr/local/etc/xray/config.json
|
||||
"publicKey": "<SERVER_B_PUBLIC_KEY>",
|
||||
"shortId": "<SHORT_ID>",
|
||||
"spiderX": "/",
|
||||
"fingerprint": "chrome"
|
||||
"fingerprint": "firefox"
|
||||
},
|
||||
"xhttpSettings": {
|
||||
"path": "/<YOUR_RANDOM_PATH>"
|
||||
|
||||
460
src/api/config_edit.rs
Normal file
460
src/api/config_edit.rs
Normal file
@@ -0,0 +1,460 @@
|
||||
//! Config-editing API: read managed sections and apply sparse field patches.
|
||||
//! `access.*` is intentionally not editable here (owned by the users API).
|
||||
|
||||
use serde_json::Value as Json;
|
||||
use toml::Value as Toml;
|
||||
|
||||
use super::ApiShared;
|
||||
use super::config_store::{
|
||||
EDITABLE_SECTIONS, compute_revision, current_revision, load_config_from_disk,
|
||||
save_sections_to_disk,
|
||||
};
|
||||
use super::model::ApiFailure;
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::config::hot_reload::classify_config_changes;
|
||||
use crate::maestro::reload::{ReloadAccepted, ReloadRequest, ReloadSubmitError};
|
||||
use crate::maestro::runtime_build::deferred_process_fields;
|
||||
use serde::Serialize;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct PatchConfigResponse {
|
||||
pub revision: String,
|
||||
pub restart_required: bool,
|
||||
pub runtime_reload_required: bool,
|
||||
pub process_restart_required: bool,
|
||||
pub deferred_process_fields: Vec<String>,
|
||||
pub changed: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reload: Option<ReloadAccepted>,
|
||||
}
|
||||
|
||||
/// Shared-state wrapper around [`apply_patch_to_path`]: serializes config
|
||||
/// mutations behind `mutation_lock`, then records a runtime event. The route
|
||||
/// handler calls this; the core logic stays decoupled for unit tests.
|
||||
pub(super) async fn patch_config(
|
||||
patch_json: Json,
|
||||
expected_revision: Option<String>,
|
||||
reload_request: Option<ReloadRequest>,
|
||||
shared: &ApiShared,
|
||||
) -> Result<PatchConfigResponse, ApiFailure> {
|
||||
let _guard = shared.mutation_lock.lock().await;
|
||||
if reload_request.is_some()
|
||||
&& let Some(reload_id) = shared.reload_control.in_progress().await
|
||||
{
|
||||
return Err(ApiFailure::new(
|
||||
hyper::StatusCode::CONFLICT,
|
||||
"reload_in_progress",
|
||||
format!("Reload {} is already in progress", reload_id),
|
||||
));
|
||||
}
|
||||
let mut resp = apply_patch_to_path(&shared.config_path, &patch_json, expected_revision).await?;
|
||||
if let Some(request) = reload_request {
|
||||
let config = Arc::new(load_config_from_disk(&shared.config_path).await?);
|
||||
let accepted = shared
|
||||
.reload_control
|
||||
.submit(config, resp.revision.clone(), request)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
ReloadSubmitError::InProgress(reload_id) => ApiFailure::new(
|
||||
hyper::StatusCode::CONFLICT,
|
||||
"reload_in_progress",
|
||||
format!("Reload {} is already in progress", reload_id),
|
||||
),
|
||||
ReloadSubmitError::MaestroUnavailable => ApiFailure::new(
|
||||
hyper::StatusCode::SERVICE_UNAVAILABLE,
|
||||
"maestro_unavailable",
|
||||
"Maestro reload coordinator is unavailable",
|
||||
),
|
||||
})?;
|
||||
resp.reload = Some(accepted);
|
||||
}
|
||||
drop(_guard);
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.config.patch.ok", format!("changed={:?}", resp.changed));
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
/// Core patch logic, decoupled from hyper/shared-state so it is unit-testable
|
||||
/// against a temp file. The route handler holds `mutation_lock` while calling this.
|
||||
pub(super) async fn apply_patch_to_path(
|
||||
config_path: &Path,
|
||||
patch_json: &Json,
|
||||
expected_revision: Option<String>,
|
||||
) -> Result<PatchConfigResponse, ApiFailure> {
|
||||
// 1. optimistic concurrency
|
||||
let current = current_revision(config_path).await?;
|
||||
if expected_revision.is_some_and(|expected| expected != current) {
|
||||
return Err(ApiFailure::new(
|
||||
hyper::StatusCode::CONFLICT,
|
||||
"revision_conflict",
|
||||
"Config revision mismatch",
|
||||
));
|
||||
}
|
||||
|
||||
// 2. convert + reject access / unknown sections
|
||||
let patch_toml = json_to_toml(patch_json)
|
||||
.map_err(|e| ApiFailure::bad_request(format!("invalid patch: {}", e)))?;
|
||||
let patch_table = patch_toml
|
||||
.as_table()
|
||||
.ok_or_else(|| ApiFailure::bad_request("patch must be a JSON object"))?;
|
||||
if patch_table.contains_key("access") {
|
||||
return Err(ApiFailure::new(
|
||||
hyper::StatusCode::BAD_REQUEST,
|
||||
"access_not_editable",
|
||||
"access.* is managed via the users API, not editable here",
|
||||
));
|
||||
}
|
||||
for key in patch_table.keys() {
|
||||
if !EDITABLE_SECTIONS.contains(&key.as_str()) {
|
||||
return Err(ApiFailure::new(
|
||||
hyper::StatusCode::BAD_REQUEST,
|
||||
"section_not_editable",
|
||||
format!("section not editable: {}", key),
|
||||
));
|
||||
}
|
||||
}
|
||||
let touched: Vec<&str> = patch_table
|
||||
.keys()
|
||||
.map(|k| k.as_str())
|
||||
.filter(|k| EDITABLE_SECTIONS.contains(k))
|
||||
.collect();
|
||||
if touched.is_empty() {
|
||||
return Err(ApiFailure::bad_request("empty patch: no editable sections"));
|
||||
}
|
||||
|
||||
// 3. Parse old + merged from the SAME deserialize path so the classifier
|
||||
// sees only the delta this patch introduces. `ProxyConfig::load` applies
|
||||
// include-expansion / legacy-compat / normalization that a bare
|
||||
// `try_into` does not; mixing the two paths would make unrelated fields
|
||||
// compare unequal and spuriously force `restart_required`.
|
||||
let original = tokio::fs::read_to_string(config_path)
|
||||
.await
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to read config: {}", e)))?;
|
||||
let original_toml: Toml = toml::from_str(&original)
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to parse config: {}", e)))?;
|
||||
let old_cfg: ProxyConfig = original_toml
|
||||
.clone()
|
||||
.try_into()
|
||||
.map_err(|e| ApiFailure::internal(format!("config does not deserialize: {}", e)))?;
|
||||
|
||||
let mut merged = original_toml;
|
||||
deep_merge(&mut merged, &patch_toml);
|
||||
|
||||
let new_cfg: ProxyConfig = merged
|
||||
.clone()
|
||||
.try_into()
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config does not deserialize: {}", e)))?;
|
||||
new_cfg
|
||||
.validate()
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
|
||||
// 4. classify changes (Telemt's own hot/restart rule)
|
||||
let class = classify_config_changes(&old_cfg, &new_cfg);
|
||||
let deferred_process_fields = deferred_process_fields(&old_cfg, &new_cfg);
|
||||
|
||||
// 5. write only the touched top-level sections
|
||||
let revision = save_sections_to_disk(config_path, &new_cfg, &touched).await?;
|
||||
|
||||
Ok(PatchConfigResponse {
|
||||
revision,
|
||||
restart_required: class.restart_required,
|
||||
runtime_reload_required: class.restart_required,
|
||||
process_restart_required: !deferred_process_fields.is_empty(),
|
||||
deferred_process_fields,
|
||||
changed: class.changed,
|
||||
reload: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return only the editable config sections + current revision.
|
||||
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
|
||||
let original = tokio::fs::read_to_string(config_path)
|
||||
.await
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to read config: {}", e)))?;
|
||||
let parsed: Toml = toml::from_str(&original)
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to parse config: {}", e)))?;
|
||||
|
||||
let parsed_table = parsed
|
||||
.as_table()
|
||||
.cloned()
|
||||
.unwrap_or_else(toml::value::Table::new);
|
||||
// Whitelist: return ONLY the editable sections. A blacklist (just removing
|
||||
// `access`) would leak `server` (carries the API `auth_header` + per-node
|
||||
// identity) and `network` (per-node addresses). Mirror the PATCH contract.
|
||||
let mut table = toml::value::Table::new();
|
||||
for section in EDITABLE_SECTIONS {
|
||||
if let Some(value) = parsed_table.get(*section) {
|
||||
table.insert((*section).to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let revision = compute_revision(&original);
|
||||
Ok((Toml::Table(table), revision))
|
||||
}
|
||||
|
||||
/// Convert a serde_json value to a toml value. `null` is dropped from objects
|
||||
/// (a patch never sets a key to TOML-null). Numbers become integers when exact,
|
||||
/// otherwise floats.
|
||||
fn json_to_toml(j: &Json) -> Result<Toml, String> {
|
||||
Ok(match j {
|
||||
Json::Null => return Err("null is not representable in TOML".into()),
|
||||
Json::Bool(b) => Toml::Boolean(*b),
|
||||
Json::Number(n) => {
|
||||
if let Some(i) = n.as_i64() {
|
||||
Toml::Integer(i)
|
||||
} else if let Some(f) = n.as_f64() {
|
||||
Toml::Float(f)
|
||||
} else {
|
||||
return Err(format!("unrepresentable number: {}", n));
|
||||
}
|
||||
}
|
||||
Json::String(s) => Toml::String(s.clone()),
|
||||
Json::Array(items) => {
|
||||
let mut out = Vec::with_capacity(items.len());
|
||||
for item in items {
|
||||
out.push(json_to_toml(item)?);
|
||||
}
|
||||
Toml::Array(out)
|
||||
}
|
||||
Json::Object(map) => {
|
||||
let mut table = toml::value::Table::new();
|
||||
for (k, v) in map {
|
||||
if v.is_null() {
|
||||
continue; // skip nulls instead of erroring at object level
|
||||
}
|
||||
table.insert(k.clone(), json_to_toml(v)?);
|
||||
}
|
||||
Toml::Table(table)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Recursively overlay `patch` onto `base`. Tables merge key-by-key; every
|
||||
/// other value type (scalars, arrays) replaces wholesale.
|
||||
fn deep_merge(base: &mut Toml, patch: &Toml) {
|
||||
match (base, patch) {
|
||||
(Toml::Table(b), Toml::Table(p)) => {
|
||||
for (k, pv) in p {
|
||||
match b.get_mut(k) {
|
||||
Some(bv) => deep_merge(bv, pv),
|
||||
None => {
|
||||
b.insert(k.clone(), pv.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(b, p) => *b = p.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn json_object_converts_to_toml_table() {
|
||||
let j: Json = serde_json::json!({"censorship": {"tls_domain": "a.com"}, "default_dc": 2});
|
||||
let t = json_to_toml(&j).expect("convertible");
|
||||
let table = t.as_table().unwrap();
|
||||
assert_eq!(table["censorship"]["tls_domain"].as_str(), Some("a.com"));
|
||||
assert_eq!(table["default_dc"].as_integer(), Some(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_merge_overlays_tables_and_replaces_scalars() {
|
||||
let mut base: Toml =
|
||||
toml::from_str("[censorship]\ntls_domain = \"old\"\nfake_cert_len = 100\n").unwrap();
|
||||
let patch: Toml = toml::from_str("[censorship]\ntls_domain = \"new\"\n").unwrap();
|
||||
|
||||
deep_merge(&mut base, &patch);
|
||||
|
||||
let cens = base["censorship"].as_table().unwrap();
|
||||
assert_eq!(cens["tls_domain"].as_str(), Some("new")); // overlaid
|
||||
assert_eq!(cens["fake_cert_len"].as_integer(), Some(100)); // preserved
|
||||
}
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn temp_config(body: &str) -> (PathBuf, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("config.toml");
|
||||
std::fs::write(&path, body).unwrap();
|
||||
(path, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_rejects_access_section() {
|
||||
let (path, _d) = temp_config("[censorship]\ntls_domain = \"a\"\n");
|
||||
let patch: Json = serde_json::json!({"access": {"users": {"x": "y"}}});
|
||||
let err = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
assert_eq!(err.code, "access_not_editable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_revision_conflict() {
|
||||
let (path, _d) = temp_config("[censorship]\ntls_domain = \"a\"\n");
|
||||
let patch: Json = serde_json::json!({"censorship": {"tls_domain": "b"}});
|
||||
let err = apply_patch_to_path(&path, &patch, Some("deadbeef".into()))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code, "revision_conflict");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_sni_reports_restart_required() {
|
||||
let (path, _d) =
|
||||
temp_config("[censorship]\ntls_domain = \"a.com\"\n[server]\nport = 443\n");
|
||||
let patch: Json = serde_json::json!({"censorship": {"tls_domain": "b.com"}});
|
||||
let resp = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||
assert!(resp.restart_required);
|
||||
assert!(resp.runtime_reload_required);
|
||||
assert!(!resp.process_restart_required);
|
||||
assert!(resp.deferred_process_fields.is_empty());
|
||||
assert!(resp.changed.iter().any(|c| c == "censorship"));
|
||||
let written = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(written.contains("tls_domain = \"b.com\""));
|
||||
assert_eq!(
|
||||
resp.revision,
|
||||
crate::api::config_store::compute_revision(&written)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_managed_config_strips_access() {
|
||||
let (path, _d) = temp_config(
|
||||
"[censorship]\ntls_domain = \"a.com\"\n[access.users]\nbob = \"deadbeef\"\n",
|
||||
);
|
||||
let (value, revision) = read_managed_config(&path).await.unwrap();
|
||||
let table = value.as_table().unwrap();
|
||||
assert!(table.contains_key("censorship"));
|
||||
assert!(!table.contains_key("access")); // secrets never leave the box here
|
||||
assert_eq!(revision, current_revision(&path).await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_managed_config_returns_only_editable_sections() {
|
||||
// server carries the API auth_header + per-node identity; network carries
|
||||
// per-node addresses. Neither must be exposed by GET /v1/config.
|
||||
let (path, _d) = temp_config(concat!(
|
||||
"[censorship]\ntls_domain = \"a\"\n",
|
||||
"[server]\nport = 443\n[server.api]\nauth_header = \"SECRET\"\n",
|
||||
"[network]\nipv4 = \"1.2.3.4\"\n",
|
||||
"[access.users]\nbob = \"deadbeef\"\n",
|
||||
));
|
||||
let (value, _rev) = read_managed_config(&path).await.unwrap();
|
||||
let table = value.as_table().unwrap();
|
||||
assert!(table.contains_key("censorship"));
|
||||
assert!(!table.contains_key("server")); // no API auth_header / identity leak
|
||||
assert!(!table.contains_key("network")); // no per-node identity leak
|
||||
assert!(!table.contains_key("access")); // no users/secrets
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_rejects_server_section() {
|
||||
let (path, _d) = temp_config("[censorship]\ntls_domain = \"a\"\n");
|
||||
let patch: Json = serde_json::json!({"server": {"port": 1}});
|
||||
let err = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
assert_eq!(err.code, "section_not_editable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_rejects_show_link_section() {
|
||||
// show_link is a legacy top-level scalar/array (not a [table]); it cannot
|
||||
// be upserted safely and is superseded by the editable general.links.show.
|
||||
let (path, _d) = temp_config("[censorship]\ntls_domain = \"a\"\n");
|
||||
let patch: Json = serde_json::json!({"show_link": "*"});
|
||||
let err = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
assert_eq!(err.code, "section_not_editable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_general_links_show_is_editable() {
|
||||
// The supported replacement path: edit show via the general.links sub-table.
|
||||
let (path, _d) = temp_config(
|
||||
"[general]\nprefer_ipv6 = false\n[general.links]\nshow = \"*\"\n\
|
||||
[censorship]\ntls_domain = \"a\"\n",
|
||||
);
|
||||
let patch: Json = serde_json::json!({"general": {"links": {"show": ["alice"]}}});
|
||||
let resp = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||
assert!(resp.changed.iter().any(|c| c == "general"));
|
||||
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
let parsed: toml::Value = toml::from_str(&written).unwrap();
|
||||
assert_eq!(
|
||||
parsed["general"]["links"]["show"][0].as_str(),
|
||||
Some("alice"),
|
||||
"{written}"
|
||||
);
|
||||
// No leaked top-level [links]/[modes] and no duplicate sub-tables.
|
||||
assert_eq!(written.matches("[general.links]").count(), 1, "{written}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_links_public_port_written_as_integer_not_float_or_string() {
|
||||
// A JSON integer must land on disk as a bare TOML integer (443), never
|
||||
// 443.0 nor "443". The write re-renders from the typed config, so the
|
||||
// u16 field dictates the output format regardless of JSON quirks.
|
||||
let (path, _d) = temp_config("[general]\nprefer_ipv6 = false\n");
|
||||
let patch: Json = serde_json::json!({"general": {"links": {"public_port": 443}}});
|
||||
apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||
|
||||
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
assert!(written.contains("public_port = 443"), "{written}");
|
||||
assert!(
|
||||
!written.contains("443.0"),
|
||||
"must not be a float:\n{written}"
|
||||
);
|
||||
assert!(
|
||||
!written.contains("\"443\""),
|
||||
"must not be a string:\n{written}"
|
||||
);
|
||||
|
||||
let parsed: toml::Value = toml::from_str(&written).unwrap();
|
||||
assert_eq!(
|
||||
parsed["general"]["links"]["public_port"].as_integer(),
|
||||
Some(443),
|
||||
"{written}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_links_public_port_rejects_float() {
|
||||
// 443.0 cannot deserialize into u16 -> rejected, not silently coerced.
|
||||
let (path, _d) = temp_config("[general]\nprefer_ipv6 = false\n");
|
||||
let patch: Json = serde_json::json!({"general": {"links": {"public_port": 443.0}}});
|
||||
let err = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
assert_eq!(err.status, hyper::StatusCode::BAD_REQUEST, "{:?}", err);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_links_public_port_rejects_string() {
|
||||
// "443" is a string, not a u16 -> rejected.
|
||||
let (path, _d) = temp_config("[general]\nprefer_ipv6 = false\n");
|
||||
let patch: Json = serde_json::json!({"general": {"links": {"public_port": "443"}}});
|
||||
let err = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||
assert_eq!(err.status, hyper::StatusCode::BAD_REQUEST, "{:?}", err);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_empty_is_rejected() {
|
||||
let (path, _d) = temp_config("[censorship]\ntls_domain = \"a\"\n");
|
||||
let patch: Json = serde_json::json!({});
|
||||
assert!(apply_patch_to_path(&path, &patch, None).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn patch_log_level_is_hot() {
|
||||
// general.log_level is hot-reloadable -> a patch changing only it must
|
||||
// report restart_required = false (exercises the full apply path, not
|
||||
// just the classifier). Default LogLevel is Normal; patch to "debug".
|
||||
let (path, _d) = temp_config("[censorship]\ntls_domain = \"a\"\n");
|
||||
let patch: Json = serde_json::json!({"general": {"log_level": "debug"}});
|
||||
let resp = apply_patch_to_path(&path, &patch, None).await.unwrap();
|
||||
assert!(!resp.restart_required);
|
||||
assert!(!resp.runtime_reload_required);
|
||||
assert!(!resp.process_restart_required);
|
||||
assert!(resp.changed.iter().any(|c| c == "general"));
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ use super::model::ApiFailure;
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum AccessSection {
|
||||
Users,
|
||||
UserEnabled,
|
||||
UserAdTags,
|
||||
UserMaxTcpConns,
|
||||
UserExpirations,
|
||||
@@ -26,6 +27,7 @@ impl AccessSection {
|
||||
fn table_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Users => "access.users",
|
||||
Self::UserEnabled => "access.user_enabled",
|
||||
Self::UserAdTags => "access.user_ad_tags",
|
||||
Self::UserMaxTcpConns => "access.user_max_tcp_conns",
|
||||
Self::UserExpirations => "access.user_expirations",
|
||||
@@ -70,6 +72,13 @@ pub(super) async fn current_revision(config_path: &Path) -> Result<String, ApiFa
|
||||
Ok(compute_revision(&content))
|
||||
}
|
||||
|
||||
pub(crate) async fn current_revision_for_maestro(config_path: &Path) -> Result<String, String> {
|
||||
let content = tokio::fs::read_to_string(config_path)
|
||||
.await
|
||||
.map_err(|error| format!("failed to read config: {}", error))?;
|
||||
Ok(compute_revision(&content))
|
||||
}
|
||||
|
||||
pub(super) fn compute_revision(content: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(content.as_bytes());
|
||||
@@ -84,6 +93,14 @@ pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyCon
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
||||
}
|
||||
|
||||
pub(super) async fn load_config_for_reload(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
|
||||
let config_path = config_path.to_path_buf();
|
||||
tokio::task::spawn_blocking(move || ProxyConfig::load(config_path))
|
||||
.await
|
||||
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {}", error)))?
|
||||
.map_err(|error| ApiFailure::bad_request(format!("invalid runtime config: {}", error)))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(super) async fn save_config_to_disk(
|
||||
config_path: &Path,
|
||||
@@ -95,6 +112,90 @@ pub(super) async fn save_config_to_disk(
|
||||
Ok(compute_revision(&serialized))
|
||||
}
|
||||
|
||||
/// Top-level config tables that may be edited via the config API.
|
||||
///
|
||||
/// Intentionally excluded (defense-in-depth, enforces the spec's per-node
|
||||
/// identity invariant at the Telemt layer too):
|
||||
///
|
||||
/// - `access` : owned by the users API.
|
||||
/// - `server` : carries per-node identity (`port`, `api`/`api_bind`, listeners).
|
||||
/// - `network` : carries per-node identity (`ipv4`/`ipv6`).
|
||||
/// - `show_link` : legacy top-level scalar/array (not a `[table]`), superseded
|
||||
/// by the editable `general.links.show` sub-table. The
|
||||
/// section-upsert machinery here only handles `[table]` /
|
||||
/// `[[array-of-tables]]` blocks; a bare top-level key cannot be
|
||||
/// located or replaced safely, so it is edited via `general`.
|
||||
///
|
||||
/// A future field-level allowlist can re-admit specific safe fields
|
||||
/// (e.g. `network.dns_overrides`) without opening the whole section.
|
||||
pub(super) const EDITABLE_SECTIONS: &[&str] = &[
|
||||
"general",
|
||||
"timeouts",
|
||||
"censorship",
|
||||
"upstreams",
|
||||
"dc_overrides",
|
||||
];
|
||||
|
||||
/// Re-render the given top-level tables from `cfg` and upsert each into the
|
||||
/// on-disk file, preserving every untouched section (and its comments).
|
||||
pub(super) async fn save_sections_to_disk(
|
||||
config_path: &Path,
|
||||
cfg: &ProxyConfig,
|
||||
sections: &[&str],
|
||||
) -> Result<String, ApiFailure> {
|
||||
let mut content = tokio::fs::read_to_string(config_path)
|
||||
.await
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to read config: {}", e)))?;
|
||||
|
||||
for section in sections {
|
||||
let rendered = render_top_level_section(cfg, section)?;
|
||||
content = upsert_toml_table(&content, section, &rendered);
|
||||
}
|
||||
|
||||
write_atomic(config_path.to_path_buf(), content.clone()).await?;
|
||||
Ok(compute_revision(&content))
|
||||
}
|
||||
|
||||
/// Render one top-level table as `[section]\n...\n` (or `[[upstreams]]` array
|
||||
/// of tables) from the typed `cfg`. Serializes via the `toml` crate so the
|
||||
/// output matches the canonical format Telemt parses.
|
||||
fn render_top_level_section(cfg: &ProxyConfig, section: &str) -> Result<String, ApiFailure> {
|
||||
let value = toml::Value::try_from(cfg)
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to serialize config: {}", e)))?;
|
||||
let table = value
|
||||
.get(section)
|
||||
.ok_or_else(|| ApiFailure::internal(format!("unknown section: {}", section)))?;
|
||||
|
||||
// upstreams is an array-of-tables -> render as [[upstreams]] blocks.
|
||||
if let toml::Value::Array(items) = table {
|
||||
let mut out = String::new();
|
||||
for item in items {
|
||||
out.push_str(&format!("[[{}]]\n", section));
|
||||
out.push_str(&toml::to_string(item).map_err(|e| {
|
||||
ApiFailure::internal(format!("failed to serialize {}: {}", section, e))
|
||||
})?);
|
||||
if !out.ends_with('\n') {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
// Serialize the table *inside a wrapper keyed by `section`* so the `toml`
|
||||
// crate emits correctly dotted headers for nested sub-tables, e.g.
|
||||
// `[general]` + `[general.modes]` + `[general.links]`. Serializing the
|
||||
// inner table alone would render bare `[modes]`/`[links]` headers, which
|
||||
// would leak as duplicate top-level tables and break config load.
|
||||
let mut wrapper = toml::value::Table::new();
|
||||
wrapper.insert(section.to_string(), table.clone());
|
||||
let mut out = toml::to_string(&toml::Value::Table(wrapper))
|
||||
.map_err(|e| ApiFailure::internal(format!("failed to serialize {}: {}", section, e)))?;
|
||||
if !out.ends_with('\n') {
|
||||
out.push('\n');
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(super) async fn save_access_sections_to_disk(
|
||||
config_path: &Path,
|
||||
cfg: &ProxyConfig,
|
||||
@@ -135,6 +236,15 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result<St
|
||||
.collect();
|
||||
serialize_table_body(&rows)?
|
||||
}
|
||||
AccessSection::UserEnabled => {
|
||||
let rows: BTreeMap<String, bool> = cfg
|
||||
.access
|
||||
.user_enabled
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), *value))
|
||||
.collect();
|
||||
serialize_table_body(&rows)?
|
||||
}
|
||||
AccessSection::UserAdTags => {
|
||||
let rows: BTreeMap<String, String> = cfg
|
||||
.access
|
||||
@@ -204,6 +314,7 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result<St
|
||||
fn access_section_is_empty(cfg: &ProxyConfig, section: AccessSection) -> bool {
|
||||
match section {
|
||||
AccessSection::Users => cfg.access.users.is_empty(),
|
||||
AccessSection::UserEnabled => cfg.access.user_enabled.is_empty(),
|
||||
AccessSection::UserAdTags => cfg.access.user_ad_tags.is_empty(),
|
||||
AccessSection::UserMaxTcpConns => cfg.access.user_max_tcp_conns.is_empty(),
|
||||
AccessSection::UserExpirations => cfg.access.user_expirations.is_empty(),
|
||||
@@ -241,11 +352,22 @@ fn serialize_toml_key(key: &str) -> Result<String, ApiFailure> {
|
||||
}
|
||||
|
||||
fn upsert_toml_table(source: &str, table_name: &str, replacement: &str) -> String {
|
||||
if let Some((start, end)) = find_toml_table_bounds(source, table_name) {
|
||||
let blocks = find_all_table_blocks(source, table_name);
|
||||
if let Some(&(first_start, first_end)) = blocks.first() {
|
||||
// Replace the first block in place and delete any further blocks that
|
||||
// also belong to this table. Telemt writes a section's sub-tables
|
||||
// contiguously, but a hand-edited config may scatter them; dropping the
|
||||
// extras here prevents the duplicate-table corruption that would
|
||||
// otherwise break config load.
|
||||
let mut out = String::with_capacity(source.len() + replacement.len());
|
||||
out.push_str(&source[..start]);
|
||||
out.push_str(&source[..first_start]);
|
||||
out.push_str(replacement);
|
||||
out.push_str(&source[end..]);
|
||||
let mut cursor = first_end;
|
||||
for &(start, end) in &blocks[1..] {
|
||||
out.push_str(&source[cursor..start]);
|
||||
cursor = end;
|
||||
}
|
||||
out.push_str(&source[cursor..]);
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -260,24 +382,62 @@ fn upsert_toml_table(source: &str, table_name: &str, replacement: &str) -> Strin
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a (comment-stripped, trimmed) TOML header line belongs to
|
||||
/// `table_name`: the table itself (`[X]` / `[[X]]`) or any of its nested
|
||||
/// sub-tables (`[X.…]` / `[[X.…]]`). The trailing dot guards against sibling
|
||||
/// prefixes — `access.users` must not match `access.user_enabled`.
|
||||
fn header_belongs_to(header: &str, table_name: &str) -> bool {
|
||||
let body = match header.strip_prefix("[[").and_then(|h| h.strip_suffix("]]")) {
|
||||
Some(body) => body,
|
||||
None => match header.strip_prefix('[').and_then(|h| h.strip_suffix(']')) {
|
||||
Some(body) => body,
|
||||
None => return false,
|
||||
},
|
||||
};
|
||||
let body = body.trim();
|
||||
body == table_name
|
||||
|| body
|
||||
.strip_prefix(table_name)
|
||||
.is_some_and(|rest| rest.starts_with('.'))
|
||||
}
|
||||
|
||||
/// Locate the first contiguous byte range covering `table_name` and the nested
|
||||
/// sub-tables immediately following it. Used for existence checks; see
|
||||
/// [`find_all_table_blocks`] for the full set of (possibly scattered) blocks.
|
||||
fn find_toml_table_bounds(source: &str, table_name: &str) -> Option<(usize, usize)> {
|
||||
let target = format!("[{}]", table_name);
|
||||
find_all_table_blocks(source, table_name).into_iter().next()
|
||||
}
|
||||
|
||||
/// Locate every byte range that belongs to `table_name`: the table header and
|
||||
/// its nested sub-tables. Returns one range per contiguous run, so a config
|
||||
/// where a section's sub-tables are scattered (e.g. hand-edited) yields several
|
||||
/// ranges — letting the caller collapse them into a single rendered block.
|
||||
fn find_all_table_blocks(source: &str, table_name: &str) -> Vec<(usize, usize)> {
|
||||
let mut blocks = Vec::new();
|
||||
let mut offset = 0usize;
|
||||
let mut start = None;
|
||||
let mut start: Option<usize> = None;
|
||||
|
||||
for line in source.split_inclusive('\n') {
|
||||
let trimmed = line.trim();
|
||||
// Drop any inline comment so a hand-edited header like
|
||||
// `[censorship] # note` still matches. Section names never contain `#`.
|
||||
let header = line.trim().split('#').next().unwrap_or("").trim();
|
||||
let is_header = header.starts_with('[');
|
||||
if let Some(start_offset) = start {
|
||||
if trimmed.starts_with('[') {
|
||||
return Some((start_offset, offset));
|
||||
if is_header && !header_belongs_to(header, table_name) {
|
||||
blocks.push((start_offset, offset));
|
||||
start = None;
|
||||
}
|
||||
} else if trimmed == target {
|
||||
}
|
||||
if start.is_none() && header_belongs_to(header, table_name) {
|
||||
start = Some(offset);
|
||||
}
|
||||
offset = offset.saturating_add(line.len());
|
||||
}
|
||||
|
||||
start.map(|start_offset| (start_offset, source.len()))
|
||||
if let Some(start_offset) = start {
|
||||
blocks.push((start_offset, source.len()));
|
||||
}
|
||||
blocks
|
||||
}
|
||||
|
||||
async fn write_atomic(path: PathBuf, contents: String) -> Result<(), ApiFailure> {
|
||||
@@ -324,6 +484,189 @@ fn write_atomic_sync(path: &Path, contents: &str) -> std::io::Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn save_sections_preserves_other_tables_and_comments() {
|
||||
let dir = std::env::temp_dir().join(format!("cfgtest-{}", rand::random::<u64>()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("config.toml");
|
||||
std::fs::write(
|
||||
&path,
|
||||
"# top comment\n[censorship]\ntls_domain = \"old.example\"\n\n[server]\nport = 443\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.censorship.tls_domain = "new.example".to_string();
|
||||
cfg.server.port = 443;
|
||||
|
||||
let rev = save_sections_to_disk(&path, &cfg, &["censorship"])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let written = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(written.contains("tls_domain = \"new.example\""));
|
||||
assert!(written.contains("# top comment")); // untouched comment kept
|
||||
assert!(written.contains("[server]\nport = 443")); // untouched table kept
|
||||
assert_eq!(rev, compute_revision(&written));
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_bounds_matches_array_of_tables() {
|
||||
let src =
|
||||
"[server]\nport = 1\n\n[[upstreams]]\nkind = \"a\"\n\n[[upstreams]]\nkind = \"b\"\n";
|
||||
let bounds = find_toml_table_bounds(src, "upstreams");
|
||||
assert!(bounds.is_some(), "should locate [[upstreams]] block start");
|
||||
let (start, end) = bounds.unwrap();
|
||||
let slice = &src[start..end];
|
||||
assert!(slice.starts_with("[[upstreams]]"));
|
||||
assert!(slice.contains("kind = \"b\"")); // spans through the last upstream block
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_bounds_matches_header_with_inline_comment() {
|
||||
let src = "[censorship] # notes\ntls_domain = \"a\"\n\n[server]\nport = 1\n";
|
||||
let bounds = find_toml_table_bounds(src, "censorship");
|
||||
assert!(bounds.is_some(), "commented header must still match");
|
||||
let (start, end) = bounds.unwrap();
|
||||
let slice = &src[start..end];
|
||||
assert!(slice.starts_with("[censorship] # notes"));
|
||||
assert!(slice.contains("tls_domain"));
|
||||
assert!(!slice.contains("[server]")); // terminates at the next header
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn save_general_section_keeps_subtables_dotted_without_duplicates() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("config.toml");
|
||||
tokio::fs::write(
|
||||
&path,
|
||||
"[general]\nprefer_ipv6 = false\n\n[general.modes]\ntls = true\n\n\
|
||||
[general.links]\npublic_host = \"old.example\"\n\n[server]\nport = 443\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.prefer_ipv6 = true;
|
||||
|
||||
save_sections_to_disk(&path, &cfg, &["general"])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
|
||||
// No bare top-level [modes] / [links] headers leaked.
|
||||
for line in written.lines() {
|
||||
let header = line.trim();
|
||||
assert_ne!(header, "[modes]", "leaked top-level [modes]:\n{written}");
|
||||
assert_ne!(header, "[links]", "leaked top-level [links]:\n{written}");
|
||||
}
|
||||
|
||||
// Sub-tables kept their dotted prefix exactly once each.
|
||||
assert_eq!(
|
||||
written.matches("[general.modes]").count(),
|
||||
1,
|
||||
"[general.modes] must appear exactly once:\n{written}"
|
||||
);
|
||||
assert_eq!(
|
||||
written.matches("[general.links]").count(),
|
||||
1,
|
||||
"[general.links] must appear exactly once:\n{written}"
|
||||
);
|
||||
|
||||
// Result parses (duplicate tables would error here).
|
||||
toml::from_str::<toml::Value>(&written)
|
||||
.unwrap_or_else(|e| panic!("written config must parse: {e}\n{written}"));
|
||||
|
||||
assert!(written.contains("[server]\nport = 443")); // untouched table kept
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn save_general_section_is_idempotent_across_repeated_saves() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("config.toml");
|
||||
tokio::fs::write(
|
||||
&path,
|
||||
"[general]\nprefer_ipv6 = false\n\n[general.modes]\ntls = true\n\n\
|
||||
[general.links]\npublic_host = \"old.example\"\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.prefer_ipv6 = true;
|
||||
|
||||
save_sections_to_disk(&path, &cfg, &["general"])
|
||||
.await
|
||||
.unwrap();
|
||||
save_sections_to_disk(&path, &cfg, &["general"])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
assert_eq!(written.matches("[general.modes]").count(), 1, "{written}");
|
||||
assert_eq!(written.matches("[general.links]").count(), 1, "{written}");
|
||||
assert_eq!(written.matches("[general]").count(), 1, "{written}");
|
||||
toml::from_str::<toml::Value>(&written)
|
||||
.unwrap_or_else(|e| panic!("written config must parse: {e}\n{written}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_bounds_spans_dotted_subtables() {
|
||||
let src = "[general]\nprefer_ipv6 = false\n\n[general.modes]\ntls = true\n\n\
|
||||
[general.links]\npublic_host = \"a\"\n\n[server]\nport = 1\n";
|
||||
let bounds = find_toml_table_bounds(src, "general");
|
||||
assert!(bounds.is_some(), "should locate [general] block");
|
||||
let (start, end) = bounds.unwrap();
|
||||
let slice = &src[start..end];
|
||||
assert!(slice.starts_with("[general]"));
|
||||
assert!(slice.contains("[general.modes]")); // spans nested sub-tables
|
||||
assert!(slice.contains("[general.links]"));
|
||||
assert!(!slice.contains("[server]")); // terminates at the next unrelated header
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_bounds_does_not_overrun_sibling_prefix() {
|
||||
// access.users must not swallow access.user_enabled (dot guards the prefix).
|
||||
let src = "[access.users]\nalice = \"x\"\n\n[access.user_enabled]\nalice = true\n";
|
||||
let bounds = find_toml_table_bounds(src, "access.users").unwrap();
|
||||
let slice = &src[bounds.0..bounds.1];
|
||||
assert!(slice.starts_with("[access.users]"));
|
||||
assert!(!slice.contains("[access.user_enabled]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn save_general_handles_non_contiguous_subtables() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("config.toml");
|
||||
// Hand-edited layout: [general.modes] sits AFTER an unrelated [server].
|
||||
tokio::fs::write(
|
||||
&path,
|
||||
"[general]\nprefer_ipv6 = false\n\n[server]\nport = 443\n\n\
|
||||
[general.modes]\ntls = true\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.general.prefer_ipv6 = true;
|
||||
|
||||
save_sections_to_disk(&path, &cfg, &["general"])
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let written = tokio::fs::read_to_string(&path).await.unwrap();
|
||||
assert_eq!(
|
||||
written.matches("[general.modes]").count(),
|
||||
1,
|
||||
"non-contiguous [general.modes] must not duplicate:\n{written}"
|
||||
);
|
||||
toml::from_str::<toml::Value>(&written)
|
||||
.unwrap_or_else(|e| panic!("written config must parse: {e}\n{written}"));
|
||||
assert!(written.contains("[server]")); // unrelated section preserved
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_user_rate_limits_section() {
|
||||
let mut cfg = ProxyConfig::default();
|
||||
|
||||
378
src/api/mod.rs
378
src/api/mod.rs
@@ -7,6 +7,7 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use http_body_util::Full;
|
||||
use hyper::body::{Bytes, Incoming};
|
||||
use hyper::header::AUTHORIZATION;
|
||||
@@ -19,19 +20,25 @@ use tokio::sync::{Mutex, RwLock, Semaphore, watch};
|
||||
use tokio::time::timeout;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::{ApiGrayAction, ProxyConfig};
|
||||
use crate::config::ApiGrayAction;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::maestro::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
use crate::maestro::reload::{ReloadAccepted, ReloadControl, ReloadRequest, ReloadSubmitError};
|
||||
use crate::proxy::route_mode::RouteRuntimeController;
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::startup::StartupTracker;
|
||||
use crate::stats::Stats;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
mod config_store;
|
||||
mod config_edit;
|
||||
pub(crate) mod config_store;
|
||||
mod events;
|
||||
mod http_utils;
|
||||
mod model;
|
||||
mod patch;
|
||||
#[cfg(test)]
|
||||
mod reload_tests;
|
||||
mod runtime_edge;
|
||||
mod runtime_init;
|
||||
mod runtime_min;
|
||||
@@ -42,7 +49,8 @@ mod runtime_zero;
|
||||
mod users;
|
||||
|
||||
use config_store::{
|
||||
current_revision, ensure_expected_revision, load_config_from_disk, parse_if_match,
|
||||
current_revision, ensure_expected_revision, load_config_for_reload, load_config_from_disk,
|
||||
parse_if_match,
|
||||
};
|
||||
use events::ApiEventStore;
|
||||
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||
@@ -51,9 +59,10 @@ use model::{
|
||||
PatchUserRequest, ResetUserQuotaResponse, RotateSecretRequest, SummaryData, UserActiveIps,
|
||||
is_valid_username,
|
||||
};
|
||||
use patch::Patch;
|
||||
use runtime_edge::{
|
||||
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
||||
build_runtime_events_recent_data,
|
||||
build_runtime_events_recent_data, build_runtime_tls_fingerprints_data,
|
||||
};
|
||||
use runtime_init::build_runtime_initialization_data;
|
||||
use runtime_min::{
|
||||
@@ -71,7 +80,8 @@ use runtime_zero::{
|
||||
build_system_info_data,
|
||||
};
|
||||
use users::{
|
||||
build_user_quota_list, create_user, delete_user, patch_user, rotate_secret, users_from_config,
|
||||
build_user_quota_list, create_user, delete_user, patch_user, rotate_secret, set_user_enabled,
|
||||
users_from_config,
|
||||
};
|
||||
|
||||
const API_MAX_CONTROL_CONNECTIONS: usize = 1024;
|
||||
@@ -81,6 +91,7 @@ const ALLOW_GET: &str = "GET";
|
||||
const ALLOW_POST: &str = "POST";
|
||||
const ALLOW_GET_POST: &str = "GET, POST";
|
||||
const ALLOW_GET_PATCH_DELETE: &str = "GET, PATCH, DELETE";
|
||||
const ALLOW_GET_PATCH: &str = "GET, PATCH";
|
||||
|
||||
pub(super) struct ApiRuntimeState {
|
||||
pub(super) process_started_at_epoch_secs: u64,
|
||||
@@ -102,11 +113,15 @@ pub(super) struct ApiShared {
|
||||
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
||||
pub(super) runtime_edge_connections_cache: Arc<Mutex<Option<EdgeConnectionsCacheEntry>>>,
|
||||
pub(super) runtime_edge_recompute_lock: Arc<Mutex<()>>,
|
||||
pub(super) cache_generation: Arc<AtomicU64>,
|
||||
pub(super) runtime_events: Arc<ApiEventStore>,
|
||||
pub(super) request_id: Arc<AtomicU64>,
|
||||
pub(super) runtime_state: Arc<ApiRuntimeState>,
|
||||
pub(super) startup_tracker: Arc<StartupTracker>,
|
||||
pub(super) route_runtime: Arc<RouteRuntimeController>,
|
||||
pub(super) proxy_shared: Arc<ProxySharedState>,
|
||||
pub(super) reload_control: ReloadControl,
|
||||
pub(super) active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
}
|
||||
|
||||
impl ApiShared {
|
||||
@@ -117,6 +132,31 @@ impl ApiShared {
|
||||
fn detected_link_ips(&self) -> (Option<IpAddr>, Option<IpAddr>) {
|
||||
*self.detected_ips_rx.borrow()
|
||||
}
|
||||
|
||||
fn for_runtime(&self, runtime: &RuntimeGeneration) -> Self {
|
||||
Self {
|
||||
stats: runtime.stats.clone(),
|
||||
ip_tracker: runtime.ip_tracker.clone(),
|
||||
me_pool: runtime.me_pool_runtime.clone(),
|
||||
upstream_manager: runtime.upstream_manager.clone(),
|
||||
config_path: self.config_path.clone(),
|
||||
quota_state_path: self.quota_state_path.clone(),
|
||||
detected_ips_rx: self.detected_ips_rx.clone(),
|
||||
mutation_lock: self.mutation_lock.clone(),
|
||||
minimal_cache: self.minimal_cache.clone(),
|
||||
runtime_edge_connections_cache: self.runtime_edge_connections_cache.clone(),
|
||||
runtime_edge_recompute_lock: self.runtime_edge_recompute_lock.clone(),
|
||||
cache_generation: self.cache_generation.clone(),
|
||||
runtime_events: self.runtime_events.clone(),
|
||||
request_id: self.request_id.clone(),
|
||||
runtime_state: self.runtime_state.clone(),
|
||||
startup_tracker: self.startup_tracker.clone(),
|
||||
route_runtime: runtime.route_runtime.clone(),
|
||||
proxy_shared: runtime.proxy_shared.clone(),
|
||||
reload_control: self.reload_control.clone(),
|
||||
active_runtime: self.active_runtime.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn auth_header_matches(actual: &str, expected: &str) -> bool {
|
||||
@@ -138,6 +178,41 @@ fn user_action_route_matches(path: &str, suffix: &str) -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn reload_status_route_id(path: &str) -> Option<u64> {
|
||||
path.strip_prefix("/v1/system/reload/")
|
||||
.filter(|id| !id.is_empty() && !id.contains('/'))
|
||||
.and_then(|id| id.parse().ok())
|
||||
}
|
||||
|
||||
async fn submit_reload_from_disk(
|
||||
config_path: &std::path::Path,
|
||||
mutation_lock: &Mutex<()>,
|
||||
reload_control: &ReloadControl,
|
||||
expected_revision: Option<&str>,
|
||||
request: ReloadRequest,
|
||||
) -> Result<(ReloadAccepted, String), ApiFailure> {
|
||||
let _guard = mutation_lock.lock().await;
|
||||
ensure_expected_revision(config_path, expected_revision).await?;
|
||||
let revision = current_revision(config_path).await?;
|
||||
let config = Arc::new(load_config_for_reload(config_path).await?);
|
||||
let accepted = reload_control
|
||||
.submit(config, revision.clone(), request)
|
||||
.await
|
||||
.map_err(|error| match error {
|
||||
ReloadSubmitError::InProgress(reload_id) => ApiFailure::new(
|
||||
StatusCode::CONFLICT,
|
||||
"reload_in_progress",
|
||||
format!("Reload {} is already in progress", reload_id),
|
||||
),
|
||||
ReloadSubmitError::MaestroUnavailable => ApiFailure::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"maestro_unavailable",
|
||||
"Maestro reload coordinator is unavailable",
|
||||
),
|
||||
})?;
|
||||
Ok((accepted, revision))
|
||||
}
|
||||
|
||||
fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||
match path {
|
||||
"/v1/health"
|
||||
@@ -165,12 +240,18 @@ fn allowed_methods_for_path(path: &str) -> Option<&'static str> {
|
||||
| "/v1/runtime/me-selftest"
|
||||
| "/v1/runtime/connections/summary"
|
||||
| "/v1/runtime/events/recent"
|
||||
| "/v1/runtime/tls-fingerprints"
|
||||
| "/v1/stats/users/active-ips"
|
||||
| "/v1/stats/users/quota"
|
||||
| "/v1/stats/users" => Some(ALLOW_GET),
|
||||
"/v1/system/reload" => Some(ALLOW_POST),
|
||||
"/v1/users" => Some(ALLOW_GET_POST),
|
||||
"/v1/config" => Some(ALLOW_GET_PATCH),
|
||||
_ if user_action_route_matches(path, "/reset-quota") => Some(ALLOW_POST),
|
||||
_ if user_action_route_matches(path, "/rotate-secret") => Some(ALLOW_POST),
|
||||
_ if user_action_route_matches(path, "/enable") => Some(ALLOW_POST),
|
||||
_ if user_action_route_matches(path, "/disable") => Some(ALLOW_POST),
|
||||
_ if reload_status_route_id(path).is_some() => Some(ALLOW_GET),
|
||||
_ if path
|
||||
.strip_prefix("/v1/users/")
|
||||
.map(|user| !user.is_empty() && !user.contains('/'))
|
||||
@@ -188,15 +269,37 @@ pub async fn serve(
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
proxy_shared: Arc<ProxySharedState>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
admission_rx: watch::Receiver<bool>,
|
||||
config_path: PathBuf,
|
||||
quota_state_path: PathBuf,
|
||||
detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
||||
process_started_at_epoch_secs: u64,
|
||||
startup_tracker: Arc<StartupTracker>,
|
||||
reload_control: ReloadControl,
|
||||
mut active_runtime_rx: watch::Receiver<Option<Arc<ArcSwap<RuntimeGeneration>>>>,
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
) {
|
||||
let active_runtime = loop {
|
||||
if let Some(active_runtime) = active_runtime_rx.borrow().clone() {
|
||||
break active_runtime;
|
||||
}
|
||||
if active_runtime_rx.changed().await.is_err() {
|
||||
warn!("Runtime generation channel closed before API bootstrap");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let initial_watch_state = loop {
|
||||
if let Some(watch_state) = runtime_watch_rx.borrow().clone() {
|
||||
break watch_state;
|
||||
}
|
||||
if runtime_watch_rx.changed().await.is_err() {
|
||||
warn!("Runtime watch channel closed before API bootstrap");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let config_rx = initial_watch_state.config_rx.clone();
|
||||
let admission_rx = initial_watch_state.admission_rx.clone();
|
||||
let listener = match TcpListener::bind(listen).await {
|
||||
Ok(listener) => listener,
|
||||
Err(error) => {
|
||||
@@ -230,6 +333,7 @@ pub async fn serve(
|
||||
minimal_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_connections_cache: Arc::new(Mutex::new(None)),
|
||||
runtime_edge_recompute_lock: Arc::new(Mutex::new(())),
|
||||
cache_generation: Arc::new(AtomicU64::new(1)),
|
||||
runtime_events: Arc::new(ApiEventStore::new(
|
||||
config_rx.borrow().server.api.runtime_edge_events_capacity,
|
||||
)),
|
||||
@@ -237,11 +341,13 @@ pub async fn serve(
|
||||
runtime_state: runtime_state.clone(),
|
||||
startup_tracker,
|
||||
route_runtime,
|
||||
proxy_shared,
|
||||
reload_control,
|
||||
active_runtime,
|
||||
});
|
||||
|
||||
spawn_runtime_watchers(
|
||||
config_rx.clone(),
|
||||
admission_rx.clone(),
|
||||
runtime_watch_rx,
|
||||
runtime_state.clone(),
|
||||
shared.runtime_events.clone(),
|
||||
);
|
||||
@@ -270,13 +376,11 @@ pub async fn serve(
|
||||
};
|
||||
|
||||
let shared_conn = shared.clone();
|
||||
let config_rx_conn = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
let _connection_permit = connection_permit;
|
||||
let svc = service_fn(move |req: Request<Incoming>| {
|
||||
let shared_req = shared_conn.clone();
|
||||
let config_rx_req = config_rx_conn.clone();
|
||||
async move { handle(req, peer, shared_req, config_rx_req).await }
|
||||
async move { handle(req, peer, shared_req).await }
|
||||
});
|
||||
match timeout(
|
||||
API_HTTP_CONNECTION_TIMEOUT,
|
||||
@@ -306,8 +410,19 @@ async fn handle(
|
||||
req: Request<Incoming>,
|
||||
peer: SocketAddr,
|
||||
shared: Arc<ApiShared>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||
let runtime = shared.active_runtime.load_full();
|
||||
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
|
||||
if previous_cache_generation != runtime.id {
|
||||
*shared.minimal_cache.lock().await = None;
|
||||
*shared.runtime_edge_connections_cache.lock().await = None;
|
||||
}
|
||||
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
|
||||
let config_rx = runtime.config_rx.clone();
|
||||
shared
|
||||
.runtime_state
|
||||
.admission_open
|
||||
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
|
||||
let request_id = shared.next_request_id();
|
||||
let cfg = config_rx.borrow().clone();
|
||||
let api_cfg = &cfg.server.api;
|
||||
@@ -532,6 +647,15 @@ async fn handle(
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/runtime/tls-fingerprints") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let data = build_runtime_tls_fingerprints_data(
|
||||
shared.as_ref(),
|
||||
cfg.as_ref(),
|
||||
query.as_deref(),
|
||||
);
|
||||
Ok(success_response(StatusCode::OK, data, revision))
|
||||
}
|
||||
("GET", "/v1/stats/users/active-ips") => {
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
||||
@@ -582,6 +706,7 @@ async fn handle(
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||
let requested_enabled = body.enabled;
|
||||
let result = create_user(body, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
@@ -594,6 +719,25 @@ async fn handle(
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||
if let Some(enabled) = requested_enabled {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.user.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled = shared
|
||||
.proxy_shared
|
||||
.cancel_user_sessions(&data.user.username);
|
||||
if cancelled > 0 {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.user.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
shared.runtime_events.record(
|
||||
"api.user.create.ok",
|
||||
format!("username={}", data.user.username),
|
||||
@@ -605,7 +749,185 @@ async fn handle(
|
||||
};
|
||||
Ok(success_response(status, data, revision))
|
||||
}
|
||||
("GET", "/v1/config") => {
|
||||
let (value, revision) =
|
||||
config_edit::read_managed_config(&shared.config_path).await?;
|
||||
Ok(success_response(StatusCode::OK, value, revision))
|
||||
}
|
||||
("POST", "/v1/system/reload") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
request.validate().map_err(ApiFailure::bad_request)?;
|
||||
|
||||
let (accepted, revision) = submit_reload_from_disk(
|
||||
&shared.config_path,
|
||||
shared.mutation_lock.as_ref(),
|
||||
&shared.reload_control,
|
||||
expected_revision.as_deref(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
|
||||
}
|
||||
("PATCH", "/v1/config") => {
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let reload_request =
|
||||
ReloadRequest::from_query(query.as_deref()).map_err(ApiFailure::bad_request)?;
|
||||
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
|
||||
match config_edit::patch_config(body, expected_revision, reload_request, &shared)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => {
|
||||
let revision = resp.revision.clone();
|
||||
let status = if resp.reload.is_some() {
|
||||
StatusCode::ACCEPTED
|
||||
} else {
|
||||
StatusCode::OK
|
||||
};
|
||||
Ok(success_response(status, resp, revision))
|
||||
}
|
||||
Err(error) => {
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.config.patch.failed", error.code);
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if method == Method::GET
|
||||
&& let Some(reload_id) = reload_status_route_id(normalized_path)
|
||||
{
|
||||
let revision = current_revision(&shared.config_path).await?;
|
||||
let status =
|
||||
shared
|
||||
.reload_control
|
||||
.status(reload_id)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"reload_not_found",
|
||||
format!("Reload {} was not found", reload_id),
|
||||
)
|
||||
})?;
|
||||
return Ok(success_response(StatusCode::OK, status, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/enable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result =
|
||||
set_user_enabled(base_user, true, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.enable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
shared.proxy_shared.set_user_enabled(base_user, true);
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.enable.ok", format!("username={}", base_user));
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(base_user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
.and_then(|path| path.strip_suffix("/disable"))
|
||||
&& !base_user.is_empty()
|
||||
&& !base_user.contains('/')
|
||||
{
|
||||
let base_user = parse_route_username(base_user)?;
|
||||
if api_cfg.read_only {
|
||||
return Ok(error_response(
|
||||
request_id,
|
||||
ApiFailure::new(
|
||||
StatusCode::FORBIDDEN,
|
||||
"read_only",
|
||||
"API runs in read-only mode",
|
||||
),
|
||||
));
|
||||
}
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let result =
|
||||
set_user_enabled(base_user, false, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
Err(error) => {
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.failed",
|
||||
format!("username={} code={}", base_user, error.code),
|
||||
);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(base_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.ok",
|
||||
format!(
|
||||
"username={} newly_disabled={} cancelled_sessions={}",
|
||||
base_user, newly_disabled, cancelled
|
||||
),
|
||||
);
|
||||
let status = if data.in_runtime {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::ACCEPTED
|
||||
};
|
||||
return Ok(success_response(status, data, revision));
|
||||
}
|
||||
if method == Method::POST
|
||||
&& let Some(user) = normalized_path
|
||||
.strip_prefix("/v1/users/")
|
||||
@@ -763,6 +1085,11 @@ async fn handle(
|
||||
let expected_revision = parse_if_match(req.headers());
|
||||
let body =
|
||||
read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||
let enabled_update = match &body.enabled {
|
||||
Patch::Unchanged => None,
|
||||
Patch::Remove => Some(true),
|
||||
Patch::Set(enabled) => Some(*enabled),
|
||||
};
|
||||
let result = patch_user(user, body, expected_revision, &shared).await;
|
||||
let (mut data, revision) = match result {
|
||||
Ok(ok) => ok,
|
||||
@@ -776,6 +1103,22 @@ async fn handle(
|
||||
};
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||
if let Some(enabled) = enabled_update {
|
||||
shared
|
||||
.proxy_shared
|
||||
.set_user_enabled(&data.username, enabled);
|
||||
if !enabled {
|
||||
let cancelled =
|
||||
shared.proxy_shared.cancel_user_sessions(&data.username);
|
||||
shared.runtime_events.record(
|
||||
"api.user.disable.runtime",
|
||||
format!(
|
||||
"username={} cancelled_sessions={}",
|
||||
data.username, cancelled
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||
@@ -809,9 +1152,12 @@ async fn handle(
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
shared
|
||||
.runtime_events
|
||||
.record("api.user.delete.ok", format!("username={}", deleted_user));
|
||||
shared.proxy_shared.set_user_enabled(&deleted_user, true);
|
||||
let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user);
|
||||
shared.runtime_events.record(
|
||||
"api.user.delete.ok",
|
||||
format!("username={} cancelled_sessions={}", deleted_user, cancelled),
|
||||
);
|
||||
let runtime_cfg = config_rx.borrow().clone();
|
||||
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
|
||||
let response = DeleteUserResponse {
|
||||
|
||||
@@ -479,6 +479,7 @@ pub(super) struct TlsDomainLink {
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct UserInfo {
|
||||
pub(super) username: String,
|
||||
pub(super) enabled: bool,
|
||||
pub(super) in_runtime: bool,
|
||||
pub(super) user_ad_tag: Option<String>,
|
||||
pub(super) max_tcp_conns: Option<usize>,
|
||||
@@ -545,6 +546,7 @@ pub(super) struct CreateUserRequest {
|
||||
pub(super) rate_limit_up_bps: Option<u64>,
|
||||
pub(super) rate_limit_down_bps: Option<u64>,
|
||||
pub(super) max_unique_ips: Option<usize>,
|
||||
pub(super) enabled: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -564,6 +566,8 @@ pub(super) struct PatchUserRequest {
|
||||
pub(super) rate_limit_down_bps: Patch<u64>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) max_unique_ips: Patch<usize>,
|
||||
#[serde(default, deserialize_with = "patch_field")]
|
||||
pub(super) enabled: Patch<bool>,
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
|
||||
119
src/api/reload_tests.rs
Normal file
119
src/api/reload_tests.rs
Normal file
@@ -0,0 +1,119 @@
|
||||
use super::*;
|
||||
use crate::config::ProxyConfig;
|
||||
|
||||
async fn config_file() -> (tempfile::TempDir, PathBuf, String) {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("config.toml");
|
||||
let mut config = ProxyConfig::default();
|
||||
config.server.max_connections = 4_242;
|
||||
let body = toml::to_string_pretty(&config).unwrap();
|
||||
tokio::fs::write(&path, &body).await.unwrap();
|
||||
let revision = config_store::compute_revision(&body);
|
||||
(directory, path, revision)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_submission_uses_matching_disk_revision_and_snapshot() {
|
||||
let (_directory, path, revision) = config_file().await;
|
||||
let mutation_lock = Mutex::new(());
|
||||
let (control, mut commands) = ReloadControl::channel(1);
|
||||
let request = ReloadRequest::default();
|
||||
|
||||
let (accepted, response_revision) = submit_reload_from_disk(
|
||||
&path,
|
||||
&mutation_lock,
|
||||
&control,
|
||||
Some(&revision),
|
||||
request.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let command = commands.recv().await.unwrap();
|
||||
|
||||
assert_eq!(response_revision, revision);
|
||||
assert_eq!(accepted.config_revision, revision);
|
||||
assert_eq!(command.config_revision, revision);
|
||||
assert_eq!(command.request, request);
|
||||
assert_eq!(command.config.server.max_connections, 4_242);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn revision_conflict_rejects_without_enqueuing_reload() {
|
||||
let (_directory, path, _revision) = config_file().await;
|
||||
let mutation_lock = Mutex::new(());
|
||||
let (control, _commands) = ReloadControl::channel(1);
|
||||
|
||||
let error = submit_reload_from_disk(
|
||||
&path,
|
||||
&mutation_lock,
|
||||
&control,
|
||||
Some("stale-revision"),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(error.status, StatusCode::CONFLICT);
|
||||
assert_eq!(error.code, "revision_conflict");
|
||||
assert_eq!(control.in_progress().await, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_conflict_and_closed_coordinator_map_to_http_contract() {
|
||||
let (_directory, path, _revision) = config_file().await;
|
||||
let mutation_lock = Mutex::new(());
|
||||
let (control, mut commands) = ReloadControl::channel(1);
|
||||
let _accepted = submit_reload_from_disk(
|
||||
&path,
|
||||
&mutation_lock,
|
||||
&control,
|
||||
None,
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = commands.recv().await.unwrap();
|
||||
|
||||
let conflict = submit_reload_from_disk(
|
||||
&path,
|
||||
&mutation_lock,
|
||||
&control,
|
||||
None,
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(conflict.status, StatusCode::CONFLICT);
|
||||
assert_eq!(conflict.code, "reload_in_progress");
|
||||
|
||||
control.fail(1, "test cleanup").await;
|
||||
drop(commands);
|
||||
let unavailable = submit_reload_from_disk(
|
||||
&path,
|
||||
&mutation_lock,
|
||||
&control,
|
||||
None,
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(unavailable.status, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(unavailable.code, "maestro_unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_routes_expose_only_documented_methods_and_ids() {
|
||||
assert_eq!(
|
||||
allowed_methods_for_path("/v1/system/reload"),
|
||||
Some(ALLOW_POST)
|
||||
);
|
||||
assert_eq!(
|
||||
allowed_methods_for_path("/v1/system/reload/42"),
|
||||
Some(ALLOW_GET)
|
||||
);
|
||||
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
|
||||
assert_eq!(
|
||||
reload_status_route_id("/v1/system/reload/not-a-number"),
|
||||
None
|
||||
);
|
||||
}
|
||||
@@ -12,6 +12,8 @@ const FEATURE_DISABLED_REASON: &str = "feature_disabled";
|
||||
const SOURCE_UNAVAILABLE_REASON: &str = "source_unavailable";
|
||||
const EVENTS_DEFAULT_LIMIT: usize = 50;
|
||||
const EVENTS_MAX_LIMIT: usize = 1000;
|
||||
const TLS_FINGERPRINTS_MAX_LIMIT: usize = 1000;
|
||||
const RUNTIME_EDGE_RETENTION_MAX_MINUTES: u64 = 24 * 60;
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
pub(super) struct RuntimeEdgeConnectionUserData {
|
||||
@@ -90,6 +92,44 @@ pub(super) struct RuntimeEdgeEventsData {
|
||||
pub(super) data: Option<RuntimeEdgeEventsPayload>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct RuntimeEdgeTlsFingerprintRow {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) scope: Option<String>,
|
||||
pub(super) ja3: String,
|
||||
pub(super) ja3_raw: String,
|
||||
pub(super) ja4: String,
|
||||
pub(super) ja4_raw: String,
|
||||
pub(super) total: u64,
|
||||
pub(super) auth_success: u64,
|
||||
pub(super) bad_or_probe: u64,
|
||||
pub(super) first_seen_epoch_secs: u64,
|
||||
pub(super) last_seen_epoch_secs: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct RuntimeEdgeTlsFingerprintsPayload {
|
||||
pub(super) limit: usize,
|
||||
pub(super) retention_secs: u64,
|
||||
pub(super) capacity: usize,
|
||||
pub(super) dropped_total: u64,
|
||||
pub(super) parse_error_total: u64,
|
||||
pub(super) by_fingerprint: Vec<RuntimeEdgeTlsFingerprintRow>,
|
||||
pub(super) by_ip: Vec<RuntimeEdgeTlsFingerprintRow>,
|
||||
pub(super) by_cidr: Vec<RuntimeEdgeTlsFingerprintRow>,
|
||||
pub(super) by_user: Vec<RuntimeEdgeTlsFingerprintRow>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(super) struct RuntimeEdgeTlsFingerprintsData {
|
||||
pub(super) enabled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) reason: Option<&'static str>,
|
||||
pub(super) generated_at_epoch_secs: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) data: Option<RuntimeEdgeTlsFingerprintsPayload>,
|
||||
}
|
||||
|
||||
pub(super) async fn build_runtime_connections_summary_data(
|
||||
shared: &ApiShared,
|
||||
cfg: &ProxyConfig,
|
||||
@@ -162,6 +202,65 @@ pub(super) fn build_runtime_events_recent_data(
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn build_runtime_tls_fingerprints_data(
|
||||
shared: &ApiShared,
|
||||
cfg: &ProxyConfig,
|
||||
query: Option<&str>,
|
||||
) -> RuntimeEdgeTlsFingerprintsData {
|
||||
let now_epoch_secs = now_epoch_secs();
|
||||
let api_cfg = &cfg.server.api;
|
||||
if !api_cfg.runtime_edge_enabled {
|
||||
return RuntimeEdgeTlsFingerprintsData {
|
||||
enabled: false,
|
||||
reason: Some(FEATURE_DISABLED_REASON),
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: None,
|
||||
};
|
||||
}
|
||||
|
||||
let limit = parse_recent_events_limit(
|
||||
query,
|
||||
api_cfg.runtime_edge_top_n.max(1),
|
||||
TLS_FINGERPRINTS_MAX_LIMIT,
|
||||
);
|
||||
let snapshot = shared
|
||||
.stats
|
||||
.tls_fingerprint_snapshot(runtime_edge_retention(cfg), limit);
|
||||
|
||||
RuntimeEdgeTlsFingerprintsData {
|
||||
enabled: true,
|
||||
reason: None,
|
||||
generated_at_epoch_secs: now_epoch_secs,
|
||||
data: Some(RuntimeEdgeTlsFingerprintsPayload {
|
||||
limit,
|
||||
retention_secs: snapshot.retention_secs,
|
||||
capacity: snapshot.capacity,
|
||||
dropped_total: snapshot.dropped_total,
|
||||
parse_error_total: snapshot.parse_error_total,
|
||||
by_fingerprint: snapshot
|
||||
.by_fingerprint
|
||||
.into_iter()
|
||||
.map(runtime_tls_fingerprint_row)
|
||||
.collect(),
|
||||
by_ip: snapshot
|
||||
.by_ip
|
||||
.into_iter()
|
||||
.map(runtime_tls_fingerprint_row)
|
||||
.collect(),
|
||||
by_cidr: snapshot
|
||||
.by_cidr
|
||||
.into_iter()
|
||||
.map(runtime_tls_fingerprint_row)
|
||||
.collect(),
|
||||
by_user: snapshot
|
||||
.by_user
|
||||
.into_iter()
|
||||
.map(runtime_tls_fingerprint_row)
|
||||
.collect(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_connections_payload_cached(
|
||||
shared: &ApiShared,
|
||||
cache_ttl_ms: u64,
|
||||
@@ -286,6 +385,35 @@ fn parse_recent_events_limit(query: Option<&str>, default_limit: usize, max_limi
|
||||
default_limit
|
||||
}
|
||||
|
||||
fn runtime_edge_retention(cfg: &ProxyConfig) -> Duration {
|
||||
let minutes = cfg
|
||||
.general
|
||||
.beobachten_minutes
|
||||
.clamp(1, RUNTIME_EDGE_RETENTION_MAX_MINUTES);
|
||||
Duration::from_secs(minutes.saturating_mul(60))
|
||||
}
|
||||
|
||||
fn runtime_tls_fingerprint_row(
|
||||
row: crate::stats::TlsFingerprintSnapshotRow,
|
||||
) -> RuntimeEdgeTlsFingerprintRow {
|
||||
RuntimeEdgeTlsFingerprintRow {
|
||||
scope: if row.scope_key.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(row.scope_key)
|
||||
},
|
||||
ja3: row.ja3,
|
||||
ja3_raw: row.ja3_raw,
|
||||
ja4: row.ja4,
|
||||
ja4_raw: row.ja4_raw,
|
||||
total: row.total,
|
||||
auth_success: row.auth_success,
|
||||
bad_or_probe: row.bad_or_probe,
|
||||
first_seen_epoch_secs: row.first_seen_epoch_secs,
|
||||
last_seen_epoch_secs: row.last_seen_epoch_secs,
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
|
||||
@@ -4,58 +4,184 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::maestro::generation::RuntimeWatchState;
|
||||
|
||||
use super::ApiRuntimeState;
|
||||
use super::events::ApiEventStore;
|
||||
|
||||
pub(super) fn spawn_runtime_watchers(
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
admission_rx: watch::Receiver<bool>,
|
||||
runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
runtime_state: Arc<ApiRuntimeState>,
|
||||
runtime_events: Arc<ApiEventStore>,
|
||||
) {
|
||||
let mut config_rx_reload = config_rx;
|
||||
let runtime_state_reload = runtime_state.clone();
|
||||
let runtime_events_reload = runtime_events.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
if config_rx_reload.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
runtime_state_reload
|
||||
.config_reload_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
runtime_state_reload
|
||||
.last_config_reload_epoch_secs
|
||||
.store(now_epoch_secs(), Ordering::Relaxed);
|
||||
runtime_events_reload.record("config.reload.applied", "config receiver updated");
|
||||
}
|
||||
});
|
||||
let _config_watcher = spawn_config_watcher(
|
||||
runtime_watch_rx.clone(),
|
||||
runtime_state.clone(),
|
||||
runtime_events.clone(),
|
||||
);
|
||||
let _admission_watcher =
|
||||
spawn_admission_watcher(runtime_watch_rx, runtime_state, runtime_events);
|
||||
}
|
||||
|
||||
let mut admission_rx_watch = admission_rx;
|
||||
fn spawn_config_watcher(
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
runtime_state: Arc<ApiRuntimeState>,
|
||||
runtime_events: Arc<ApiEventStore>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
runtime_state
|
||||
.admission_open
|
||||
.store(*admission_rx_watch.borrow(), Ordering::Relaxed);
|
||||
runtime_events.record(
|
||||
"admission.state",
|
||||
format!("accepting_new_connections={}", *admission_rx_watch.borrow()),
|
||||
);
|
||||
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
||||
return;
|
||||
};
|
||||
loop {
|
||||
if admission_rx_watch.changed().await.is_err() {
|
||||
break;
|
||||
tokio::select! {
|
||||
biased;
|
||||
changed = runtime_watch_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break;
|
||||
}
|
||||
let Some(next) = runtime_watch_rx.borrow().clone() else {
|
||||
continue;
|
||||
};
|
||||
if next.generation_id != current.generation_id {
|
||||
current = next;
|
||||
record_config_reload(
|
||||
&runtime_state,
|
||||
&runtime_events,
|
||||
format!("runtime generation {} activated", current.generation_id),
|
||||
);
|
||||
}
|
||||
}
|
||||
changed = current.config_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
let Some(next) = wait_for_new_generation(
|
||||
&mut runtime_watch_rx,
|
||||
current.generation_id,
|
||||
).await else {
|
||||
break;
|
||||
};
|
||||
current = next;
|
||||
record_config_reload(
|
||||
&runtime_state,
|
||||
&runtime_events,
|
||||
format!("runtime generation {} activated", current.generation_id),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if active_generation_id(&runtime_watch_rx) != Some(current.generation_id) {
|
||||
continue;
|
||||
}
|
||||
record_config_reload(
|
||||
&runtime_state,
|
||||
&runtime_events,
|
||||
format!("generation {} config receiver updated", current.generation_id),
|
||||
);
|
||||
}
|
||||
}
|
||||
let admission_open = *admission_rx_watch.borrow();
|
||||
runtime_state
|
||||
.admission_open
|
||||
.store(admission_open, Ordering::Relaxed);
|
||||
runtime_events.record(
|
||||
"admission.state",
|
||||
format!("accepting_new_connections={}", admission_open),
|
||||
);
|
||||
}
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
fn spawn_admission_watcher(
|
||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
runtime_state: Arc<ApiRuntimeState>,
|
||||
runtime_events: Arc<ApiEventStore>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
||||
return;
|
||||
};
|
||||
record_admission_state(&runtime_state, &runtime_events, ¤t);
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
changed = runtime_watch_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break;
|
||||
}
|
||||
let Some(next) = runtime_watch_rx.borrow().clone() else {
|
||||
continue;
|
||||
};
|
||||
if next.generation_id != current.generation_id {
|
||||
current = next;
|
||||
record_admission_state(&runtime_state, &runtime_events, ¤t);
|
||||
}
|
||||
}
|
||||
changed = current.admission_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
let Some(next) = wait_for_new_generation(
|
||||
&mut runtime_watch_rx,
|
||||
current.generation_id,
|
||||
).await else {
|
||||
break;
|
||||
};
|
||||
current = next;
|
||||
record_admission_state(&runtime_state, &runtime_events, ¤t);
|
||||
continue;
|
||||
}
|
||||
if active_generation_id(&runtime_watch_rx) == Some(current.generation_id) {
|
||||
record_admission_state(&runtime_state, &runtime_events, ¤t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn active_generation_id(
|
||||
runtime_watch_rx: &watch::Receiver<Option<RuntimeWatchState>>,
|
||||
) -> Option<u64> {
|
||||
runtime_watch_rx
|
||||
.borrow()
|
||||
.as_ref()
|
||||
.map(|state| state.generation_id)
|
||||
}
|
||||
|
||||
async fn wait_for_new_generation(
|
||||
runtime_watch_rx: &mut watch::Receiver<Option<RuntimeWatchState>>,
|
||||
previous_generation_id: u64,
|
||||
) -> Option<RuntimeWatchState> {
|
||||
loop {
|
||||
if let Some(state) = runtime_watch_rx.borrow().clone()
|
||||
&& state.generation_id != previous_generation_id
|
||||
{
|
||||
return Some(state);
|
||||
}
|
||||
if runtime_watch_rx.changed().await.is_err() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_config_reload(
|
||||
runtime_state: &ApiRuntimeState,
|
||||
runtime_events: &ApiEventStore,
|
||||
context: String,
|
||||
) {
|
||||
runtime_state
|
||||
.config_reload_count
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
runtime_state
|
||||
.last_config_reload_epoch_secs
|
||||
.store(now_epoch_secs(), Ordering::Relaxed);
|
||||
runtime_events.record("config.reload.applied", context);
|
||||
}
|
||||
|
||||
fn record_admission_state(
|
||||
runtime_state: &ApiRuntimeState,
|
||||
runtime_events: &ApiEventStore,
|
||||
current: &RuntimeWatchState,
|
||||
) {
|
||||
let admission_open = *current.admission_rx.borrow();
|
||||
runtime_state
|
||||
.admission_open
|
||||
.store(admission_open, Ordering::Relaxed);
|
||||
runtime_events.record(
|
||||
"admission.state",
|
||||
format!(
|
||||
"generation={} accepting_new_connections={}",
|
||||
current.generation_id, admission_open
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
@@ -64,3 +190,130 @@ fn now_epoch_secs() -> u64 {
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::ProxyConfig;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64};
|
||||
use std::time::Duration;
|
||||
|
||||
fn state(
|
||||
generation_id: u64,
|
||||
) -> (
|
||||
RuntimeWatchState,
|
||||
watch::Sender<Arc<ProxyConfig>>,
|
||||
watch::Sender<bool>,
|
||||
) {
|
||||
let (config_tx, config_rx) = watch::channel(Arc::new(ProxyConfig::default()));
|
||||
let (admission_tx, admission_rx) = watch::channel(true);
|
||||
(
|
||||
RuntimeWatchState {
|
||||
generation_id,
|
||||
config_rx,
|
||||
admission_rx,
|
||||
},
|
||||
config_tx,
|
||||
admission_tx,
|
||||
)
|
||||
}
|
||||
|
||||
fn runtime_state() -> Arc<ApiRuntimeState> {
|
||||
Arc::new(ApiRuntimeState {
|
||||
process_started_at_epoch_secs: 1,
|
||||
config_reload_count: AtomicU64::new(0),
|
||||
last_config_reload_epoch_secs: AtomicU64::new(0),
|
||||
admission_open: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
async fn wait_for_count(runtime_state: &ApiRuntimeState, expected: u64) {
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if runtime_state.config_reload_count.load(Ordering::Relaxed) == expected {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watchers_follow_only_the_active_generation() {
|
||||
let (initial, initial_config_tx, initial_admission_tx) = state(1);
|
||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
||||
let runtime_state = runtime_state();
|
||||
let events = Arc::new(ApiEventStore::new(16));
|
||||
spawn_runtime_watchers(runtime_watch_rx, runtime_state.clone(), events.clone());
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 0);
|
||||
initial_config_tx.send_replace(Arc::new(ProxyConfig::default()));
|
||||
wait_for_count(&runtime_state, 1).await;
|
||||
|
||||
let (next, next_config_tx, next_admission_tx) = state(2);
|
||||
runtime_watch_tx.send_replace(Some(next));
|
||||
wait_for_count(&runtime_state, 2).await;
|
||||
|
||||
initial_config_tx.send_replace(Arc::new(ProxyConfig::default()));
|
||||
initial_admission_tx.send_replace(false);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 2);
|
||||
assert!(runtime_state.admission_open.load(Ordering::Relaxed));
|
||||
|
||||
next_config_tx.send_replace(Arc::new(ProxyConfig::default()));
|
||||
next_admission_tx.send_replace(false);
|
||||
wait_for_count(&runtime_state, 3).await;
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while runtime_state.admission_open.load(Ordering::Relaxed) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let snapshot = events.snapshot(16);
|
||||
assert_eq!(
|
||||
snapshot
|
||||
.events
|
||||
.iter()
|
||||
.filter(|event| event.event_type == "config.reload.applied")
|
||||
.count(),
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn watcher_recovers_from_closed_generation_and_exits_with_process_channel() {
|
||||
let (initial, initial_config_tx, _initial_admission_tx) = state(1);
|
||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
||||
let runtime_state = runtime_state();
|
||||
let events = Arc::new(ApiEventStore::new(16));
|
||||
let watcher = spawn_config_watcher(runtime_watch_rx, runtime_state.clone(), events.clone());
|
||||
drop(initial_config_tx);
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let (next, next_config_tx, _next_admission_tx) = state(2);
|
||||
runtime_watch_tx.send_replace(Some(next));
|
||||
wait_for_count(&runtime_state, 1).await;
|
||||
next_config_tx.send_replace(Arc::new(ProxyConfig::default()));
|
||||
wait_for_count(&runtime_state, 2).await;
|
||||
|
||||
drop(runtime_watch_tx);
|
||||
tokio::time::timeout(Duration::from_secs(1), watcher)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
events
|
||||
.snapshot(16)
|
||||
.events
|
||||
.iter()
|
||||
.filter(|event| event.event_type == "config.reload.applied")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -162,7 +163,7 @@ pub(super) fn build_system_info_data(
|
||||
build_time_utc,
|
||||
rustc_version,
|
||||
process_started_at_epoch_secs: shared.runtime_state.process_started_at_epoch_secs,
|
||||
uptime_seconds: shared.stats.uptime_secs(),
|
||||
uptime_seconds: process_uptime_seconds(shared.runtime_state.process_started_at_epoch_secs),
|
||||
config_path: shared.config_path.display().to_string(),
|
||||
config_hash: revision.to_string(),
|
||||
config_reload_count: shared
|
||||
@@ -173,6 +174,18 @@ pub(super) fn build_system_info_data(
|
||||
}
|
||||
}
|
||||
|
||||
fn process_uptime_seconds(process_started_at_epoch_secs: u64) -> f64 {
|
||||
let now_epoch_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
process_uptime_seconds_at(process_started_at_epoch_secs, now_epoch_secs)
|
||||
}
|
||||
|
||||
fn process_uptime_seconds_at(process_started_at_epoch_secs: u64, now_epoch_secs: u64) -> f64 {
|
||||
now_epoch_secs.saturating_sub(process_started_at_epoch_secs) as f64
|
||||
}
|
||||
|
||||
pub(super) async fn build_runtime_gates_data(
|
||||
shared: &ApiShared,
|
||||
cfg: &ProxyConfig,
|
||||
@@ -339,3 +352,14 @@ fn me_writer_pick_mode_label(mode: MeWriterPickMode) -> &'static str {
|
||||
MeWriterPickMode::P2c => "p2c",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::process_uptime_seconds_at;
|
||||
|
||||
#[test]
|
||||
fn process_uptime_is_monotonic_and_saturating() {
|
||||
assert_eq!(process_uptime_seconds_at(100, 135), 35.0);
|
||||
assert_eq!(process_uptime_seconds_at(135, 100), 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
111
src/api/users.rs
111
src/api/users.rs
@@ -32,6 +32,7 @@ pub(super) async fn create_user(
|
||||
let touches_user_rate_limits =
|
||||
body.rate_limit_up_bps.is_some() || body.rate_limit_down_bps.is_some();
|
||||
let touches_user_max_unique_ips = body.max_unique_ips.is_some();
|
||||
let touches_user_enabled = matches!(body.enabled, Some(false));
|
||||
|
||||
if !is_valid_username(&body.username) {
|
||||
return Err(ApiFailure::bad_request(
|
||||
@@ -111,6 +112,9 @@ pub(super) async fn create_user(
|
||||
.user_max_unique_ips
|
||||
.insert(body.username.clone(), limit);
|
||||
}
|
||||
if matches!(body.enabled, Some(false)) {
|
||||
cfg.access.user_enabled.insert(body.username.clone(), false);
|
||||
}
|
||||
|
||||
cfg.validate()
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
@@ -134,6 +138,9 @@ pub(super) async fn create_user(
|
||||
if touches_user_max_unique_ips {
|
||||
touched_sections.push(AccessSection::UserMaxUniqueIps);
|
||||
}
|
||||
if touches_user_enabled {
|
||||
touched_sections.push(AccessSection::UserEnabled);
|
||||
}
|
||||
|
||||
let revision =
|
||||
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
|
||||
@@ -161,6 +168,7 @@ pub(super) async fn create_user(
|
||||
.find(|entry| entry.username == body.username)
|
||||
.unwrap_or(UserInfo {
|
||||
username: body.username.clone(),
|
||||
enabled: cfg.access.is_user_enabled(&body.username),
|
||||
in_runtime: false,
|
||||
user_ad_tag: None,
|
||||
max_tcp_conns: cfg
|
||||
@@ -202,6 +210,7 @@ pub(super) async fn patch_user(
|
||||
let touches_user_rate_limits = !matches!(&body.rate_limit_up_bps, Patch::Unchanged)
|
||||
|| !matches!(&body.rate_limit_down_bps, Patch::Unchanged);
|
||||
let touches_user_max_unique_ips = !matches!(&body.max_unique_ips, Patch::Unchanged);
|
||||
let touches_user_enabled = !matches!(&body.enabled, Patch::Unchanged);
|
||||
|
||||
if let Some(secret) = body.secret.as_ref()
|
||||
&& !is_valid_user_secret(secret)
|
||||
@@ -313,6 +322,15 @@ pub(super) async fn patch_user(
|
||||
Some(Some(limit))
|
||||
}
|
||||
};
|
||||
match body.enabled {
|
||||
Patch::Unchanged => {}
|
||||
Patch::Remove | Patch::Set(true) => {
|
||||
cfg.access.user_enabled.remove(user);
|
||||
}
|
||||
Patch::Set(false) => {
|
||||
cfg.access.user_enabled.insert(user.to_string(), false);
|
||||
}
|
||||
}
|
||||
|
||||
cfg.validate()
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
@@ -339,6 +357,9 @@ pub(super) async fn patch_user(
|
||||
if touches_user_max_unique_ips {
|
||||
touched_sections.push(AccessSection::UserMaxUniqueIps);
|
||||
}
|
||||
if touches_user_enabled {
|
||||
touched_sections.push(AccessSection::UserEnabled);
|
||||
}
|
||||
|
||||
let revision = if touched_sections.is_empty() {
|
||||
current_revision(&shared.config_path).await?
|
||||
@@ -399,6 +420,7 @@ pub(super) async fn rotate_secret(
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
let touched_sections = [
|
||||
AccessSection::Users,
|
||||
AccessSection::UserEnabled,
|
||||
AccessSection::UserAdTags,
|
||||
AccessSection::UserMaxTcpConns,
|
||||
AccessSection::UserExpirations,
|
||||
@@ -434,6 +456,55 @@ pub(super) async fn rotate_secret(
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) async fn set_user_enabled(
|
||||
user: &str,
|
||||
enabled: bool,
|
||||
expected_revision: Option<String>,
|
||||
shared: &ApiShared,
|
||||
) -> Result<(UserInfo, String), ApiFailure> {
|
||||
let _guard = shared.mutation_lock.lock().await;
|
||||
let mut cfg = load_config_from_disk(&shared.config_path).await?;
|
||||
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
|
||||
|
||||
if !cfg.access.users.contains_key(user) {
|
||||
return Err(ApiFailure::new(
|
||||
StatusCode::NOT_FOUND,
|
||||
"not_found",
|
||||
"User not found",
|
||||
));
|
||||
}
|
||||
|
||||
if enabled {
|
||||
cfg.access.user_enabled.remove(user);
|
||||
} else {
|
||||
cfg.access.user_enabled.insert(user.to_string(), false);
|
||||
}
|
||||
|
||||
cfg.validate()
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
let revision =
|
||||
save_access_sections_to_disk(&shared.config_path, &cfg, &[AccessSection::UserEnabled])
|
||||
.await?;
|
||||
drop(_guard);
|
||||
|
||||
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||
let users = users_from_config(
|
||||
&cfg,
|
||||
&shared.stats,
|
||||
&shared.ip_tracker,
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let user_info = users
|
||||
.into_iter()
|
||||
.find(|entry| entry.username == user)
|
||||
.ok_or_else(|| ApiFailure::internal("failed to build updated user view"))?;
|
||||
|
||||
Ok((user_info, revision))
|
||||
}
|
||||
|
||||
pub(super) async fn delete_user(
|
||||
user: &str,
|
||||
expected_revision: Option<String>,
|
||||
@@ -459,6 +530,7 @@ pub(super) async fn delete_user(
|
||||
}
|
||||
|
||||
cfg.access.users.remove(user);
|
||||
cfg.access.user_enabled.remove(user);
|
||||
cfg.access.user_ad_tags.remove(user);
|
||||
cfg.access.user_max_tcp_conns.remove(user);
|
||||
cfg.access.user_expirations.remove(user);
|
||||
@@ -470,6 +542,7 @@ pub(super) async fn delete_user(
|
||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||
let touched_sections = [
|
||||
AccessSection::Users,
|
||||
AccessSection::UserEnabled,
|
||||
AccessSection::UserAdTags,
|
||||
AccessSection::UserMaxTcpConns,
|
||||
AccessSection::UserExpirations,
|
||||
@@ -518,6 +591,7 @@ pub(super) async fn users_from_config(
|
||||
})
|
||||
.unwrap_or_else(empty_user_links);
|
||||
users.push(UserInfo {
|
||||
enabled: cfg.access.is_user_enabled(&username),
|
||||
in_runtime: runtime_cfg
|
||||
.map(|runtime| runtime.access.users.contains_key(&username))
|
||||
.unwrap_or(false),
|
||||
@@ -876,6 +950,43 @@ mod tests {
|
||||
assert_eq!(alice.rate_limit_down_bps, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn users_from_config_reports_user_enabled_default_and_override() {
|
||||
let mut cfg = ProxyConfig::default();
|
||||
cfg.access.users.insert(
|
||||
"alice".to_string(),
|
||||
"0123456789abcdef0123456789abcdef".to_string(),
|
||||
);
|
||||
cfg.access.users.insert(
|
||||
"bob".to_string(),
|
||||
"fedcba9876543210fedcba9876543210".to_string(),
|
||||
);
|
||||
cfg.access.user_enabled.insert("bob".to_string(), false);
|
||||
|
||||
let stats = Stats::new();
|
||||
let tracker = UserIpTracker::new();
|
||||
let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await;
|
||||
let alice = users
|
||||
.iter()
|
||||
.find(|entry| entry.username == "alice")
|
||||
.expect("alice must be present");
|
||||
let bob = users
|
||||
.iter()
|
||||
.find(|entry| entry.username == "bob")
|
||||
.expect("bob must be present");
|
||||
|
||||
assert!(alice.enabled);
|
||||
assert!(!bob.enabled);
|
||||
|
||||
cfg.access.user_enabled.insert("bob".to_string(), true);
|
||||
let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await;
|
||||
let bob = users
|
||||
.iter()
|
||||
.find(|entry| entry.username == "bob")
|
||||
.expect("bob must be present");
|
||||
assert!(bob.enabled);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn users_from_config_marks_runtime_membership_when_snapshot_is_provided() {
|
||||
let mut disk_cfg = ProxyConfig::default();
|
||||
|
||||
@@ -705,6 +705,9 @@ ignore_time_skew = false
|
||||
type = "direct"
|
||||
enabled = true
|
||||
weight = 10
|
||||
# Optional per-upstream DC family policy:
|
||||
# ipv6 = true
|
||||
# prefer = 6
|
||||
"#,
|
||||
username = username,
|
||||
secret = secret,
|
||||
|
||||
@@ -24,6 +24,10 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256;
|
||||
const DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED: bool = false;
|
||||
const DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED: bool = false;
|
||||
const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 4096;
|
||||
pub(crate) const ME_WRITER_BYTE_PERMIT_UNIT_BYTES: usize = 16 * 1024;
|
||||
pub(crate) const ME_WRITER_FRAME_OVERHEAD_RESERVE_BYTES: usize = 256;
|
||||
const DEFAULT_ME_WRITER_BYTE_BUDGET_BYTES: usize =
|
||||
32 * 1024 * 1024 + ME_WRITER_BYTE_PERMIT_UNIT_BYTES;
|
||||
const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 768;
|
||||
const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024;
|
||||
const DEFAULT_ME_READER_ROUTE_DATA_WAIT_MS: u64 = 2;
|
||||
@@ -35,6 +39,8 @@ const DEFAULT_ME_QUOTA_SOFT_OVERSHOOT_BYTES: u64 = 64 * 1024;
|
||||
const DEFAULT_ME_D2C_FRAME_BUF_SHRINK_THRESHOLD_BYTES: usize = 256 * 1024;
|
||||
const DEFAULT_DIRECT_RELAY_COPY_BUF_C2S_BYTES: usize = 64 * 1024;
|
||||
const DEFAULT_DIRECT_RELAY_COPY_BUF_S2C_BYTES: usize = 256 * 1024;
|
||||
pub(crate) const DIRECT_RELAY_BUFFER_BUDGET_UNIT_BYTES: usize = 4 * 1024;
|
||||
const DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES: usize = 0;
|
||||
const DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE: u8 = 3;
|
||||
const DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY: u64 = 1000;
|
||||
const DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY: u64 = 3000;
|
||||
@@ -54,6 +60,14 @@ const DEFAULT_CONNTRACK_CONTROL_ENABLED: bool = true;
|
||||
const DEFAULT_CONNTRACK_PRESSURE_HIGH_WATERMARK_PCT: u8 = 85;
|
||||
const DEFAULT_CONNTRACK_PRESSURE_LOW_WATERMARK_PCT: u8 = 70;
|
||||
const DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC: u64 = 4096;
|
||||
const DEFAULT_SYNLIMIT_SECONDS: u32 = 60;
|
||||
const DEFAULT_SYNLIMIT_HITCOUNT: u32 = 48;
|
||||
const DEFAULT_SYNLIMIT_BURST: u32 = 1;
|
||||
const DEFAULT_SYNLIMIT_IOS_SECONDS: u32 = 1;
|
||||
const DEFAULT_SYNLIMIT_IOS_HITCOUNT: u32 = 12;
|
||||
const DEFAULT_SYNLIMIT_IOS_BURST: u32 = 24;
|
||||
const DEFAULT_SYNLIMIT_HASHLIMIT_EXPIRE_MS: u32 = 60_000;
|
||||
const DEFAULT_SYNLIMIT_HASHLIMIT_SIZE: u32 = 32_768;
|
||||
const DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS: u32 = 2;
|
||||
const DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD: u32 = 5;
|
||||
const DEFAULT_UPSTREAM_CONNECT_BUDGET_MS: u64 = 3000;
|
||||
@@ -243,6 +257,38 @@ pub(crate) fn default_conntrack_delete_budget_per_sec() -> u64 {
|
||||
DEFAULT_CONNTRACK_DELETE_BUDGET_PER_SEC
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_seconds() -> u32 {
|
||||
DEFAULT_SYNLIMIT_SECONDS
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_hitcount() -> u32 {
|
||||
DEFAULT_SYNLIMIT_HITCOUNT
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_burst() -> u32 {
|
||||
DEFAULT_SYNLIMIT_BURST
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_ios_seconds() -> u32 {
|
||||
DEFAULT_SYNLIMIT_IOS_SECONDS
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_ios_hitcount() -> u32 {
|
||||
DEFAULT_SYNLIMIT_IOS_HITCOUNT
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_ios_burst() -> u32 {
|
||||
DEFAULT_SYNLIMIT_IOS_BURST
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_hashlimit_expire_ms() -> u32 {
|
||||
DEFAULT_SYNLIMIT_HASHLIMIT_EXPIRE_MS
|
||||
}
|
||||
|
||||
pub(crate) fn default_synlimit_hashlimit_size() -> u32 {
|
||||
DEFAULT_SYNLIMIT_HASHLIMIT_SIZE
|
||||
}
|
||||
|
||||
pub(crate) fn default_prefer_4() -> u8 {
|
||||
4
|
||||
}
|
||||
@@ -415,6 +461,18 @@ pub(crate) fn default_me_writer_cmd_channel_capacity() -> usize {
|
||||
DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_writer_byte_budget_bytes() -> usize {
|
||||
DEFAULT_ME_WRITER_BYTE_BUDGET_BYTES
|
||||
}
|
||||
|
||||
pub(crate) fn minimum_me_writer_byte_budget_bytes(max_client_frame: usize) -> usize {
|
||||
max_client_frame
|
||||
.saturating_mul(2)
|
||||
.saturating_add(ME_WRITER_FRAME_OVERHEAD_RESERVE_BYTES)
|
||||
.div_ceil(ME_WRITER_BYTE_PERMIT_UNIT_BYTES)
|
||||
.saturating_mul(ME_WRITER_BYTE_PERMIT_UNIT_BYTES)
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_route_channel_capacity() -> usize {
|
||||
DEFAULT_ME_ROUTE_CHANNEL_CAPACITY
|
||||
}
|
||||
@@ -459,6 +517,10 @@ pub(crate) fn default_direct_relay_copy_buf_s2c_bytes() -> usize {
|
||||
DEFAULT_DIRECT_RELAY_COPY_BUF_S2C_BYTES
|
||||
}
|
||||
|
||||
pub(crate) fn default_direct_relay_buffer_budget_max_bytes() -> usize {
|
||||
DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES
|
||||
}
|
||||
|
||||
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||
}
|
||||
|
||||
@@ -16,10 +16,12 @@
|
||||
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
|
||||
//! | `network` | `dns_overrides` | Applied immediately |
|
||||
//! | `access` | All user/quota fields | Effective immediately |
|
||||
//! | `server.listeners` | `synlimit*` for existing endpoints | Netfilter rules reconciled immediately |
|
||||
//!
|
||||
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
||||
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||
//! applied; a warning is emitted.
|
||||
//! applied, except for SYN limiter fields on unchanged listener endpoints; a
|
||||
//! warning is emitted.
|
||||
//! Non-hot changes are never mixed into the runtime config snapshot.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
@@ -34,7 +36,8 @@ use tracing::{error, info, warn};
|
||||
|
||||
use super::load::{LoadedConfig, ProxyConfig};
|
||||
use crate::config::{
|
||||
LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel, MeWriterPickMode,
|
||||
CidrRateLimitKey, ListenerConfig, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy,
|
||||
MeTelemetryLevel, MeWriterPickMode, SynLimitMode,
|
||||
};
|
||||
|
||||
const HOT_RELOAD_DEBOUNCE: Duration = Duration::from_millis(50);
|
||||
@@ -118,18 +121,34 @@ pub struct HotFields {
|
||||
pub me_admission_poll_ms: u64,
|
||||
pub me_warn_rate_limit_ms: u64,
|
||||
pub users: std::collections::HashMap<String, String>,
|
||||
pub user_enabled: std::collections::HashMap<String, bool>,
|
||||
pub user_ad_tags: std::collections::HashMap<String, String>,
|
||||
pub user_max_tcp_conns: std::collections::HashMap<String, usize>,
|
||||
pub user_max_tcp_conns_global_each: usize,
|
||||
pub user_expirations: std::collections::HashMap<String, chrono::DateTime<chrono::Utc>>,
|
||||
pub user_data_quota: std::collections::HashMap<String, u64>,
|
||||
pub user_rate_limits: std::collections::HashMap<String, crate::config::RateLimitBps>,
|
||||
pub cidr_rate_limits:
|
||||
std::collections::HashMap<ipnetwork::IpNetwork, crate::config::RateLimitBps>,
|
||||
pub cidr_rate_limits: std::collections::HashMap<CidrRateLimitKey, crate::config::RateLimitBps>,
|
||||
pub user_max_unique_ips: std::collections::HashMap<String, usize>,
|
||||
pub user_max_unique_ips_global_each: usize,
|
||||
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
||||
pub user_max_unique_ips_window_secs: u64,
|
||||
pub listener_synlimit: Vec<ListenerSynLimitHotFields>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ListenerSynLimitHotFields {
|
||||
pub ip: IpAddr,
|
||||
pub port: Option<u16>,
|
||||
pub synlimit: SynLimitMode,
|
||||
pub synlimit_seconds: u32,
|
||||
pub synlimit_hitcount: u32,
|
||||
pub synlimit_burst: u32,
|
||||
pub synlimit_ios_seconds: u32,
|
||||
pub synlimit_ios_hitcount: u32,
|
||||
pub synlimit_ios_burst: u32,
|
||||
pub synlimit_hashlimit_expire_ms: u32,
|
||||
pub synlimit_hashlimit_size: u32,
|
||||
}
|
||||
|
||||
impl HotFields {
|
||||
@@ -247,6 +266,7 @@ impl HotFields {
|
||||
me_admission_poll_ms: cfg.general.me_admission_poll_ms,
|
||||
me_warn_rate_limit_ms: cfg.general.me_warn_rate_limit_ms,
|
||||
users: cfg.access.users.clone(),
|
||||
user_enabled: cfg.access.user_enabled.clone(),
|
||||
user_ad_tags: cfg.access.user_ad_tags.clone(),
|
||||
user_max_tcp_conns: cfg.access.user_max_tcp_conns.clone(),
|
||||
user_max_tcp_conns_global_each: cfg.access.user_max_tcp_conns_global_each,
|
||||
@@ -258,6 +278,30 @@ impl HotFields {
|
||||
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
||||
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
||||
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
|
||||
listener_synlimit: cfg
|
||||
.server
|
||||
.listeners
|
||||
.iter()
|
||||
.map(ListenerSynLimitHotFields::from_listener)
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ListenerSynLimitHotFields {
|
||||
fn from_listener(listener: &ListenerConfig) -> Self {
|
||||
Self {
|
||||
ip: listener.ip,
|
||||
port: listener.port,
|
||||
synlimit: listener.synlimit,
|
||||
synlimit_seconds: listener.synlimit_seconds,
|
||||
synlimit_hitcount: listener.synlimit_hitcount,
|
||||
synlimit_burst: listener.synlimit_burst,
|
||||
synlimit_ios_seconds: listener.synlimit_ios_seconds,
|
||||
synlimit_ios_hitcount: listener.synlimit_ios_hitcount,
|
||||
synlimit_ios_burst: listener.synlimit_ios_burst,
|
||||
synlimit_hashlimit_expire_ms: listener.synlimit_hashlimit_expire_ms,
|
||||
synlimit_hashlimit_size: listener.synlimit_hashlimit_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,6 +354,7 @@ fn listeners_equal(
|
||||
lhs.iter().zip(rhs.iter()).all(|(a, b)| {
|
||||
a.ip == b.ip
|
||||
&& a.port == b.port
|
||||
&& a.client_mss == b.client_mss
|
||||
&& a.announce == b.announce
|
||||
&& a.announce_ip == b.announce_ip
|
||||
&& a.proxy_protocol == b.proxy_protocol
|
||||
@@ -551,6 +596,7 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
||||
cfg.general.me_warn_rate_limit_ms = new.general.me_warn_rate_limit_ms;
|
||||
|
||||
cfg.access.users = new.access.users.clone();
|
||||
cfg.access.user_enabled = new.access.user_enabled.clone();
|
||||
cfg.access.user_ad_tags = new.access.user_ad_tags.clone();
|
||||
cfg.access.user_max_tcp_conns = new.access.user_max_tcp_conns.clone();
|
||||
cfg.access.user_max_tcp_conns_global_each = new.access.user_max_tcp_conns_global_each;
|
||||
@@ -562,6 +608,7 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
||||
cfg.access.user_max_unique_ips_global_each = new.access.user_max_unique_ips_global_each;
|
||||
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
|
||||
cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs;
|
||||
overlay_listener_synlimit_fields(&mut cfg.server.listeners, &new.server.listeners);
|
||||
|
||||
if cfg.rebuild_runtime_user_auth().is_err() {
|
||||
cfg.runtime_user_auth = None;
|
||||
@@ -570,6 +617,26 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
|
||||
cfg
|
||||
}
|
||||
|
||||
fn overlay_listener_synlimit_fields(old: &mut [ListenerConfig], new: &[ListenerConfig]) {
|
||||
if old.len() != new.len() {
|
||||
return;
|
||||
}
|
||||
for (old_listener, new_listener) in old.iter_mut().zip(new.iter()) {
|
||||
if old_listener.ip != new_listener.ip || old_listener.port != new_listener.port {
|
||||
continue;
|
||||
}
|
||||
old_listener.synlimit = new_listener.synlimit;
|
||||
old_listener.synlimit_seconds = new_listener.synlimit_seconds;
|
||||
old_listener.synlimit_hitcount = new_listener.synlimit_hitcount;
|
||||
old_listener.synlimit_burst = new_listener.synlimit_burst;
|
||||
old_listener.synlimit_ios_seconds = new_listener.synlimit_ios_seconds;
|
||||
old_listener.synlimit_ios_hitcount = new_listener.synlimit_ios_hitcount;
|
||||
old_listener.synlimit_ios_burst = new_listener.synlimit_ios_burst;
|
||||
old_listener.synlimit_hashlimit_expire_ms = new_listener.synlimit_hashlimit_expire_ms;
|
||||
old_listener.synlimit_hashlimit_size = new_listener.synlimit_hashlimit_size;
|
||||
}
|
||||
}
|
||||
|
||||
/// Warn if any non-hot fields changed (require restart).
|
||||
fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: bool) {
|
||||
let mut warned = false;
|
||||
@@ -605,6 +672,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
||||
|| old.server.listen_addr_ipv4 != new.server.listen_addr_ipv4
|
||||
|| old.server.listen_addr_ipv6 != new.server.listen_addr_ipv6
|
||||
|| old.server.listen_tcp != new.server.listen_tcp
|
||||
|| old.server.client_mss != new.server.client_mss
|
||||
|| old.server.listen_unix_sock != new.server.listen_unix_sock
|
||||
|| old.server.listen_unix_sock_perm != new.server.listen_unix_sock_perm
|
||||
{
|
||||
@@ -615,6 +683,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
||||
|| old.censorship.tls_domains != new.censorship.tls_domains
|
||||
|| old.censorship.tls_fetch_scope != new.censorship.tls_fetch_scope
|
||||
|| old.censorship.mask != new.censorship.mask
|
||||
|| old.censorship.mask_dynamic != new.censorship.mask_dynamic
|
||||
|| old.censorship.mask_host != new.censorship.mask_host
|
||||
|| old.censorship.mask_port != new.censorship.mask_port
|
||||
|| old.censorship.exclusive_mask != new.censorship.exclusive_mask
|
||||
@@ -844,6 +913,13 @@ fn log_changes(
|
||||
);
|
||||
}
|
||||
|
||||
if old_hot.listener_synlimit != new_hot.listener_synlimit {
|
||||
info!(
|
||||
"config reload: server.listeners SYN limiter updated ({} listeners)",
|
||||
new_hot.listener_synlimit.len()
|
||||
);
|
||||
}
|
||||
|
||||
if old_hot.desync_all_full != new_hot.desync_all_full {
|
||||
info!(
|
||||
"config reload: desync_all_full: {} → {}",
|
||||
@@ -1178,6 +1254,16 @@ fn log_changes(
|
||||
}
|
||||
}
|
||||
|
||||
if old_hot.user_enabled != new_hot.user_enabled {
|
||||
info!(
|
||||
"config reload: user_enabled updated ({} disabled overrides)",
|
||||
new_hot
|
||||
.user_enabled
|
||||
.values()
|
||||
.filter(|enabled| !**enabled)
|
||||
.count()
|
||||
);
|
||||
}
|
||||
if old_hot.user_max_tcp_conns != new_hot.user_max_tcp_conns {
|
||||
info!(
|
||||
"config reload: user_max_tcp_conns updated ({} entries)",
|
||||
@@ -1317,11 +1403,13 @@ fn reload_config(
|
||||
/// `detected_ip_v4` / `detected_ip_v6` are the IPs discovered during the
|
||||
/// startup probe — used when generating proxy links for newly added users,
|
||||
/// matching the same logic as the startup output.
|
||||
/// The watcher releases its notify and signal resources when `cancellation` fires.
|
||||
pub fn spawn_config_watcher(
|
||||
config_path: PathBuf,
|
||||
initial: Arc<ProxyConfig>,
|
||||
detected_ip_v4: Option<IpAddr>,
|
||||
detected_ip_v6: Option<IpAddr>,
|
||||
cancellation: tokio_util::sync::CancellationToken,
|
||||
) -> (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) {
|
||||
let initial_level = initial.general.log_level.clone();
|
||||
let (config_tx, config_rx) = watch::channel(initial);
|
||||
@@ -1429,10 +1517,14 @@ pub fn spawn_config_watcher(
|
||||
_ = sighup.recv() => {
|
||||
info!("SIGHUP received — reloading {:?}", config_path);
|
||||
}
|
||||
_ = cancellation.cancelled() => break,
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
if notify_rx.recv().await.is_none() {
|
||||
break;
|
||||
tokio::select! {
|
||||
msg = notify_rx.recv() => {
|
||||
if msg.is_none() { break; }
|
||||
}
|
||||
_ = cancellation.cancelled() => break,
|
||||
}
|
||||
|
||||
// Debounce: drain extra events that arrive within a short quiet window.
|
||||
@@ -1474,6 +1566,48 @@ pub fn spawn_config_watcher(
|
||||
(config_rx, log_rx)
|
||||
}
|
||||
|
||||
// ── Change classification ─────────────────────────────────────────────────────
|
||||
|
||||
/// Which top-level config sections changed and whether any require a restart.
|
||||
#[derive(Debug, Default, Clone, serde::Serialize)]
|
||||
pub struct ChangeClassification {
|
||||
pub changed: Vec<String>,
|
||||
pub restart_required: bool,
|
||||
}
|
||||
|
||||
/// Classify old->new using Telemt's OWN reload rule: overlay the hot fields and
|
||||
/// see if anything non-hot remains different. This guarantees `restart_required`
|
||||
/// matches actual runtime behavior and never drifts as new fields are added.
|
||||
pub fn classify_config_changes(old: &ProxyConfig, new: &ProxyConfig) -> ChangeClassification {
|
||||
let applied = overlay_hot_fields(old, new);
|
||||
let restart_required = !config_equal(&applied, new);
|
||||
ChangeClassification {
|
||||
changed: changed_sections(old, new),
|
||||
restart_required,
|
||||
}
|
||||
}
|
||||
|
||||
/// Top-level config sections whose canonical serialized form differs between
|
||||
/// old and new. Uses the same serialize+canonicalize path as `config_equal`.
|
||||
fn changed_sections(old: &ProxyConfig, new: &ProxyConfig) -> Vec<String> {
|
||||
let mut lhs = serde_json::to_value(old).unwrap_or(serde_json::Value::Null);
|
||||
let mut rhs = serde_json::to_value(new).unwrap_or(serde_json::Value::Null);
|
||||
canonicalize_json(&mut lhs);
|
||||
canonicalize_json(&mut rhs);
|
||||
|
||||
let mut out = Vec::new();
|
||||
if let (Some(lo), Some(ro)) = (lhs.as_object(), rhs.as_object()) {
|
||||
let mut keys: std::collections::BTreeSet<&String> = lo.keys().collect();
|
||||
keys.extend(ro.keys());
|
||||
for key in keys {
|
||||
if lo.get(key) != ro.get(key) {
|
||||
out.push(key.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1596,6 +1730,51 @@ mod tests {
|
||||
assert!(!config_equal(&applied, &new));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listener_synlimit_extended_fields_are_hot() {
|
||||
let mut old = sample_config();
|
||||
old.server.listeners.push(ListenerConfig {
|
||||
ip: "0.0.0.0".parse().unwrap(),
|
||||
port: Some(443),
|
||||
client_mss: None,
|
||||
synlimit: SynLimitMode::Iptables,
|
||||
synlimit_seconds: 60,
|
||||
synlimit_hitcount: 48,
|
||||
synlimit_burst: 1,
|
||||
synlimit_ios_seconds: 1,
|
||||
synlimit_ios_hitcount: 12,
|
||||
synlimit_ios_burst: 24,
|
||||
synlimit_hashlimit_expire_ms: 60_000,
|
||||
synlimit_hashlimit_size: 32_768,
|
||||
announce: None,
|
||||
announce_ip: None,
|
||||
proxy_protocol: None,
|
||||
reuse_allow: false,
|
||||
});
|
||||
let mut new = old.clone();
|
||||
new.server.port = 8443;
|
||||
new.server.listeners[0].synlimit_seconds = 120;
|
||||
new.server.listeners[0].synlimit_hitcount = 96;
|
||||
new.server.listeners[0].synlimit_burst = 2;
|
||||
new.server.listeners[0].synlimit_ios_seconds = 2;
|
||||
new.server.listeners[0].synlimit_ios_hitcount = 18;
|
||||
new.server.listeners[0].synlimit_ios_burst = 36;
|
||||
new.server.listeners[0].synlimit_hashlimit_expire_ms = 90_000;
|
||||
new.server.listeners[0].synlimit_hashlimit_size = 65_536;
|
||||
|
||||
let applied = overlay_hot_fields(&old, &new);
|
||||
let listener = &applied.server.listeners[0];
|
||||
assert_eq!(applied.server.port, old.server.port);
|
||||
assert_eq!(listener.synlimit_seconds, 120);
|
||||
assert_eq!(listener.synlimit_hitcount, 96);
|
||||
assert_eq!(listener.synlimit_burst, 2);
|
||||
assert_eq!(listener.synlimit_ios_seconds, 2);
|
||||
assert_eq!(listener.synlimit_ios_hitcount, 18);
|
||||
assert_eq!(listener.synlimit_ios_burst, 36);
|
||||
assert_eq!(listener.synlimit_hashlimit_expire_ms, 90_000);
|
||||
assert_eq!(listener.synlimit_hashlimit_size, 65_536);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_applies_hot_change_on_first_observed_snapshot() {
|
||||
let initial_tag = "11111111111111111111111111111111";
|
||||
@@ -1646,6 +1825,41 @@ mod tests {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_sni_change_requires_restart() {
|
||||
// censorship.* is not in overlay_hot_fields -> restart.
|
||||
let old = ProxyConfig::default();
|
||||
let mut new = ProxyConfig::default();
|
||||
new.censorship.tls_domain = "front.example".to_string();
|
||||
|
||||
let class = classify_config_changes(&old, &new);
|
||||
assert!(class.restart_required);
|
||||
assert!(class.changed.iter().any(|c| c == "censorship"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_dns_overrides_change_is_hot() {
|
||||
// network.dns_overrides IS in overlay_hot_fields -> no restart.
|
||||
let old = ProxyConfig::default();
|
||||
let mut new = ProxyConfig::default();
|
||||
new.network.dns_overrides.push("1.1.1.1".to_string());
|
||||
|
||||
let class = classify_config_changes(&old, &new);
|
||||
assert!(!class.restart_required);
|
||||
assert!(class.changed.iter().any(|c| c == "network"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_timeouts_change_requires_restart() {
|
||||
// timeouts.* is NOT in overlay_hot_fields -> restart.
|
||||
let old = ProxyConfig::default();
|
||||
let mut new = ProxyConfig::default();
|
||||
new.timeouts.client_handshake = old.timeouts.client_handshake + 1;
|
||||
|
||||
let class = classify_config_changes(&old, &new);
|
||||
assert!(class.restart_required);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reload_recovers_after_parse_error_on_next_attempt() {
|
||||
let initial_tag = "cccccccccccccccccccccccccccccccc";
|
||||
|
||||
3227
src/config/load.rs
3227
src/config/load.rs
File diff suppressed because it is too large
Load Diff
60
src/config/load/includes.rs
Normal file
60
src/config/load/includes.rs
Normal file
@@ -0,0 +1,60 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::hash::{DefaultHasher, Hash, Hasher};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::error::{ProxyError, Result};
|
||||
|
||||
pub(super) fn normalize_config_path(path: &Path) -> PathBuf {
|
||||
path.canonicalize().unwrap_or_else(|_| {
|
||||
if path.is_absolute() {
|
||||
path.to_path_buf()
|
||||
} else {
|
||||
std::env::current_dir()
|
||||
.map(|cwd| cwd.join(path))
|
||||
.unwrap_or_else(|_| path.to_path_buf())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn hash_rendered_snapshot(rendered: &str) -> u64 {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
rendered.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
pub(super) fn preprocess_includes(
|
||||
content: &str,
|
||||
base_dir: &Path,
|
||||
depth: u8,
|
||||
source_files: &mut BTreeSet<PathBuf>,
|
||||
) -> Result<String> {
|
||||
if depth > 10 {
|
||||
return Err(ProxyError::Config("Include depth > 10".into()));
|
||||
}
|
||||
let mut output = String::with_capacity(content.len());
|
||||
for line in content.lines() {
|
||||
let trimmed = line.trim();
|
||||
if let Some(rest) = trimmed.strip_prefix("include") {
|
||||
let rest = rest.trim();
|
||||
if let Some(rest) = rest.strip_prefix('=') {
|
||||
let path_str = rest.trim().trim_matches('"');
|
||||
let resolved = base_dir.join(path_str);
|
||||
source_files.insert(normalize_config_path(&resolved));
|
||||
let included = std::fs::read_to_string(&resolved)
|
||||
.map_err(|e| ProxyError::Config(e.to_string()))?;
|
||||
let included_dir = resolved.parent().unwrap_or(base_dir);
|
||||
output.push_str(&preprocess_includes(
|
||||
&included,
|
||||
included_dir,
|
||||
depth + 1,
|
||||
source_files,
|
||||
)?);
|
||||
output.push('\n');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
output.push_str(line);
|
||||
output.push('\n');
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
115
src/config/load/normalize.rs
Normal file
115
src/config/load/normalize.rs
Normal file
@@ -0,0 +1,115 @@
|
||||
use crate::error::{ProxyError, Result};
|
||||
use tracing::warn;
|
||||
|
||||
pub(super) fn is_valid_tls_domain_name(domain: &str) -> bool {
|
||||
!domain.is_empty()
|
||||
&& !domain
|
||||
.chars()
|
||||
.any(|ch| ch.is_whitespace() || matches!(ch, '/' | '\\'))
|
||||
}
|
||||
|
||||
pub(super) fn normalize_domain_to_ascii(domain: &str, field: &str) -> Result<String> {
|
||||
let domain = domain.trim();
|
||||
if !is_valid_tls_domain_name(domain) {
|
||||
return Err(ProxyError::Config(format!(
|
||||
"Invalid {field}: '{}'. Must be a valid domain name",
|
||||
domain
|
||||
)));
|
||||
}
|
||||
|
||||
let parsed = url::Url::parse(&format!("https://{domain}/")).map_err(|error| {
|
||||
ProxyError::Config(format!(
|
||||
"Invalid {field}: '{}'. IDNA conversion failed: {error}",
|
||||
domain
|
||||
))
|
||||
})?;
|
||||
let host = parsed.host_str().ok_or_else(|| {
|
||||
ProxyError::Config(format!("Invalid {field}: '{}'. Host is empty", domain))
|
||||
})?;
|
||||
Ok(host.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
pub(super) fn normalize_mask_host_to_ascii(host: &str, field: &str) -> Result<String> {
|
||||
let host = host.trim();
|
||||
if host.starts_with('[') && host.ends_with(']') {
|
||||
let inner = &host[1..host.len() - 1];
|
||||
let ip = inner.parse::<std::net::IpAddr>().map_err(|_| {
|
||||
ProxyError::Config(format!(
|
||||
"Invalid {field}: '{}'. IPv6 literal is invalid",
|
||||
host
|
||||
))
|
||||
})?;
|
||||
return match ip {
|
||||
std::net::IpAddr::V6(v6) => Ok(format!("[{v6}]")),
|
||||
std::net::IpAddr::V4(v4) => Ok(v4.to_string()),
|
||||
};
|
||||
}
|
||||
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
|
||||
return match ip {
|
||||
std::net::IpAddr::V4(v4) => Ok(v4.to_string()),
|
||||
std::net::IpAddr::V6(v6) => Ok(format!("[{v6}]")),
|
||||
};
|
||||
}
|
||||
|
||||
normalize_domain_to_ascii(host, field)
|
||||
}
|
||||
|
||||
pub(super) fn parse_exclusive_mask_target(target: &str) -> Option<(&str, u16)> {
|
||||
let target = target.trim();
|
||||
if target.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if target.starts_with('[') {
|
||||
let end = target.find(']')?;
|
||||
if target.get(end + 1..end + 2)? != ":" {
|
||||
return None;
|
||||
}
|
||||
let host = &target[..=end];
|
||||
let port = target[end + 2..].parse::<u16>().ok()?;
|
||||
return (port > 0).then_some((host, port));
|
||||
}
|
||||
|
||||
let (host, port) = target.rsplit_once(':')?;
|
||||
if host.is_empty() || host.contains(':') {
|
||||
return None;
|
||||
}
|
||||
let port = port.parse::<u16>().ok()?;
|
||||
(port > 0).then_some((host, port))
|
||||
}
|
||||
|
||||
pub(super) fn normalize_exclusive_mask_target(target: &str, field: &str) -> Result<String> {
|
||||
let (host, port) = parse_exclusive_mask_target(target).ok_or_else(|| {
|
||||
ProxyError::Config(format!(
|
||||
"Invalid {field}: '{}'. Expected host:port with port > 0",
|
||||
target
|
||||
))
|
||||
})?;
|
||||
let host = normalize_mask_host_to_ascii(host, field)?;
|
||||
Ok(format!("{host}:{port}"))
|
||||
}
|
||||
|
||||
pub(super) fn push_unique_nonempty(target: &mut Vec<String>, value: String) {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.is_empty() {
|
||||
return;
|
||||
}
|
||||
if !target.iter().any(|existing| existing == trimmed) {
|
||||
target.push(trimmed.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_valid_ad_tag(tag: &str) -> bool {
|
||||
tag.len() == 32 && tag.chars().all(|ch| ch.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
pub(super) fn sanitize_ad_tag(ad_tag: &mut Option<String>) {
|
||||
let Some(tag) = ad_tag.as_ref() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if !is_valid_ad_tag(tag) {
|
||||
warn!("Invalid general.ad_tag value, expected exactly 32 hex chars; ad_tag is disabled");
|
||||
*ad_tag = None;
|
||||
}
|
||||
}
|
||||
112
src/config/load/runtime_auth.rs
Normal file
112
src/config/load/runtime_auth.rs
Normal file
@@ -0,0 +1,112 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::Hasher;
|
||||
|
||||
use crate::error::{ProxyError, Result};
|
||||
|
||||
const ACCESS_SECRET_BYTES: usize = 16;
|
||||
|
||||
/// Precomputed, immutable user authentication data used by handshake hot paths.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct UserAuthSnapshot {
|
||||
entries: Vec<UserAuthEntry>,
|
||||
by_name: HashMap<String, u32>,
|
||||
sni_index: HashMap<u64, Vec<u32>>,
|
||||
sni_initial_index: HashMap<u8, Vec<u32>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct UserAuthEntry {
|
||||
pub(crate) user: String,
|
||||
pub(crate) secret: [u8; ACCESS_SECRET_BYTES],
|
||||
}
|
||||
|
||||
impl UserAuthSnapshot {
|
||||
pub(super) fn from_users(users: &HashMap<String, String>) -> Result<Self> {
|
||||
let mut entries = Vec::with_capacity(users.len());
|
||||
let mut by_name = HashMap::with_capacity(users.len());
|
||||
let mut sni_index = HashMap::with_capacity(users.len());
|
||||
let mut sni_initial_index = HashMap::with_capacity(users.len());
|
||||
|
||||
for (user, secret_hex) in users {
|
||||
let decoded = hex::decode(secret_hex).map_err(|_| ProxyError::InvalidSecret {
|
||||
user: user.clone(),
|
||||
reason: "Must be 32 hex characters".to_string(),
|
||||
})?;
|
||||
if decoded.len() != ACCESS_SECRET_BYTES {
|
||||
return Err(ProxyError::InvalidSecret {
|
||||
user: user.clone(),
|
||||
reason: "Must be 32 hex characters".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let user_id = u32::try_from(entries.len()).map_err(|_| {
|
||||
ProxyError::Config("Too many users for runtime auth snapshot".to_string())
|
||||
})?;
|
||||
|
||||
let mut secret = [0u8; ACCESS_SECRET_BYTES];
|
||||
secret.copy_from_slice(&decoded);
|
||||
entries.push(UserAuthEntry {
|
||||
user: user.clone(),
|
||||
secret,
|
||||
});
|
||||
by_name.insert(user.clone(), user_id);
|
||||
sni_index
|
||||
.entry(Self::sni_lookup_hash(user))
|
||||
.or_insert_with(Vec::new)
|
||||
.push(user_id);
|
||||
if let Some(initial) = user
|
||||
.as_bytes()
|
||||
.first()
|
||||
.map(|byte| byte.to_ascii_lowercase())
|
||||
{
|
||||
sni_initial_index
|
||||
.entry(initial)
|
||||
.or_insert_with(Vec::new)
|
||||
.push(user_id);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
entries,
|
||||
by_name,
|
||||
sni_index,
|
||||
sni_initial_index,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn entries(&self) -> &[UserAuthEntry] {
|
||||
&self.entries
|
||||
}
|
||||
|
||||
pub(crate) fn user_id_by_name(&self, user: &str) -> Option<u32> {
|
||||
self.by_name.get(user).copied()
|
||||
}
|
||||
|
||||
pub(crate) fn entry_by_id(&self, user_id: u32) -> Option<&UserAuthEntry> {
|
||||
let idx = usize::try_from(user_id).ok()?;
|
||||
self.entries.get(idx)
|
||||
}
|
||||
|
||||
pub(crate) fn sni_candidates(&self, sni: &str) -> Option<&[u32]> {
|
||||
self.sni_index
|
||||
.get(&Self::sni_lookup_hash(sni))
|
||||
.map(Vec::as_slice)
|
||||
}
|
||||
|
||||
pub(crate) fn sni_initial_candidates(&self, sni: &str) -> Option<&[u32]> {
|
||||
let initial = sni
|
||||
.as_bytes()
|
||||
.first()
|
||||
.map(|byte| byte.to_ascii_lowercase())?;
|
||||
self.sni_initial_index.get(&initial).map(Vec::as_slice)
|
||||
}
|
||||
|
||||
fn sni_lookup_hash(value: &str) -> u64 {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
for byte in value.bytes() {
|
||||
hasher.write_u8(byte.to_ascii_lowercase());
|
||||
}
|
||||
hasher.finish()
|
||||
}
|
||||
}
|
||||
695
src/config/load/strict_keys.rs
Normal file
695
src/config/load/strict_keys.rs
Normal file
@@ -0,0 +1,695 @@
|
||||
use tracing::warn;
|
||||
|
||||
use crate::error::{ProxyError, Result};
|
||||
|
||||
const TOP_LEVEL_CONFIG_KEYS: &[&str] = &[
|
||||
"general",
|
||||
"logging",
|
||||
"network",
|
||||
"server",
|
||||
"timeouts",
|
||||
"censorship",
|
||||
"access",
|
||||
"upstreams",
|
||||
"show_link",
|
||||
"dc_overrides",
|
||||
"default_dc",
|
||||
"beobachten",
|
||||
"beobachten_minutes",
|
||||
"beobachten_flush_secs",
|
||||
"beobachten_file",
|
||||
"include",
|
||||
];
|
||||
|
||||
const GENERAL_CONFIG_KEYS: &[&str] = &[
|
||||
"data_path",
|
||||
"quota_state_path",
|
||||
"config_strict",
|
||||
"modes",
|
||||
"prefer_ipv6",
|
||||
"fast_mode",
|
||||
"use_middle_proxy",
|
||||
"proxy_secret_path",
|
||||
"proxy_secret_url",
|
||||
"proxy_config_v4_cache_path",
|
||||
"proxy_config_v4_url",
|
||||
"proxy_config_v6_cache_path",
|
||||
"proxy_config_v6_url",
|
||||
"ad_tag",
|
||||
"middle_proxy_nat_ip",
|
||||
"middle_proxy_nat_probe",
|
||||
"middle_proxy_nat_stun",
|
||||
"middle_proxy_nat_stun_servers",
|
||||
"stun_nat_probe_concurrency",
|
||||
"middle_proxy_pool_size",
|
||||
"middle_proxy_warm_standby",
|
||||
"me_init_retry_attempts",
|
||||
"me2dc_fallback",
|
||||
"me2dc_fast",
|
||||
"me_keepalive_enabled",
|
||||
"me_keepalive_interval_secs",
|
||||
"me_keepalive_jitter_secs",
|
||||
"me_keepalive_payload_random",
|
||||
"rpc_proxy_req_every",
|
||||
"me_writer_cmd_channel_capacity",
|
||||
"me_writer_byte_budget_bytes",
|
||||
"me_route_channel_capacity",
|
||||
"me_c2me_channel_capacity",
|
||||
"me_c2me_send_timeout_ms",
|
||||
"me_reader_route_data_wait_ms",
|
||||
"me_d2c_flush_batch_max_frames",
|
||||
"me_d2c_flush_batch_max_bytes",
|
||||
"me_d2c_flush_batch_max_delay_us",
|
||||
"me_d2c_ack_flush_immediate",
|
||||
"me_quota_soft_overshoot_bytes",
|
||||
"me_d2c_frame_buf_shrink_threshold_bytes",
|
||||
"direct_relay_copy_buf_c2s_bytes",
|
||||
"direct_relay_copy_buf_s2c_bytes",
|
||||
"direct_relay_buffer_budget_max_bytes",
|
||||
"crypto_pending_buffer",
|
||||
"max_client_frame",
|
||||
"desync_all_full",
|
||||
"beobachten",
|
||||
"beobachten_minutes",
|
||||
"beobachten_flush_secs",
|
||||
"beobachten_file",
|
||||
"hardswap",
|
||||
"me_warmup_stagger_enabled",
|
||||
"me_warmup_step_delay_ms",
|
||||
"me_warmup_step_jitter_ms",
|
||||
"me_reconnect_max_concurrent_per_dc",
|
||||
"me_reconnect_backoff_base_ms",
|
||||
"me_reconnect_backoff_cap_ms",
|
||||
"me_reconnect_fast_retry_count",
|
||||
"me_single_endpoint_shadow_writers",
|
||||
"me_single_endpoint_outage_mode_enabled",
|
||||
"me_single_endpoint_outage_disable_quarantine",
|
||||
"me_single_endpoint_outage_backoff_min_ms",
|
||||
"me_single_endpoint_outage_backoff_max_ms",
|
||||
"me_single_endpoint_shadow_rotate_every_secs",
|
||||
"me_floor_mode",
|
||||
"me_adaptive_floor_idle_secs",
|
||||
"me_adaptive_floor_min_writers_single_endpoint",
|
||||
"me_adaptive_floor_min_writers_multi_endpoint",
|
||||
"me_adaptive_floor_recover_grace_secs",
|
||||
"me_adaptive_floor_writers_per_core_total",
|
||||
"me_adaptive_floor_cpu_cores_override",
|
||||
"me_adaptive_floor_max_extra_writers_single_per_core",
|
||||
"me_adaptive_floor_max_extra_writers_multi_per_core",
|
||||
"me_adaptive_floor_max_active_writers_per_core",
|
||||
"me_adaptive_floor_max_warm_writers_per_core",
|
||||
"me_adaptive_floor_max_active_writers_global",
|
||||
"me_adaptive_floor_max_warm_writers_global",
|
||||
"upstream_connect_retry_attempts",
|
||||
"upstream_connect_retry_backoff_ms",
|
||||
"upstream_connect_budget_ms",
|
||||
"tg_connect",
|
||||
"upstream_unhealthy_fail_threshold",
|
||||
"upstream_connect_failfast_hard_errors",
|
||||
"stun_iface_mismatch_ignore",
|
||||
"unknown_dc_log_path",
|
||||
"unknown_dc_file_log_enabled",
|
||||
"log_level",
|
||||
"disable_colors",
|
||||
"telemetry",
|
||||
"me_socks_kdf_policy",
|
||||
"me_route_backpressure_enabled",
|
||||
"me_route_fairshare_enabled",
|
||||
"me_route_backpressure_base_timeout_ms",
|
||||
"me_route_backpressure_high_timeout_ms",
|
||||
"me_route_backpressure_high_watermark_pct",
|
||||
"me_health_interval_ms_unhealthy",
|
||||
"me_health_interval_ms_healthy",
|
||||
"me_admission_poll_ms",
|
||||
"me_warn_rate_limit_ms",
|
||||
"me_route_no_writer_mode",
|
||||
"me_route_no_writer_wait_ms",
|
||||
"me_route_hybrid_max_wait_ms",
|
||||
"me_route_blocking_send_timeout_ms",
|
||||
"me_route_inline_recovery_attempts",
|
||||
"me_route_inline_recovery_wait_ms",
|
||||
"links",
|
||||
"fast_mode_min_tls_record",
|
||||
"update_every",
|
||||
"me_reinit_every_secs",
|
||||
"me_hardswap_warmup_delay_min_ms",
|
||||
"me_hardswap_warmup_delay_max_ms",
|
||||
"me_hardswap_warmup_extra_passes",
|
||||
"me_hardswap_warmup_pass_backoff_base_ms",
|
||||
"me_config_stable_snapshots",
|
||||
"me_config_apply_cooldown_secs",
|
||||
"me_snapshot_require_http_2xx",
|
||||
"me_snapshot_reject_empty_map",
|
||||
"me_snapshot_min_proxy_for_lines",
|
||||
"proxy_secret_stable_snapshots",
|
||||
"proxy_secret_rotate_runtime",
|
||||
"me_secret_atomic_snapshot",
|
||||
"proxy_secret_len_max",
|
||||
"me_pool_drain_ttl_secs",
|
||||
"me_instadrain",
|
||||
"me_pool_drain_threshold",
|
||||
"me_pool_drain_soft_evict_enabled",
|
||||
"me_pool_drain_soft_evict_grace_secs",
|
||||
"me_pool_drain_soft_evict_per_writer",
|
||||
"me_pool_drain_soft_evict_budget_per_core",
|
||||
"me_pool_drain_soft_evict_cooldown_ms",
|
||||
"me_bind_stale_mode",
|
||||
"me_bind_stale_ttl_secs",
|
||||
"me_pool_min_fresh_ratio",
|
||||
"me_reinit_drain_timeout_secs",
|
||||
"proxy_secret_auto_reload_secs",
|
||||
"proxy_config_auto_reload_secs",
|
||||
"me_reinit_singleflight",
|
||||
"me_reinit_trigger_channel",
|
||||
"me_reinit_coalesce_window_ms",
|
||||
"me_deterministic_writer_sort",
|
||||
"me_writer_pick_mode",
|
||||
"me_writer_pick_sample_size",
|
||||
"ntp_check",
|
||||
"ntp_servers",
|
||||
"auto_degradation_enabled",
|
||||
"degradation_min_unavailable_dc_groups",
|
||||
"rst_on_close",
|
||||
];
|
||||
|
||||
const NETWORK_CONFIG_KEYS: &[&str] = &[
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
"prefer",
|
||||
"multipath",
|
||||
"stun_use",
|
||||
"stun_servers",
|
||||
"stun_tcp_fallback",
|
||||
"http_ip_detect_urls",
|
||||
"cache_public_ip_path",
|
||||
"dns_overrides",
|
||||
];
|
||||
|
||||
const SERVER_CONFIG_KEYS: &[&str] = &[
|
||||
"port",
|
||||
"listen_addr_ipv4",
|
||||
"listen_addr_ipv6",
|
||||
"listen_unix_sock",
|
||||
"listen_unix_sock_perm",
|
||||
"listen_tcp",
|
||||
"client_mss",
|
||||
"client_mss_bulk",
|
||||
"proxy_protocol",
|
||||
"proxy_protocol_header_timeout_ms",
|
||||
"proxy_protocol_trusted_cidrs",
|
||||
"metrics_port",
|
||||
"metrics_listen",
|
||||
"metrics_whitelist",
|
||||
"api",
|
||||
"admin_api",
|
||||
"listeners",
|
||||
"listen_backlog",
|
||||
"max_connections",
|
||||
"accept_permit_timeout_ms",
|
||||
"conntrack_control",
|
||||
];
|
||||
|
||||
const API_CONFIG_KEYS: &[&str] = &[
|
||||
"enabled",
|
||||
"listen",
|
||||
"whitelist",
|
||||
"gray_action",
|
||||
"auth_header",
|
||||
"request_body_limit_bytes",
|
||||
"minimal_runtime_enabled",
|
||||
"minimal_runtime_cache_ttl_ms",
|
||||
"runtime_edge_enabled",
|
||||
"runtime_edge_cache_ttl_ms",
|
||||
"runtime_edge_top_n",
|
||||
"runtime_edge_events_capacity",
|
||||
"read_only",
|
||||
];
|
||||
|
||||
const CONNTRACK_CONTROL_CONFIG_KEYS: &[&str] = &[
|
||||
"inline_conntrack_control",
|
||||
"mode",
|
||||
"backend",
|
||||
"profile",
|
||||
"hybrid_listener_ips",
|
||||
"pressure_high_watermark_pct",
|
||||
"pressure_low_watermark_pct",
|
||||
"delete_budget_per_sec",
|
||||
];
|
||||
|
||||
const LISTENER_CONFIG_KEYS: &[&str] = &[
|
||||
"ip",
|
||||
"port",
|
||||
"client_mss",
|
||||
"synlimit",
|
||||
"synlimit_seconds",
|
||||
"synlimit_hitcount",
|
||||
"synlimit_burst",
|
||||
"synlimit_ios_seconds",
|
||||
"synlimit_ios_hitcount",
|
||||
"synlimit_ios_burst",
|
||||
"synlimit_hashlimit_expire_ms",
|
||||
"synlimit_hashlimit_size",
|
||||
"announce",
|
||||
"announce_ip",
|
||||
"proxy_protocol",
|
||||
"reuse_allow",
|
||||
];
|
||||
|
||||
const TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||
"client_first_byte_idle_secs",
|
||||
"client_handshake",
|
||||
"relay_idle_policy_v2_enabled",
|
||||
"relay_client_idle_soft_secs",
|
||||
"relay_client_idle_hard_secs",
|
||||
"relay_idle_grace_after_downstream_activity_secs",
|
||||
"client_keepalive",
|
||||
"client_ack",
|
||||
"me_one_retry",
|
||||
"me_one_timeout_ms",
|
||||
];
|
||||
|
||||
const CENSORSHIP_CONFIG_KEYS: &[&str] = &[
|
||||
"tls_domain",
|
||||
"tls_domains",
|
||||
"unknown_sni_action",
|
||||
"tls_fetch_scope",
|
||||
"tls_fetch",
|
||||
"mask",
|
||||
"mask_dynamic",
|
||||
"mask_host",
|
||||
"mask_port",
|
||||
"exclusive_mask",
|
||||
"mask_unix_sock",
|
||||
"fake_cert_len",
|
||||
"tls_emulation",
|
||||
"tls_front_dir",
|
||||
"server_hello_delay_min_ms",
|
||||
"server_hello_delay_max_ms",
|
||||
"tls_new_session_tickets",
|
||||
"serverhello_compact",
|
||||
"tls_full_cert_ttl_secs",
|
||||
"alpn_enforce",
|
||||
"mask_proxy_protocol",
|
||||
"mask_shape_hardening",
|
||||
"mask_shape_hardening_aggressive_mode",
|
||||
"mask_shape_bucket_floor_bytes",
|
||||
"mask_shape_bucket_cap_bytes",
|
||||
"mask_shape_above_cap_blur",
|
||||
"mask_shape_above_cap_blur_max_bytes",
|
||||
"mask_relay_max_bytes",
|
||||
"mask_relay_timeout_ms",
|
||||
"mask_relay_idle_timeout_ms",
|
||||
"mask_classifier_prefetch_timeout_ms",
|
||||
"mask_timing_normalization_enabled",
|
||||
"mask_timing_normalization_floor_ms",
|
||||
"mask_timing_normalization_ceiling_ms",
|
||||
];
|
||||
|
||||
const TLS_FETCH_CONFIG_KEYS: &[&str] = &[
|
||||
"profiles",
|
||||
"strict_route",
|
||||
"attempt_timeout_ms",
|
||||
"total_budget_ms",
|
||||
"grease_enabled",
|
||||
"deterministic",
|
||||
"profile_cache_ttl_secs",
|
||||
];
|
||||
|
||||
const ACCESS_CONFIG_KEYS: &[&str] = &[
|
||||
"users",
|
||||
"user_enabled",
|
||||
"user_ad_tags",
|
||||
"user_max_tcp_conns",
|
||||
"user_max_tcp_conns_global_each",
|
||||
"user_expirations",
|
||||
"user_data_quota",
|
||||
"user_rate_limits",
|
||||
"cidr_rate_limits",
|
||||
"user_max_unique_ips",
|
||||
"user_max_unique_ips_global_each",
|
||||
"user_max_unique_ips_mode",
|
||||
"user_max_unique_ips_window_secs",
|
||||
"replay_check_len",
|
||||
"replay_window_secs",
|
||||
"ignore_time_skew",
|
||||
];
|
||||
|
||||
const RATE_LIMIT_BPS_CONFIG_KEYS: &[&str] = &["up_bps", "down_bps"];
|
||||
|
||||
const UPSTREAM_CONFIG_KEYS: &[&str] = &[
|
||||
"type",
|
||||
"interface",
|
||||
"bind_addresses",
|
||||
"bindtodevice",
|
||||
"force_bind",
|
||||
"address",
|
||||
"user_id",
|
||||
"username",
|
||||
"password",
|
||||
"url",
|
||||
"weight",
|
||||
"enabled",
|
||||
"scopes",
|
||||
"ipv4",
|
||||
"ipv6",
|
||||
];
|
||||
|
||||
const PROXY_MODES_CONFIG_KEYS: &[&str] = &["classic", "secure", "tls"];
|
||||
const TELEMETRY_CONFIG_KEYS: &[&str] = &["core_enabled", "user_enabled", "me_level"];
|
||||
const LINKS_CONFIG_KEYS: &[&str] = &["show", "public_host", "public_port"];
|
||||
const LOGGING_CONFIG_KEYS: &[&str] = &[
|
||||
"destination",
|
||||
"path",
|
||||
"rotation",
|
||||
"max_size_bytes",
|
||||
"max_files",
|
||||
"max_age_secs",
|
||||
];
|
||||
|
||||
#[derive(Debug)]
|
||||
struct UnknownConfigKey {
|
||||
path: String,
|
||||
suggestion: Option<String>,
|
||||
}
|
||||
|
||||
fn table_at<'a>(value: &'a toml::Value, path: &[&str]) -> Option<&'a toml::Table> {
|
||||
let mut current = value;
|
||||
for segment in path {
|
||||
current = current.get(*segment)?;
|
||||
}
|
||||
current.as_table()
|
||||
}
|
||||
|
||||
fn is_strict_config(parsed_toml: &toml::Value) -> bool {
|
||||
table_at(parsed_toml, &["general"])
|
||||
.and_then(|table| table.get("config_strict"))
|
||||
.and_then(toml::Value::as_bool)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn known_config_keys_for_suggestion() -> Vec<&'static str> {
|
||||
let mut keys = Vec::new();
|
||||
for group in [
|
||||
TOP_LEVEL_CONFIG_KEYS,
|
||||
GENERAL_CONFIG_KEYS,
|
||||
NETWORK_CONFIG_KEYS,
|
||||
SERVER_CONFIG_KEYS,
|
||||
API_CONFIG_KEYS,
|
||||
CONNTRACK_CONTROL_CONFIG_KEYS,
|
||||
LISTENER_CONFIG_KEYS,
|
||||
TIMEOUTS_CONFIG_KEYS,
|
||||
CENSORSHIP_CONFIG_KEYS,
|
||||
TLS_FETCH_CONFIG_KEYS,
|
||||
ACCESS_CONFIG_KEYS,
|
||||
RATE_LIMIT_BPS_CONFIG_KEYS,
|
||||
UPSTREAM_CONFIG_KEYS,
|
||||
PROXY_MODES_CONFIG_KEYS,
|
||||
TELEMETRY_CONFIG_KEYS,
|
||||
LINKS_CONFIG_KEYS,
|
||||
LOGGING_CONFIG_KEYS,
|
||||
] {
|
||||
keys.extend_from_slice(group);
|
||||
}
|
||||
keys
|
||||
}
|
||||
|
||||
fn levenshtein_distance(a: &str, b: &str) -> usize {
|
||||
let b_chars: Vec<char> = b.chars().collect();
|
||||
let mut prev: Vec<usize> = (0..=b_chars.len()).collect();
|
||||
let mut curr = vec![0usize; b_chars.len() + 1];
|
||||
|
||||
for (i, ca) in a.chars().enumerate() {
|
||||
curr[0] = i + 1;
|
||||
for (j, cb) in b_chars.iter().enumerate() {
|
||||
let replace = if ca == *cb { prev[j] } else { prev[j] + 1 };
|
||||
curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(replace);
|
||||
}
|
||||
std::mem::swap(&mut prev, &mut curr);
|
||||
}
|
||||
|
||||
prev[b_chars.len()]
|
||||
}
|
||||
|
||||
fn unknown_key_suggestion(key: &str, known_keys: &[&'static str]) -> Option<String> {
|
||||
let normalized = key.to_ascii_lowercase();
|
||||
let mut best: Option<(&str, usize)> = None;
|
||||
for known in known_keys {
|
||||
let distance = levenshtein_distance(&normalized, known);
|
||||
let is_better = match best {
|
||||
Some((_, best_distance)) => distance < best_distance,
|
||||
None => true,
|
||||
};
|
||||
if distance <= 4 && is_better {
|
||||
best = Some((known, distance));
|
||||
}
|
||||
}
|
||||
best.map(|(known, _)| known.to_string())
|
||||
}
|
||||
|
||||
fn push_unknown_keys(
|
||||
unknown: &mut Vec<UnknownConfigKey>,
|
||||
known_for_suggestion: &[&'static str],
|
||||
path: &str,
|
||||
table: &toml::Table,
|
||||
allowed: &[&str],
|
||||
) {
|
||||
for key in table.keys() {
|
||||
if !allowed.contains(&key.as_str()) {
|
||||
let full_path = if path.is_empty() {
|
||||
key.clone()
|
||||
} else {
|
||||
format!("{path}.{key}")
|
||||
};
|
||||
unknown.push(UnknownConfigKey {
|
||||
path: full_path,
|
||||
suggestion: unknown_key_suggestion(key, known_for_suggestion),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_known_table(
|
||||
parsed_toml: &toml::Value,
|
||||
unknown: &mut Vec<UnknownConfigKey>,
|
||||
known_for_suggestion: &[&'static str],
|
||||
path: &[&str],
|
||||
allowed: &[&str],
|
||||
) {
|
||||
if let Some(table) = table_at(parsed_toml, path) {
|
||||
push_unknown_keys(
|
||||
unknown,
|
||||
known_for_suggestion,
|
||||
&path.join("."),
|
||||
table,
|
||||
allowed,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_nested_table_value(
|
||||
unknown: &mut Vec<UnknownConfigKey>,
|
||||
known_for_suggestion: &[&'static str],
|
||||
path: String,
|
||||
value: &toml::Value,
|
||||
allowed: &[&str],
|
||||
) {
|
||||
if let Some(table) = value.as_table() {
|
||||
push_unknown_keys(unknown, known_for_suggestion, &path, table, allowed);
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<UnknownConfigKey> {
|
||||
let known_for_suggestion = known_config_keys_for_suggestion();
|
||||
let mut unknown = Vec::new();
|
||||
|
||||
if let Some(root) = parsed_toml.as_table() {
|
||||
push_unknown_keys(
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
"",
|
||||
root,
|
||||
TOP_LEVEL_CONFIG_KEYS,
|
||||
);
|
||||
}
|
||||
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["general"],
|
||||
GENERAL_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["general", "modes"],
|
||||
PROXY_MODES_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["general", "telemetry"],
|
||||
TELEMETRY_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["general", "links"],
|
||||
LINKS_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["logging"],
|
||||
LOGGING_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["network"],
|
||||
NETWORK_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["server"],
|
||||
SERVER_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["server", "api"],
|
||||
API_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["server", "admin_api"],
|
||||
API_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["server", "conntrack_control"],
|
||||
CONNTRACK_CONTROL_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["timeouts"],
|
||||
TIMEOUTS_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["censorship"],
|
||||
CENSORSHIP_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["censorship", "tls_fetch"],
|
||||
TLS_FETCH_CONFIG_KEYS,
|
||||
);
|
||||
check_known_table(
|
||||
parsed_toml,
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
&["access"],
|
||||
ACCESS_CONFIG_KEYS,
|
||||
);
|
||||
|
||||
if let Some(listeners) = table_at(parsed_toml, &["server"])
|
||||
.and_then(|table| table.get("listeners"))
|
||||
.and_then(toml::Value::as_array)
|
||||
{
|
||||
for (idx, listener) in listeners.iter().enumerate() {
|
||||
check_nested_table_value(
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
format!("server.listeners[{idx}]"),
|
||||
listener,
|
||||
LISTENER_CONFIG_KEYS,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(upstreams) = parsed_toml.get("upstreams").and_then(toml::Value::as_array) {
|
||||
for (idx, upstream) in upstreams.iter().enumerate() {
|
||||
check_nested_table_value(
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
format!("upstreams[{idx}]"),
|
||||
upstream,
|
||||
UPSTREAM_CONFIG_KEYS,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for access_map in ["user_rate_limits", "cidr_rate_limits"] {
|
||||
if let Some(table) = table_at(parsed_toml, &["access"])
|
||||
.and_then(|access| access.get(access_map))
|
||||
.and_then(toml::Value::as_table)
|
||||
{
|
||||
for (entry_name, value) in table {
|
||||
check_nested_table_value(
|
||||
&mut unknown,
|
||||
&known_for_suggestion,
|
||||
format!("access.{access_map}.{entry_name}"),
|
||||
value,
|
||||
RATE_LIMIT_BPS_CONFIG_KEYS,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unknown
|
||||
}
|
||||
|
||||
pub(super) fn handle_unknown_config_keys(parsed_toml: &toml::Value) -> Result<()> {
|
||||
let unknown = collect_unknown_config_keys(parsed_toml);
|
||||
if unknown.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for item in &unknown {
|
||||
if let Some(suggestion) = item.suggestion.as_deref() {
|
||||
warn!(
|
||||
key = %item.path,
|
||||
suggestion = %suggestion,
|
||||
"Unknown config key ignored; did you mean the suggested key?"
|
||||
);
|
||||
} else {
|
||||
warn!(key = %item.path, "Unknown config key ignored");
|
||||
}
|
||||
}
|
||||
|
||||
if is_strict_config(parsed_toml) {
|
||||
let mut paths = Vec::with_capacity(unknown.len());
|
||||
for item in unknown {
|
||||
if let Some(suggestion) = item.suggestion {
|
||||
paths.push(format!("{} (did you mean `{}`?)", item.path, suggestion));
|
||||
} else {
|
||||
paths.push(item.path);
|
||||
}
|
||||
}
|
||||
return Err(ProxyError::Config(format!(
|
||||
"unknown config keys are not allowed when general.config_strict=true: {}",
|
||||
paths.join(", ")
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
111
src/config/load/validation.rs
Normal file
111
src/config/load/validation.rs
Normal file
@@ -0,0 +1,111 @@
|
||||
use shadowsocks::config::ServerConfig as ShadowsocksServerConfig;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::error::{ProxyError, Result};
|
||||
|
||||
use super::super::types::{LoggingConfig, LoggingDestination, NetworkConfig, UpstreamType};
|
||||
use super::ProxyConfig;
|
||||
|
||||
pub(super) fn validate_network_cfg(net: &mut NetworkConfig) -> Result<()> {
|
||||
if !net.ipv4 && matches!(net.ipv6, Some(false)) {
|
||||
return Err(ProxyError::Config(
|
||||
"Both ipv4 and ipv6 are disabled in [network]".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if net.prefer != 4 && net.prefer != 6 {
|
||||
return Err(ProxyError::Config(
|
||||
"network.prefer must be 4 or 6".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if !net.ipv4 && net.prefer == 4 {
|
||||
warn!("prefer=4 but ipv4=false; forcing prefer=6");
|
||||
net.prefer = 6;
|
||||
}
|
||||
|
||||
if matches!(net.ipv6, Some(false)) && net.prefer == 6 {
|
||||
warn!("prefer=6 but ipv6=false; forcing prefer=4");
|
||||
net.prefer = 4;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_logging_config(logging: &LoggingConfig) -> Result<()> {
|
||||
if let Some(path) = logging.path.as_ref()
|
||||
&& path.trim().is_empty()
|
||||
{
|
||||
return Err(ProxyError::Config(
|
||||
"logging.path cannot be empty when provided".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if matches!(logging.destination, LoggingDestination::File) && logging.path.is_none() {
|
||||
return Err(ProxyError::Config(
|
||||
"logging.path must be set when logging.destination=\"file\"".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_upstreams(config: &ProxyConfig) -> Result<()> {
|
||||
let has_enabled_shadowsocks = config.upstreams.iter().any(|upstream| {
|
||||
upstream.enabled && matches!(upstream.upstream_type, UpstreamType::Shadowsocks { .. })
|
||||
});
|
||||
|
||||
if has_enabled_shadowsocks && config.general.use_middle_proxy {
|
||||
return Err(ProxyError::Config(
|
||||
"shadowsocks upstreams require general.use_middle_proxy = false".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
for upstream in &config.upstreams {
|
||||
if matches!(upstream.ipv4, Some(false)) && matches!(upstream.ipv6, Some(false)) {
|
||||
return Err(ProxyError::Config(
|
||||
"upstream.ipv4 and upstream.ipv6 cannot both be false".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(prefer) = upstream.prefer
|
||||
&& prefer != 4
|
||||
&& prefer != 6
|
||||
{
|
||||
return Err(ProxyError::Config(
|
||||
"upstream.prefer must be 4 or 6".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if let UpstreamType::Shadowsocks { url, .. } = &upstream.upstream_type {
|
||||
let parsed = ShadowsocksServerConfig::from_url(url)
|
||||
.map_err(|error| ProxyError::Config(format!("invalid shadowsocks url: {error}")))?;
|
||||
if parsed.plugin().is_some() {
|
||||
return Err(ProxyError::Config(
|
||||
"shadowsocks plugins are not supported".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn normalize_upstream_family_policy(config: &mut ProxyConfig) {
|
||||
for (idx, upstream) in config.upstreams.iter_mut().enumerate() {
|
||||
if matches!(upstream.ipv4, Some(false)) && upstream.prefer == Some(4) {
|
||||
warn!(
|
||||
upstream = idx,
|
||||
"upstream.prefer=4 but upstream.ipv4=false; forcing prefer=6"
|
||||
);
|
||||
upstream.prefer = Some(6);
|
||||
}
|
||||
|
||||
if matches!(upstream.ipv6, Some(false)) && upstream.prefer == Some(6) {
|
||||
warn!(
|
||||
upstream = idx,
|
||||
"upstream.prefer=6 but upstream.ipv6=false; forcing prefer=4"
|
||||
);
|
||||
upstream.prefer = Some(4);
|
||||
}
|
||||
}
|
||||
}
|
||||
2446
src/config/tests/load_basic_tests.rs
Normal file
2446
src/config/tests/load_basic_tests.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,21 @@
|
||||
use super::*;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
static TEMP_CONFIG_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn write_temp_config(contents: &str) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system time must be after unix epoch")
|
||||
.as_nanos();
|
||||
let path = std::env::temp_dir().join(format!("telemt-load-mask-shape-security-{nonce}.toml"));
|
||||
let seq = TEMP_CONFIG_COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
let pid = std::process::id();
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"telemt-load-mask-shape-security-{pid}-{seq}-{nonce}.toml"
|
||||
));
|
||||
fs::write(&path, contents).expect("temp config write must succeed");
|
||||
path
|
||||
}
|
||||
|
||||
@@ -95,22 +95,162 @@ max_client_frame = 16777217
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_writer_byte_budget_below_frame_residency_minimum() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[general]
|
||||
max_client_frame = 16777216
|
||||
me_writer_byte_budget_bytes = 33554432
|
||||
"#,
|
||||
);
|
||||
|
||||
let err = ProxyConfig::load(&path)
|
||||
.expect_err("writer byte budget below frame residency minimum must fail");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("general.me_writer_byte_budget_bytes must be within [33570816, 268435456]"),
|
||||
"error must explain writer byte budget minimum, got: {msg}"
|
||||
);
|
||||
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_unaligned_writer_byte_budget() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[general]
|
||||
me_writer_byte_budget_bytes = 33570817
|
||||
"#,
|
||||
);
|
||||
|
||||
let err = ProxyConfig::load(&path)
|
||||
.expect_err("writer byte budget outside permit granularity must fail");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("general.me_writer_byte_budget_bytes must be a multiple of 16384"),
|
||||
"error must explain writer byte budget alignment, got: {msg}"
|
||||
);
|
||||
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_writer_byte_budget_above_hard_cap() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[general]
|
||||
me_writer_byte_budget_bytes = 268451840
|
||||
"#,
|
||||
);
|
||||
|
||||
let err = ProxyConfig::load(&path).expect_err("writer byte budget above hard cap must fail");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("general.me_writer_byte_budget_bytes must be within [33570816, 268435456]"),
|
||||
"error must explain writer byte budget hard cap, got: {msg}"
|
||||
);
|
||||
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_unaligned_direct_relay_buffer_budget() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[general]
|
||||
direct_relay_buffer_budget_max_bytes = 16777217
|
||||
"#,
|
||||
);
|
||||
|
||||
let err = ProxyConfig::load(&path).expect_err("unaligned direct relay buffer budget must fail");
|
||||
assert!(
|
||||
err.to_string().contains(
|
||||
"general.direct_relay_buffer_budget_max_bytes must be 0 or a multiple of 4096"
|
||||
)
|
||||
);
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_direct_relay_buffer_budget_above_hard_cap() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[general]
|
||||
direct_relay_buffer_budget_max_bytes = 2147487744
|
||||
"#,
|
||||
);
|
||||
|
||||
let err =
|
||||
ProxyConfig::load(&path).expect_err("direct relay buffer budget above hard cap must fail");
|
||||
assert!(err.to_string().contains(
|
||||
"general.direct_relay_buffer_budget_max_bytes must be 0 or within [16777216, 2147483648]"
|
||||
));
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_listen_backlog_above_i32_upper_bound() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[server]
|
||||
listen_backlog = 2147483648
|
||||
"#,
|
||||
);
|
||||
|
||||
let err = ProxyConfig::load(&path).expect_err("listen_backlog above socket cap must fail");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("server.listen_backlog must be within [1, 2147483647]"),
|
||||
"error must explain listen_backlog hard cap, got: {msg}"
|
||||
);
|
||||
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_rejects_zero_listen_backlog() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[server]
|
||||
listen_backlog = 0
|
||||
"#,
|
||||
);
|
||||
|
||||
let err = ProxyConfig::load(&path).expect_err("zero listen_backlog must fail");
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("server.listen_backlog must be within [1, 2147483647]"),
|
||||
"error must explain listen_backlog lower bound, got: {msg}"
|
||||
);
|
||||
|
||||
remove_temp_config(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_accepts_memory_limits_at_hard_upper_bounds() {
|
||||
let path = write_temp_config(
|
||||
r#"
|
||||
[general]
|
||||
me_writer_cmd_channel_capacity = 16384
|
||||
me_writer_byte_budget_bytes = 268435456
|
||||
me_route_channel_capacity = 8192
|
||||
me_c2me_channel_capacity = 8192
|
||||
direct_relay_buffer_budget_max_bytes = 2147483648
|
||||
max_client_frame = 16777216
|
||||
"#,
|
||||
);
|
||||
|
||||
let cfg = ProxyConfig::load(&path).expect("hard upper bound values must be accepted");
|
||||
assert_eq!(cfg.general.me_writer_cmd_channel_capacity, 16384);
|
||||
assert_eq!(cfg.general.me_writer_byte_budget_bytes, 256 * 1024 * 1024);
|
||||
assert_eq!(cfg.general.me_route_channel_capacity, 8192);
|
||||
assert_eq!(cfg.general.me_c2me_channel_capacity, 8192);
|
||||
assert_eq!(
|
||||
cfg.general.direct_relay_buffer_budget_max_bytes,
|
||||
2 * 1024 * 1024 * 1024
|
||||
);
|
||||
assert_eq!(cfg.general.max_client_frame, 16 * 1024 * 1024);
|
||||
|
||||
remove_temp_config(&path);
|
||||
|
||||
@@ -2,6 +2,7 @@ use chrono::{DateTime, Utc};
|
||||
use ipnetwork::IpNetwork;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::net::IpAddr;
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -63,6 +64,86 @@ impl std::fmt::Display for LogLevel {
|
||||
}
|
||||
}
|
||||
|
||||
/// Logging output destination.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum LoggingDestination {
|
||||
/// Write logs to stderr.
|
||||
#[default]
|
||||
Stderr,
|
||||
/// Write logs to syslog on Unix platforms.
|
||||
Syslog,
|
||||
/// Write logs to a file.
|
||||
File,
|
||||
}
|
||||
|
||||
/// Time-based log rotation interval for file logging.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum LogRotation {
|
||||
/// Do not rotate logs by time.
|
||||
#[default]
|
||||
Never,
|
||||
/// Rotate once per minute.
|
||||
Minutely,
|
||||
/// Rotate once per hour.
|
||||
Hourly,
|
||||
/// Rotate once per day.
|
||||
Daily,
|
||||
/// Rotate once per week.
|
||||
Weekly,
|
||||
}
|
||||
|
||||
impl LogRotation {
|
||||
/// Parse a CLI rotation value.
|
||||
pub fn from_cli_arg(value: &str) -> Option<Self> {
|
||||
match value.to_ascii_lowercase().as_str() {
|
||||
"never" | "none" | "off" => Some(Self::Never),
|
||||
"minutely" | "minute" => Some(Self::Minutely),
|
||||
"hourly" | "hour" => Some(Self::Hourly),
|
||||
"daily" | "day" => Some(Self::Daily),
|
||||
"weekly" | "week" => Some(Self::Weekly),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// File logging and retention settings.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LoggingConfig {
|
||||
/// Effective logging destination.
|
||||
#[serde(default)]
|
||||
pub destination: LoggingDestination,
|
||||
/// File path used when `destination = "file"`.
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
/// Time rotation interval for file logs.
|
||||
#[serde(default)]
|
||||
pub rotation: LogRotation,
|
||||
/// Maximum active log file size before rotating. `0` disables size rotation.
|
||||
#[serde(default)]
|
||||
pub max_size_bytes: u64,
|
||||
/// Maximum number of matching log files to keep. `0` disables count retention.
|
||||
#[serde(default)]
|
||||
pub max_files: usize,
|
||||
/// Maximum age for rotated log files in seconds. `0` disables age retention.
|
||||
#[serde(default)]
|
||||
pub max_age_secs: u64,
|
||||
}
|
||||
|
||||
impl Default for LoggingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
destination: LoggingDestination::Stderr,
|
||||
path: None,
|
||||
rotation: LogRotation::Never,
|
||||
max_size_bytes: 0,
|
||||
max_files: 0,
|
||||
max_age_secs: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Middle-End telemetry verbosity level.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
@@ -429,7 +510,7 @@ pub struct GeneralConfig {
|
||||
pub ad_tag: Option<String>,
|
||||
|
||||
/// Public IP override for middle-proxy NAT environments.
|
||||
/// When set, this IP is used in ME key derivation and RPC_PROXY_REQ "our_addr".
|
||||
/// When set, this IP is used in ME key derivation and local address translation.
|
||||
#[serde(default)]
|
||||
pub middle_proxy_nat_ip: Option<IpAddr>,
|
||||
|
||||
@@ -498,6 +579,10 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_me_writer_cmd_channel_capacity")]
|
||||
pub me_writer_cmd_channel_capacity: usize,
|
||||
|
||||
/// Resident-memory budget in bytes for each ME writer data queue.
|
||||
#[serde(default = "default_me_writer_byte_budget_bytes")]
|
||||
pub me_writer_byte_budget_bytes: usize,
|
||||
|
||||
/// Capacity of per-connection ME response route channel.
|
||||
#[serde(default = "default_me_route_channel_capacity")]
|
||||
pub me_route_channel_capacity: usize,
|
||||
@@ -541,7 +626,7 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_me_d2c_frame_buf_shrink_threshold_bytes")]
|
||||
pub me_d2c_frame_buf_shrink_threshold_bytes: usize,
|
||||
|
||||
/// Copy buffer size for client->DC direction in direct relay.
|
||||
/// Copy buffer ceiling for client->DC direction in direct relay.
|
||||
///
|
||||
/// This is also the upper bound for one amortized upload rate-limit burst:
|
||||
/// upload debt is settled before the next relay read instead of blocking
|
||||
@@ -549,13 +634,18 @@ pub struct GeneralConfig {
|
||||
#[serde(default = "default_direct_relay_copy_buf_c2s_bytes")]
|
||||
pub direct_relay_copy_buf_c2s_bytes: usize,
|
||||
|
||||
/// Copy buffer size for DC->client direction in direct relay.
|
||||
/// Copy buffer ceiling for DC->client direction in direct relay.
|
||||
///
|
||||
/// This bounds one direct download rate-limit grant because writes are
|
||||
/// clipped to the currently available shaper budget.
|
||||
#[serde(default = "default_direct_relay_copy_buf_s2c_bytes")]
|
||||
pub direct_relay_copy_buf_s2c_bytes: usize,
|
||||
|
||||
/// Process-wide hard ceiling for Direct relay copy buffers.
|
||||
/// `0` derives the ceiling from host and cgroup memory limits.
|
||||
#[serde(default = "default_direct_relay_buffer_budget_max_bytes")]
|
||||
pub direct_relay_buffer_budget_max_bytes: usize,
|
||||
|
||||
/// Max pending ciphertext buffer per client writer (bytes).
|
||||
/// Controls FakeTLS backpressure vs throughput.
|
||||
#[serde(default = "default_crypto_pending_buffer")]
|
||||
@@ -1022,6 +1112,7 @@ impl Default for GeneralConfig {
|
||||
me_keepalive_payload_random: default_true(),
|
||||
rpc_proxy_req_every: default_rpc_proxy_req_every(),
|
||||
me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(),
|
||||
me_writer_byte_budget_bytes: default_me_writer_byte_budget_bytes(),
|
||||
me_route_channel_capacity: default_me_route_channel_capacity(),
|
||||
me_c2me_channel_capacity: default_me_c2me_channel_capacity(),
|
||||
me_c2me_send_timeout_ms: default_me_c2me_send_timeout_ms(),
|
||||
@@ -1035,6 +1126,7 @@ impl Default for GeneralConfig {
|
||||
default_me_d2c_frame_buf_shrink_threshold_bytes(),
|
||||
direct_relay_copy_buf_c2s_bytes: default_direct_relay_copy_buf_c2s_bytes(),
|
||||
direct_relay_copy_buf_s2c_bytes: default_direct_relay_copy_buf_s2c_bytes(),
|
||||
direct_relay_buffer_budget_max_bytes: default_direct_relay_buffer_budget_max_bytes(),
|
||||
me_warmup_stagger_enabled: default_true(),
|
||||
me_warmup_step_delay_ms: default_warmup_step_delay_ms(),
|
||||
me_warmup_step_jitter_ms: default_warmup_step_jitter_ms(),
|
||||
@@ -1369,6 +1461,77 @@ impl ConntrackPressureProfile {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-listener SYN limiter mode.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum SynLimitMode {
|
||||
/// Disable SYN limiting for this listener.
|
||||
#[default]
|
||||
Off,
|
||||
/// Use iptables/ip6tables two-tier SYN-fix rules with the hashlimit match.
|
||||
Iptables,
|
||||
/// Use nftables two-tier SYN-fix rules with per-source token-bucket meters.
|
||||
Nftables,
|
||||
}
|
||||
|
||||
impl Serialize for SynLimitMode {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Self::Off => serializer.serialize_bool(false),
|
||||
Self::Iptables => serializer.serialize_str("iptables"),
|
||||
Self::Nftables => serializer.serialize_str("nftables"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SynLimitMode {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
struct SynLimitModeVisitor;
|
||||
|
||||
impl<'de> serde::de::Visitor<'de> for SynLimitModeVisitor {
|
||||
type Value = SynLimitMode;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("false, iptables, or nftables")
|
||||
}
|
||||
|
||||
fn visit_bool<E>(self, value: bool) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
if value {
|
||||
Err(E::custom(
|
||||
"synlimit=true is ambiguous; use \"iptables\" or \"nftables\"",
|
||||
))
|
||||
} else {
|
||||
Ok(SynLimitMode::Off)
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: serde::de::Error,
|
||||
{
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"false" | "off" | "disabled" | "none" => Ok(SynLimitMode::Off),
|
||||
"iptables" => Ok(SynLimitMode::Iptables),
|
||||
"nftables" => Ok(SynLimitMode::Nftables),
|
||||
_ => Err(E::custom(
|
||||
"synlimit must be false, \"iptables\", or \"nftables\"",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(SynLimitModeVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ConntrackControlConfig {
|
||||
/// Enables runtime conntrack-control worker for pressure mitigation.
|
||||
@@ -1451,6 +1614,20 @@ pub struct ServerConfig {
|
||||
#[serde(default)]
|
||||
pub listen_tcp: Option<bool>,
|
||||
|
||||
/// Client-facing TCP MSS preset or custom value for all TCP listeners.
|
||||
/// Empty string or omitted value keeps the kernel default.
|
||||
#[serde(default)]
|
||||
pub client_mss: Option<String>,
|
||||
|
||||
/// Client-facing TCP MSS to switch to AFTER the TLS handshake (ServerHello)
|
||||
/// is sent. Lets `client_mss` fragment ONLY the handshake (the DPI-inspected
|
||||
/// part) while the bulk transfer uses normal-size packets — avoids the ~10x
|
||||
/// packets-per-second blowup that triggers anti-DDoS abuse blocks on
|
||||
/// pps-policing hosts. Empty/omitted = keep the handshake MSS for the whole
|
||||
/// connection (previous behavior). Same preset/int grammar as `client_mss`.
|
||||
#[serde(default)]
|
||||
pub client_mss_bulk: Option<String>,
|
||||
|
||||
/// Accept HAProxy PROXY protocol headers on incoming connections.
|
||||
/// When enabled, real client IPs are extracted from PROXY v1/v2 headers.
|
||||
#[serde(default)]
|
||||
@@ -1517,6 +1694,8 @@ impl Default for ServerConfig {
|
||||
listen_unix_sock: None,
|
||||
listen_unix_sock_perm: None,
|
||||
listen_tcp: None,
|
||||
client_mss: None,
|
||||
client_mss_bulk: None,
|
||||
proxy_protocol: false,
|
||||
proxy_protocol_header_timeout_ms: default_proxy_protocol_header_timeout_ms(),
|
||||
proxy_protocol_trusted_cidrs: default_proxy_protocol_trusted_cidrs(),
|
||||
@@ -1720,6 +1899,10 @@ pub struct AntiCensorshipConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub mask: bool,
|
||||
|
||||
/// Use the ClientHello SNI as the mask TCP target for configured TLS domains.
|
||||
#[serde(default = "default_true")]
|
||||
pub mask_dynamic: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub mask_host: Option<String>,
|
||||
|
||||
@@ -1855,6 +2038,7 @@ impl Default for AntiCensorshipConfig {
|
||||
tls_fetch_scope: default_tls_fetch_scope(),
|
||||
tls_fetch: TlsFetchConfig::default(),
|
||||
mask: default_true(),
|
||||
mask_dynamic: default_true(),
|
||||
mask_host: None,
|
||||
mask_port: default_mask_port(),
|
||||
exclusive_mask: HashMap::new(),
|
||||
@@ -1892,6 +2076,9 @@ pub struct AccessConfig {
|
||||
#[serde(default = "default_access_users")]
|
||||
pub users: HashMap<String, String>,
|
||||
|
||||
#[serde(default)]
|
||||
pub user_enabled: HashMap<String, bool>,
|
||||
|
||||
/// Per-user ad_tag (32 hex chars from @MTProxybot).
|
||||
#[serde(default)]
|
||||
pub user_ad_tags: HashMap<String, String>,
|
||||
@@ -1923,11 +2110,13 @@ pub struct AccessConfig {
|
||||
|
||||
/// Per-CIDR aggregate transport rate limits in bits-per-second.
|
||||
///
|
||||
/// Matching uses longest-prefix-wins semantics. A value of `0` in one
|
||||
/// direction means "unlimited" for that direction. Limits are amortized
|
||||
/// with the same bounded-burst contract as per-user rate limits.
|
||||
/// Explicit CIDR keys use longest-prefix-wins semantics. Auto-template
|
||||
/// keys (`*4/N`, `*6/N`, `*/N`) lazily create per-source-subnet buckets
|
||||
/// after explicit CIDR matching misses. A value of `0` in one direction
|
||||
/// means "unlimited" for that direction. Limits are amortized with the
|
||||
/// same bounded-burst contract as per-user rate limits.
|
||||
#[serde(default)]
|
||||
pub cidr_rate_limits: HashMap<IpNetwork, RateLimitBps>,
|
||||
pub cidr_rate_limits: HashMap<CidrRateLimitKey, RateLimitBps>,
|
||||
|
||||
/// Per-username client source IP/CIDR deny list. Checked after successful
|
||||
/// authentication; matching IPs get the same rejection path as invalid auth
|
||||
@@ -1963,6 +2152,7 @@ impl Default for AccessConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
users: default_access_users(),
|
||||
user_enabled: HashMap::new(),
|
||||
user_ad_tags: HashMap::new(),
|
||||
user_max_tcp_conns: HashMap::new(),
|
||||
user_max_tcp_conns_global_each: default_user_max_tcp_conns_global_each(),
|
||||
@@ -1983,6 +2173,10 @@ impl Default for AccessConfig {
|
||||
}
|
||||
|
||||
impl AccessConfig {
|
||||
pub fn is_user_enabled(&self, username: &str) -> bool {
|
||||
self.user_enabled.get(username).copied().unwrap_or(true)
|
||||
}
|
||||
|
||||
/// Returns true if `ip` is contained in any CIDR listed for `username` under `user_source_deny`.
|
||||
pub fn is_user_source_ip_denied(&self, username: &str, ip: IpAddr) -> bool {
|
||||
self.user_source_deny
|
||||
@@ -1991,10 +2185,156 @@ impl AccessConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Key used by `access.cidr_rate_limits`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum CidrRateLimitKey {
|
||||
/// Explicit source CIDR rule.
|
||||
Network(IpNetwork),
|
||||
/// IPv4 auto-template that creates one bucket for each matching `/N`.
|
||||
AutoV4(u8),
|
||||
/// IPv6 auto-template that creates one bucket for each matching `/N`.
|
||||
AutoV6(u8),
|
||||
/// Dual-stack auto-template; IPv4 uses `/N`, IPv6 uses `/(N * 4)`.
|
||||
AutoDual(u8),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub(crate) enum CidrAutoTemplateFamily {
|
||||
V4,
|
||||
V6,
|
||||
}
|
||||
|
||||
impl CidrAutoTemplateFamily {
|
||||
pub(crate) fn marker(self) -> &'static str {
|
||||
match self {
|
||||
Self::V4 => "*4",
|
||||
Self::V6 => "*6",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub(crate) struct CidrAutoTemplate {
|
||||
pub(crate) family: CidrAutoTemplateFamily,
|
||||
pub(crate) prefix_len: u8,
|
||||
}
|
||||
|
||||
impl fmt::Display for CidrAutoTemplate {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(formatter, "{}/{}", self.family.marker(), self.prefix_len)
|
||||
}
|
||||
}
|
||||
|
||||
impl CidrRateLimitKey {
|
||||
pub(crate) fn auto_templates(&self) -> [Option<CidrAutoTemplate>; 2] {
|
||||
match *self {
|
||||
Self::Network(_) => [None, None],
|
||||
Self::AutoV4(prefix_len) => [
|
||||
Some(CidrAutoTemplate {
|
||||
family: CidrAutoTemplateFamily::V4,
|
||||
prefix_len,
|
||||
}),
|
||||
None,
|
||||
],
|
||||
Self::AutoV6(prefix_len) => [
|
||||
Some(CidrAutoTemplate {
|
||||
family: CidrAutoTemplateFamily::V6,
|
||||
prefix_len,
|
||||
}),
|
||||
None,
|
||||
],
|
||||
Self::AutoDual(prefix_len) => [
|
||||
Some(CidrAutoTemplate {
|
||||
family: CidrAutoTemplateFamily::V4,
|
||||
prefix_len,
|
||||
}),
|
||||
Some(CidrAutoTemplate {
|
||||
family: CidrAutoTemplateFamily::V6,
|
||||
prefix_len: prefix_len.saturating_mul(4),
|
||||
}),
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for CidrRateLimitKey {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Network(cidr) => write!(formatter, "{cidr}"),
|
||||
Self::AutoV4(prefix_len) => write!(formatter, "*4/{prefix_len}"),
|
||||
Self::AutoV6(prefix_len) => write!(formatter, "*6/{prefix_len}"),
|
||||
Self::AutoDual(prefix_len) => write!(formatter, "*/{prefix_len}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for CidrRateLimitKey {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
serializer.collect_str(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for CidrRateLimitKey {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = String::deserialize(deserializer)?;
|
||||
parse_cidr_rate_limit_key(&value).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_cidr_rate_limit_key(value: &str) -> std::result::Result<CidrRateLimitKey, String> {
|
||||
if let Some(prefix) = value.strip_prefix("*4/") {
|
||||
return parse_cidr_auto_prefix(value, prefix, 32).map(CidrRateLimitKey::AutoV4);
|
||||
}
|
||||
if let Some(prefix) = value.strip_prefix("*6/") {
|
||||
return parse_cidr_auto_prefix(value, prefix, 128).map(CidrRateLimitKey::AutoV6);
|
||||
}
|
||||
if let Some(prefix) = value.strip_prefix("*/") {
|
||||
return parse_cidr_auto_prefix(value, prefix, 32).map(CidrRateLimitKey::AutoDual);
|
||||
}
|
||||
if value.starts_with('*') {
|
||||
return Err(format!(
|
||||
"invalid CIDR rate limit key {value:?}; expected CIDR, *4/N, *6/N, or */N"
|
||||
));
|
||||
}
|
||||
value
|
||||
.parse::<IpNetwork>()
|
||||
.map(CidrRateLimitKey::Network)
|
||||
.map_err(|error| {
|
||||
format!(
|
||||
"invalid CIDR rate limit key {value:?}: {error}; expected CIDR, *4/N, *6/N, or */N"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_cidr_auto_prefix(
|
||||
key: &str,
|
||||
prefix: &str,
|
||||
max_prefix: u8,
|
||||
) -> std::result::Result<u8, String> {
|
||||
let prefix = prefix.parse::<u8>().map_err(|_| {
|
||||
format!("invalid CIDR auto-template key {key:?}; prefix must be within 0..={max_prefix}")
|
||||
})?;
|
||||
if prefix > max_prefix {
|
||||
return Err(format!(
|
||||
"invalid CIDR auto-template key {key:?}; prefix must be within 0..={max_prefix}"
|
||||
));
|
||||
}
|
||||
Ok(prefix)
|
||||
}
|
||||
|
||||
/// Transport rate limit in bits-per-second.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RateLimitBps {
|
||||
/// Upload direction limit in bits-per-second; `0` means unlimited.
|
||||
#[serde(default)]
|
||||
pub up_bps: u64,
|
||||
/// Download direction limit in bits-per-second; `0` means unlimited.
|
||||
#[serde(default)]
|
||||
pub down_bps: u64,
|
||||
}
|
||||
@@ -2057,6 +2397,20 @@ pub struct UpstreamConfig {
|
||||
/// `None` means auto-detect from runtime connectivity state.
|
||||
#[serde(default)]
|
||||
pub ipv6: Option<bool>,
|
||||
/// Per-upstream IP family preference for Telegram DC targets.
|
||||
/// `None` inherits the effective global `[network].prefer` decision.
|
||||
#[serde(default)]
|
||||
pub prefer: Option<u8>,
|
||||
}
|
||||
|
||||
impl UpstreamConfig {
|
||||
pub fn prefer_ipv6(&self, default_prefer_ipv6: bool) -> bool {
|
||||
match self.prefer {
|
||||
Some(6) => true,
|
||||
Some(4) => false,
|
||||
_ => default_prefer_ipv6,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -2065,6 +2419,37 @@ pub struct ListenerConfig {
|
||||
/// Per-listener TCP port. If omitted, falls back to legacy `server.port`.
|
||||
#[serde(default)]
|
||||
pub port: Option<u16>,
|
||||
/// Per-listener client-facing TCP MSS preset or custom value.
|
||||
/// Empty string disables MSS shaping for this listener.
|
||||
#[serde(default)]
|
||||
pub client_mss: Option<String>,
|
||||
/// Per-listener SYN limiter mode.
|
||||
#[serde(default)]
|
||||
pub synlimit: SynLimitMode,
|
||||
/// Generic SYN-fix token-bucket rate interval.
|
||||
#[serde(default = "default_synlimit_seconds")]
|
||||
pub synlimit_seconds: u32,
|
||||
/// Generic SYN-fix token-bucket rate amount.
|
||||
#[serde(default = "default_synlimit_hitcount")]
|
||||
pub synlimit_hitcount: u32,
|
||||
/// Generic SYN-fix token-bucket burst size.
|
||||
#[serde(default = "default_synlimit_burst")]
|
||||
pub synlimit_burst: u32,
|
||||
/// iOS-like SYN-fix token-bucket rate interval.
|
||||
#[serde(default = "default_synlimit_ios_seconds")]
|
||||
pub synlimit_ios_seconds: u32,
|
||||
/// iOS-like SYN-fix token-bucket rate amount.
|
||||
#[serde(default = "default_synlimit_ios_hitcount")]
|
||||
pub synlimit_ios_hitcount: u32,
|
||||
/// iOS-like SYN-fix token-bucket burst size.
|
||||
#[serde(default = "default_synlimit_ios_burst")]
|
||||
pub synlimit_ios_burst: u32,
|
||||
/// Hashlimit entry expiration in milliseconds for iptables/ip6tables rules.
|
||||
#[serde(default = "default_synlimit_hashlimit_expire_ms")]
|
||||
pub synlimit_hashlimit_expire_ms: u32,
|
||||
/// Hashlimit table size for iptables/ip6tables rules.
|
||||
#[serde(default = "default_synlimit_hashlimit_size")]
|
||||
pub synlimit_hashlimit_size: u32,
|
||||
/// IP address or hostname to announce in proxy links.
|
||||
/// Takes precedence over `announce_ip` if both are set.
|
||||
#[serde(default)]
|
||||
@@ -2082,6 +2467,69 @@ pub struct ListenerConfig {
|
||||
pub reuse_allow: bool,
|
||||
}
|
||||
|
||||
/// Client-facing TCP MSS preset for extreme-low fragmentation profiles.
|
||||
pub const CLIENT_MSS_EXTREME_LOW: u16 = 88;
|
||||
/// Client-facing TCP MSS preset matching TSPU-oriented deployments.
|
||||
pub const CLIENT_MSS_TSPU: u16 = 92;
|
||||
/// Client-facing TCP MSS preset for 2-in-8 segment shaping.
|
||||
pub const CLIENT_MSS_2IN8: u16 = 256;
|
||||
/// Minimum accepted custom client-facing TCP MSS value.
|
||||
pub const CLIENT_MSS_MIN: u16 = CLIENT_MSS_EXTREME_LOW;
|
||||
/// Maximum accepted custom client-facing TCP MSS value.
|
||||
pub const CLIENT_MSS_MAX: u16 = 4096;
|
||||
|
||||
impl ServerConfig {
|
||||
/// Resolves the global client-facing TCP MSS setting.
|
||||
pub fn client_mss_value(&self) -> std::result::Result<Option<u16>, String> {
|
||||
parse_client_mss(self.client_mss.as_deref())
|
||||
}
|
||||
|
||||
/// Resolves the post-handshake (bulk transfer) client MSS, if configured.
|
||||
pub fn client_mss_bulk_value(&self) -> std::result::Result<Option<u16>, String> {
|
||||
parse_client_mss(self.client_mss_bulk.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
impl ListenerConfig {
|
||||
/// Resolves the listener MSS override, falling back to the global server value.
|
||||
pub fn effective_client_mss(
|
||||
&self,
|
||||
server: &ServerConfig,
|
||||
) -> std::result::Result<Option<u16>, String> {
|
||||
match self.client_mss.as_deref() {
|
||||
Some(value) => parse_client_mss(Some(value)),
|
||||
None => server.client_mss_value(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_client_mss(raw: Option<&str>) -> std::result::Result<Option<u16>, String> {
|
||||
let Some(raw) = raw else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value = raw.trim();
|
||||
if value.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
match value.to_ascii_lowercase().as_str() {
|
||||
"extreme-low" => return Ok(Some(CLIENT_MSS_EXTREME_LOW)),
|
||||
"tspu" => return Ok(Some(CLIENT_MSS_TSPU)),
|
||||
"2in8" => return Ok(Some(CLIENT_MSS_2IN8)),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let parsed = value
|
||||
.parse::<u16>()
|
||||
.map_err(|_| "must be \"\", extreme-low, tspu, 2in8, or a decimal value".to_string())?;
|
||||
if !(CLIENT_MSS_MIN..=CLIENT_MSS_MAX).contains(&parsed) {
|
||||
return Err(format!(
|
||||
"custom value must be within [{CLIENT_MSS_MIN}, {CLIENT_MSS_MAX}]"
|
||||
));
|
||||
}
|
||||
Ok(Some(parsed))
|
||||
}
|
||||
|
||||
// ============= ShowLink =============
|
||||
|
||||
/// Controls which users' proxy links are displayed at startup.
|
||||
|
||||
@@ -7,6 +7,7 @@ use std::time::Duration;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::process::Command;
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::{ConntrackBackend, ConntrackMode, ProxyConfig};
|
||||
@@ -57,10 +58,11 @@ impl PressureState {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_conntrack_controller(
|
||||
pub(crate) async fn run_conntrack_controller(
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
stats: Arc<Stats>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
cancellation: CancellationToken,
|
||||
) {
|
||||
if !cfg!(target_os = "linux") {
|
||||
let cfg = config_rx.borrow();
|
||||
@@ -87,16 +89,15 @@ pub(crate) fn spawn_conntrack_controller(
|
||||
|
||||
let (tx, rx) = mpsc::channel(CONNTRACK_EVENT_QUEUE_CAPACITY);
|
||||
shared.set_conntrack_close_sender(tx);
|
||||
tokio::spawn(async move {
|
||||
run_conntrack_controller(config_rx, stats, shared, rx).await;
|
||||
});
|
||||
run_conntrack_controller_worker(config_rx, stats, shared, rx, cancellation).await;
|
||||
}
|
||||
|
||||
async fn run_conntrack_controller(
|
||||
async fn run_conntrack_controller_worker(
|
||||
mut config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
stats: Arc<Stats>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
mut close_rx: mpsc::Receiver<ConntrackCloseEvent>,
|
||||
cancellation: CancellationToken,
|
||||
) {
|
||||
let mut cfg = config_rx.borrow().clone();
|
||||
let mut pressure_state = PressureState::new(stats.as_ref());
|
||||
@@ -115,6 +116,7 @@ async fn run_conntrack_controller(
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = cancellation.cancelled() => break,
|
||||
changed = config_rx.changed() => {
|
||||
if changed.is_err() {
|
||||
break;
|
||||
@@ -705,7 +707,7 @@ fn nofile_soft_limit() -> Option<u64> {
|
||||
if rc != 0 {
|
||||
return None;
|
||||
}
|
||||
return Some(lim.rlim_cur);
|
||||
return Some(lim.rlim_cur.into());
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
//!
|
||||
//! ## Zeroize policy
|
||||
//!
|
||||
//! - `AesCbc` stores raw key/IV bytes and zeroizes them on drop.
|
||||
//! - `AesCtr` wraps an opaque `Aes256Ctr` cipher from the `ctr` crate.
|
||||
//! The expanded key schedule lives inside that type and cannot be
|
||||
//! zeroized from outside. Callers that hold raw key material (e.g.
|
||||
//! `HandshakeSuccess`, `ObfuscationParams`) are responsible for
|
||||
//! zeroizing their own copies.
|
||||
//! - `AesCbc` stores raw key/IV bytes and zeroizes them on drop. Temporary
|
||||
//! expanded key schedules are also zeroized by the RustCrypto backend.
|
||||
//! - `AesCtr` uses the RustCrypto `zeroize` contract to clear its expanded
|
||||
//! key schedule, counter, and buffered keystream on drop.
|
||||
//! - Callers that hold raw key material (e.g. `HandshakeSuccess`,
|
||||
//! `ObfuscationParams`) remain responsible for zeroizing their own copies.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
@@ -19,24 +19,28 @@ use ctr::{
|
||||
Ctr128BE,
|
||||
cipher::{KeyIvInit, StreamCipher},
|
||||
};
|
||||
use zeroize::Zeroize;
|
||||
use zeroize::{Zeroize, ZeroizeOnDrop};
|
||||
|
||||
type Aes256Ctr = Ctr128BE<Aes256>;
|
||||
|
||||
static_assertions::assert_impl_all!(Aes256: ZeroizeOnDrop);
|
||||
static_assertions::assert_impl_all!(Aes256Ctr: ZeroizeOnDrop);
|
||||
|
||||
// ============= AES-256-CTR =============
|
||||
|
||||
/// AES-256-CTR encryptor/decryptor
|
||||
///
|
||||
/// CTR mode is symmetric — encryption and decryption are the same operation.
|
||||
///
|
||||
/// **Zeroize note:** The inner `Aes256Ctr` cipher state (expanded key schedule
|
||||
/// + counter) is opaque and cannot be zeroized. If you need to protect key
|
||||
/// material, zeroize the `[u8; 32]` key and `u128` IV at the call site
|
||||
/// before dropping them.
|
||||
/// **Zeroize note:** The inner `Aes256Ctr` zeroizes its expanded key schedule,
|
||||
/// counter, and buffered keystream on drop. Callers remain responsible for
|
||||
/// zeroizing their own raw key and IV copies.
|
||||
pub struct AesCtr {
|
||||
cipher: Aes256Ctr,
|
||||
}
|
||||
|
||||
impl ZeroizeOnDrop for AesCtr {}
|
||||
|
||||
impl AesCtr {
|
||||
/// Create new AES-CTR cipher with key and IV
|
||||
pub fn new(key: &[u8; 32], iv: u128) -> Self {
|
||||
@@ -92,7 +96,7 @@ impl AesCtr {
|
||||
/// are different operations. This implementation handles CBC chaining
|
||||
/// correctly across multiple blocks.
|
||||
///
|
||||
/// Key and IV are zeroized on drop.
|
||||
/// Key, IV, and temporary expanded key schedules are zeroized on drop.
|
||||
pub struct AesCbc {
|
||||
key: [u8; 32],
|
||||
iv: [u8; 16],
|
||||
@@ -105,6 +109,8 @@ impl Drop for AesCbc {
|
||||
}
|
||||
}
|
||||
|
||||
impl ZeroizeOnDrop for AesCbc {}
|
||||
|
||||
impl AesCbc {
|
||||
/// AES block size
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
@@ -5,7 +5,7 @@ pub mod hash;
|
||||
pub mod random;
|
||||
|
||||
pub use aes::{AesCbc, AesCtr};
|
||||
pub use hash::{
|
||||
build_middleproxy_prekey, crc32, crc32c, derive_middleproxy_keys, sha256, sha256_hmac,
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use hash::build_middleproxy_prekey;
|
||||
pub use hash::{crc32, crc32c, derive_middleproxy_keys, sha256, sha256_hmac};
|
||||
pub use random::SecureRandom;
|
||||
|
||||
@@ -245,6 +245,9 @@ pub enum ProxyError {
|
||||
InvalidSecret { user: String, reason: String },
|
||||
|
||||
// ============= User Errors =============
|
||||
#[error("User {user} disabled")]
|
||||
UserDisabled { user: String },
|
||||
|
||||
#[error("User {user} expired")]
|
||||
UserExpired { user: String },
|
||||
|
||||
|
||||
323
src/logging.rs
323
src/logging.rs
@@ -5,16 +5,41 @@
|
||||
//! - syslog (Unix only, for traditional init systems)
|
||||
//! - file (with optional rotation)
|
||||
|
||||
#![allow(dead_code)] // Infrastructure module - used via CLI flags
|
||||
// Infrastructure module used via CLI flags.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::config::{LogRotation, LoggingConfig, LoggingDestination};
|
||||
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
use tracing_subscriber::{EnvFilter, fmt, reload};
|
||||
|
||||
// Submodules:
|
||||
// - file: bounded file appender for size and retention controls.
|
||||
mod file;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
/// File logging and retention options resolved from config and CLI.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FileLogOptions {
|
||||
/// Log file path or rolling filename prefix path.
|
||||
pub path: String,
|
||||
/// Time rotation interval.
|
||||
pub rotation: LogRotation,
|
||||
/// Maximum active file size before size rotation. `0` disables it.
|
||||
pub max_size_bytes: u64,
|
||||
/// Maximum number of matching log files to keep. `0` disables it.
|
||||
pub max_files: usize,
|
||||
/// Maximum rotated file age in seconds. `0` disables it.
|
||||
pub max_age_secs: u64,
|
||||
}
|
||||
|
||||
/// Log destination configuration.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub enum LogDestination {
|
||||
/// Log to stderr (default, captured by systemd journald).
|
||||
#[default]
|
||||
@@ -24,12 +49,29 @@ pub enum LogDestination {
|
||||
Syslog,
|
||||
/// Log to a file with optional rotation.
|
||||
File {
|
||||
path: String,
|
||||
/// Rotate daily if true.
|
||||
rotate_daily: bool,
|
||||
/// Resolved file logging options.
|
||||
options: FileLogOptions,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LogCliDestination {
|
||||
Stderr,
|
||||
Syslog,
|
||||
File,
|
||||
}
|
||||
|
||||
/// Logging-related CLI overrides.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct LogCliOptions {
|
||||
destination: Option<LogCliDestination>,
|
||||
path: Option<String>,
|
||||
rotation: Option<LogRotation>,
|
||||
max_size_bytes: Option<u64>,
|
||||
max_files: Option<usize>,
|
||||
max_age_secs: Option<u64>,
|
||||
}
|
||||
|
||||
/// Logging options parsed from CLI/config.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct LoggingOptions {
|
||||
@@ -101,23 +143,29 @@ pub fn init_logging(
|
||||
(filter_handle, LoggingGuard::noop())
|
||||
}
|
||||
|
||||
LogDestination::File { path, rotate_daily } => {
|
||||
let (non_blocking, guard) = if *rotate_daily {
|
||||
// Extract directory and filename prefix
|
||||
let path = Path::new(path);
|
||||
let dir = path.parent().unwrap_or(Path::new("/var/log"));
|
||||
let prefix = path
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("telemt");
|
||||
|
||||
let file_appender = tracing_appender::rolling::daily(dir, prefix);
|
||||
LogDestination::File { options } => {
|
||||
let (non_blocking, guard) = if options.max_size_bytes > 0
|
||||
|| options.max_files > 0
|
||||
|| options.max_age_secs > 0
|
||||
{
|
||||
let file_appender = file::BoundedFileAppender::new(options.clone())
|
||||
.expect("Failed to open log file");
|
||||
tracing_appender::non_blocking(file_appender)
|
||||
} else if !matches!(options.rotation, LogRotation::Never) {
|
||||
let path = Path::new(&options.path);
|
||||
let dir = log_file_dir(path);
|
||||
let prefix = log_file_name(path);
|
||||
let file_appender = tracing_appender::rolling::RollingFileAppender::builder()
|
||||
.rotation(to_tracing_rotation(options.rotation))
|
||||
.filename_prefix(prefix)
|
||||
.build(dir)
|
||||
.expect("Failed to open log file");
|
||||
tracing_appender::non_blocking(file_appender)
|
||||
} else {
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)
|
||||
.open(&options.path)
|
||||
.expect("Failed to open log file");
|
||||
tracing_appender::non_blocking(file)
|
||||
};
|
||||
@@ -137,6 +185,28 @@ pub fn init_logging(
|
||||
}
|
||||
}
|
||||
|
||||
fn log_file_dir(path: &Path) -> &Path {
|
||||
path.parent()
|
||||
.filter(|parent| !parent.as_os_str().is_empty())
|
||||
.unwrap_or_else(|| Path::new("."))
|
||||
}
|
||||
|
||||
fn log_file_name(path: &Path) -> &str {
|
||||
path.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.unwrap_or("telemt")
|
||||
}
|
||||
|
||||
fn to_tracing_rotation(rotation: LogRotation) -> tracing_appender::rolling::Rotation {
|
||||
match rotation {
|
||||
LogRotation::Never => tracing_appender::rolling::Rotation::NEVER,
|
||||
LogRotation::Minutely => tracing_appender::rolling::Rotation::MINUTELY,
|
||||
LogRotation::Hourly => tracing_appender::rolling::Rotation::HOURLY,
|
||||
LogRotation::Daily => tracing_appender::rolling::Rotation::DAILY,
|
||||
LogRotation::Weekly => tracing_appender::rolling::Rotation::WEEKLY,
|
||||
}
|
||||
}
|
||||
|
||||
/// Syslog writer for tracing.
|
||||
#[cfg(unix)]
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -223,121 +293,172 @@ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for SyslogMakeWriter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse log destination from CLI arguments.
|
||||
pub fn parse_log_destination(args: &[String]) -> LogDestination {
|
||||
/// Parse logging overrides from CLI arguments.
|
||||
pub fn parse_log_cli_options(args: &[String]) -> Result<LogCliOptions, String> {
|
||||
let mut options = LogCliOptions::default();
|
||||
let mut i = 0;
|
||||
while i < args.len() {
|
||||
match args[i].as_str() {
|
||||
#[cfg(unix)]
|
||||
"--syslog" => {
|
||||
return LogDestination::Syslog;
|
||||
options.destination = Some(LogCliDestination::Syslog);
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
"--syslog" => {
|
||||
options.destination = Some(LogCliDestination::Syslog);
|
||||
}
|
||||
"--log-file" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
return LogDestination::File {
|
||||
path: args[i].clone(),
|
||||
rotate_daily: false,
|
||||
};
|
||||
options.destination = Some(LogCliDestination::File);
|
||||
options.path = Some(args[i].clone());
|
||||
} else {
|
||||
return Err("Missing value for --log-file".to_string());
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--log-file=") => {
|
||||
return LogDestination::File {
|
||||
path: s.trim_start_matches("--log-file=").to_string(),
|
||||
rotate_daily: false,
|
||||
};
|
||||
options.destination = Some(LogCliDestination::File);
|
||||
options.path = Some(s.trim_start_matches("--log-file=").to_string());
|
||||
}
|
||||
"--log-file-daily" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
return LogDestination::File {
|
||||
path: args[i].clone(),
|
||||
rotate_daily: true,
|
||||
};
|
||||
options.destination = Some(LogCliDestination::File);
|
||||
options.path = Some(args[i].clone());
|
||||
options.rotation = Some(LogRotation::Daily);
|
||||
} else {
|
||||
return Err("Missing value for --log-file-daily".to_string());
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--log-file-daily=") => {
|
||||
return LogDestination::File {
|
||||
path: s.trim_start_matches("--log-file-daily=").to_string(),
|
||||
rotate_daily: true,
|
||||
};
|
||||
options.destination = Some(LogCliDestination::File);
|
||||
options.path = Some(s.trim_start_matches("--log-file-daily=").to_string());
|
||||
options.rotation = Some(LogRotation::Daily);
|
||||
}
|
||||
"--log-rotation" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
options.rotation = Some(parse_rotation_cli_value(&args[i])?);
|
||||
} else {
|
||||
return Err("Missing value for --log-rotation".to_string());
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--log-rotation=") => {
|
||||
options.rotation = Some(parse_rotation_cli_value(
|
||||
s.trim_start_matches("--log-rotation="),
|
||||
)?);
|
||||
}
|
||||
"--log-max-size-bytes" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
options.max_size_bytes =
|
||||
Some(parse_u64_cli_value("--log-max-size-bytes", &args[i])?);
|
||||
} else {
|
||||
return Err("Missing value for --log-max-size-bytes".to_string());
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--log-max-size-bytes=") => {
|
||||
options.max_size_bytes = Some(parse_u64_cli_value(
|
||||
"--log-max-size-bytes",
|
||||
s.trim_start_matches("--log-max-size-bytes="),
|
||||
)?);
|
||||
}
|
||||
"--log-max-files" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
options.max_files = Some(parse_usize_cli_value("--log-max-files", &args[i])?);
|
||||
} else {
|
||||
return Err("Missing value for --log-max-files".to_string());
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--log-max-files=") => {
|
||||
options.max_files = Some(parse_usize_cli_value(
|
||||
"--log-max-files",
|
||||
s.trim_start_matches("--log-max-files="),
|
||||
)?);
|
||||
}
|
||||
"--log-max-age-secs" => {
|
||||
i += 1;
|
||||
if i < args.len() {
|
||||
options.max_age_secs =
|
||||
Some(parse_u64_cli_value("--log-max-age-secs", &args[i])?);
|
||||
} else {
|
||||
return Err("Missing value for --log-max-age-secs".to_string());
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--log-max-age-secs=") => {
|
||||
options.max_age_secs = Some(parse_u64_cli_value(
|
||||
"--log-max-age-secs",
|
||||
s.trim_start_matches("--log-max-age-secs="),
|
||||
)?);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
LogDestination::Stderr
|
||||
Ok(options)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
fn parse_rotation_cli_value(value: &str) -> Result<LogRotation, String> {
|
||||
LogRotation::from_cli_arg(value).ok_or_else(|| {
|
||||
format!(
|
||||
"Invalid --log-rotation value '{value}'. Expected never|minutely|hourly|daily|weekly"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_destination_default() {
|
||||
let args: Vec<String> = vec![];
|
||||
assert!(matches!(
|
||||
parse_log_destination(&args),
|
||||
LogDestination::Stderr
|
||||
));
|
||||
}
|
||||
fn parse_u64_cli_value(flag: &str, value: &str) -> Result<u64, String> {
|
||||
value
|
||||
.parse::<u64>()
|
||||
.map_err(|_| format!("Invalid {flag} value '{value}'. Expected unsigned integer"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_destination_file() {
|
||||
let args = vec!["--log-file".to_string(), "/var/log/telemt.log".to_string()];
|
||||
match parse_log_destination(&args) {
|
||||
LogDestination::File { path, rotate_daily } => {
|
||||
assert_eq!(path, "/var/log/telemt.log");
|
||||
assert!(!rotate_daily);
|
||||
fn parse_usize_cli_value(flag: &str, value: &str) -> Result<usize, String> {
|
||||
value
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("Invalid {flag} value '{value}'. Expected unsigned integer"))
|
||||
}
|
||||
|
||||
/// Resolve effective logging destination from config and CLI overrides.
|
||||
pub fn resolve_log_destination(
|
||||
config: &LoggingConfig,
|
||||
cli: &LogCliOptions,
|
||||
) -> Result<LogDestination, String> {
|
||||
let destination = cli.destination.unwrap_or(match config.destination {
|
||||
LoggingDestination::Stderr => LogCliDestination::Stderr,
|
||||
LoggingDestination::Syslog => LogCliDestination::Syslog,
|
||||
LoggingDestination::File => LogCliDestination::File,
|
||||
});
|
||||
|
||||
match destination {
|
||||
LogCliDestination::Stderr => Ok(LogDestination::Stderr),
|
||||
LogCliDestination::Syslog => {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
Ok(LogDestination::Syslog)
|
||||
}
|
||||
_ => panic!("Expected File destination"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_destination_file_daily() {
|
||||
let args = vec!["--log-file-daily=/var/log/telemt".to_string()];
|
||||
match parse_log_destination(&args) {
|
||||
LogDestination::File { path, rotate_daily } => {
|
||||
assert_eq!(path, "/var/log/telemt");
|
||||
assert!(rotate_daily);
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Err("Syslog logging is only supported on Unix platforms".to_string())
|
||||
}
|
||||
_ => panic!("Expected File destination"),
|
||||
}
|
||||
}
|
||||
LogCliDestination::File => {
|
||||
let path = cli.path.as_ref().or(config.path.as_ref()).ok_or_else(|| {
|
||||
"logging.path or --log-file must be set when file logging is enabled".to_string()
|
||||
})?;
|
||||
if path.trim().is_empty() {
|
||||
return Err("Log file path cannot be empty".to_string());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn test_parse_log_destination_syslog() {
|
||||
let args = vec!["--syslog".to_string()];
|
||||
assert!(matches!(
|
||||
parse_log_destination(&args),
|
||||
LogDestination::Syslog
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn test_syslog_priority_for_level_mapping() {
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::ERROR),
|
||||
libc::LOG_ERR
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::WARN),
|
||||
libc::LOG_WARNING
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::INFO),
|
||||
libc::LOG_INFO
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::DEBUG),
|
||||
libc::LOG_DEBUG
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::TRACE),
|
||||
libc::LOG_DEBUG
|
||||
);
|
||||
Ok(LogDestination::File {
|
||||
options: FileLogOptions {
|
||||
path: path.clone(),
|
||||
rotation: cli.rotation.unwrap_or(config.rotation),
|
||||
max_size_bytes: cli.max_size_bytes.unwrap_or(config.max_size_bytes),
|
||||
max_files: cli.max_files.unwrap_or(config.max_files),
|
||||
max_age_secs: cli.max_age_secs.unwrap_or(config.max_age_secs),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
395
src/logging/file.rs
Normal file
395
src/logging/file.rs
Normal file
@@ -0,0 +1,395 @@
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use chrono::{DateTime, Datelike, Duration as ChronoDuration, Utc};
|
||||
|
||||
use crate::config::LogRotation;
|
||||
|
||||
use super::FileLogOptions;
|
||||
|
||||
const CLEANUP_INTERVAL_SECS: i64 = 60;
|
||||
|
||||
/// File appender with size rotation and local retention cleanup.
|
||||
pub(crate) struct BoundedFileAppender {
|
||||
options: FileLogOptions,
|
||||
dir: PathBuf,
|
||||
base_name: String,
|
||||
current_path: PathBuf,
|
||||
current_size: u64,
|
||||
last_cleanup: DateTime<Utc>,
|
||||
file: Option<File>,
|
||||
now: Box<dyn Fn() -> DateTime<Utc> + Send + Sync>,
|
||||
}
|
||||
|
||||
impl BoundedFileAppender {
|
||||
pub(crate) fn new(options: FileLogOptions) -> io::Result<Self> {
|
||||
Self::with_now(options, Box::new(Utc::now))
|
||||
}
|
||||
|
||||
fn with_now(
|
||||
options: FileLogOptions,
|
||||
now: Box<dyn Fn() -> DateTime<Utc> + Send + Sync>,
|
||||
) -> io::Result<Self> {
|
||||
let path = Path::new(&options.path);
|
||||
let dir = path
|
||||
.parent()
|
||||
.filter(|parent| !parent.as_os_str().is_empty())
|
||||
.unwrap_or_else(|| Path::new("."))
|
||||
.to_path_buf();
|
||||
let base_name = path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("telemt")
|
||||
.to_string();
|
||||
|
||||
let start = now();
|
||||
let current_path = active_path_for(&dir, &base_name, options.rotation, &start);
|
||||
let (file, current_size) = open_append_file(¤t_path)?;
|
||||
let mut appender = Self {
|
||||
options,
|
||||
dir,
|
||||
base_name,
|
||||
current_path,
|
||||
current_size,
|
||||
last_cleanup: start,
|
||||
file: Some(file),
|
||||
now,
|
||||
};
|
||||
appender.cleanup(&start);
|
||||
Ok(appender)
|
||||
}
|
||||
|
||||
fn now(&self) -> DateTime<Utc> {
|
||||
(self.now)()
|
||||
}
|
||||
|
||||
fn refresh_active_path(&mut self, now: &DateTime<Utc>) -> io::Result<bool> {
|
||||
let next_path = active_path_for(&self.dir, &self.base_name, self.options.rotation, now);
|
||||
if next_path == self.current_path {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
self.close_current()?;
|
||||
self.current_path = next_path;
|
||||
self.open_current()?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn rotate_for_size(&mut self, now: &DateTime<Utc>) -> io::Result<()> {
|
||||
self.close_current()?;
|
||||
if self.current_path.exists() {
|
||||
let archive_path = self.archive_path(now);
|
||||
fs::rename(&self.current_path, archive_path)?;
|
||||
}
|
||||
self.open_current()
|
||||
}
|
||||
|
||||
fn archive_path(&self, now: &DateTime<Utc>) -> PathBuf {
|
||||
let file_name = self
|
||||
.current_path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or(&self.base_name);
|
||||
let stamp = now.format("%Y%m%d%H%M%S");
|
||||
for seq in 0..1000 {
|
||||
let candidate = self.dir.join(format!("{file_name}.{stamp}.{seq}"));
|
||||
if !candidate.exists() {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
self.dir.join(format!("{file_name}.{stamp}.overflow"))
|
||||
}
|
||||
|
||||
fn open_current(&mut self) -> io::Result<()> {
|
||||
let (file, current_size) = open_append_file(&self.current_path)?;
|
||||
self.file = Some(file);
|
||||
self.current_size = current_size;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn close_current(&mut self) -> io::Result<()> {
|
||||
if let Some(mut file) = self.file.take() {
|
||||
file.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn should_rotate_for_size(&self, incoming_len: usize) -> bool {
|
||||
self.options.max_size_bytes > 0
|
||||
&& self.current_size > 0
|
||||
&& self.current_size.saturating_add(incoming_len as u64) > self.options.max_size_bytes
|
||||
}
|
||||
|
||||
fn cleanup_due(&self, now: &DateTime<Utc>) -> bool {
|
||||
self.options.max_age_secs > 0
|
||||
&& now.signed_duration_since(self.last_cleanup)
|
||||
>= ChronoDuration::seconds(CLEANUP_INTERVAL_SECS)
|
||||
}
|
||||
|
||||
fn cleanup(&mut self, now: &DateTime<Utc>) {
|
||||
self.last_cleanup = now.clone();
|
||||
let Ok(entries) = fs::read_dir(&self.dir) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut candidates = Vec::new();
|
||||
let prefix = format!("{}.", self.base_name);
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let Ok(file_type) = entry.file_type() else {
|
||||
continue;
|
||||
};
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let is_current = path == self.current_path;
|
||||
let Some(name) = entry.file_name().to_str().map(|name| name.to_string()) else {
|
||||
continue;
|
||||
};
|
||||
if !is_current && !name.starts_with(&prefix) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Ok(metadata) = entry.metadata() else {
|
||||
continue;
|
||||
};
|
||||
let modified = metadata.modified().unwrap_or(UNIX_EPOCH);
|
||||
candidates.push(LogFileCandidate {
|
||||
path,
|
||||
modified,
|
||||
is_current,
|
||||
});
|
||||
}
|
||||
|
||||
if self.options.max_age_secs > 0 {
|
||||
let cutoff = system_time_from_utc(now)
|
||||
.checked_sub(Duration::from_secs(self.options.max_age_secs))
|
||||
.unwrap_or(UNIX_EPOCH);
|
||||
candidates.retain(|candidate| {
|
||||
if candidate.is_current || candidate.modified >= cutoff {
|
||||
true
|
||||
} else {
|
||||
let _ = fs::remove_file(&candidate.path);
|
||||
false
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if self.options.max_files > 0 && candidates.len() > self.options.max_files {
|
||||
let mut archives: Vec<_> = candidates
|
||||
.into_iter()
|
||||
.filter(|candidate| !candidate.is_current)
|
||||
.collect();
|
||||
archives.sort_by_key(|candidate| candidate.modified);
|
||||
let mut total = archives.len() + 1;
|
||||
for candidate in archives {
|
||||
if total <= self.options.max_files {
|
||||
break;
|
||||
}
|
||||
let _ = fs::remove_file(candidate.path);
|
||||
total -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for BoundedFileAppender {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
let now = self.now();
|
||||
let rotated_by_time = self.refresh_active_path(&now)?;
|
||||
if self.should_rotate_for_size(buf.len()) {
|
||||
self.rotate_for_size(&now)?;
|
||||
self.cleanup(&now);
|
||||
} else if rotated_by_time || self.cleanup_due(&now) {
|
||||
self.cleanup(&now);
|
||||
}
|
||||
|
||||
let Some(file) = self.file.as_mut() else {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"bounded log file is not open",
|
||||
));
|
||||
};
|
||||
file.write_all(buf)?;
|
||||
self.current_size = self.current_size.saturating_add(buf.len() as u64);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
if let Some(file) = self.file.as_mut() {
|
||||
file.flush()
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct LogFileCandidate {
|
||||
path: PathBuf,
|
||||
modified: SystemTime,
|
||||
is_current: bool,
|
||||
}
|
||||
|
||||
fn open_append_file(path: &Path) -> io::Result<(File, u64)> {
|
||||
let mut options = OpenOptions::new();
|
||||
options.create(true).append(true);
|
||||
|
||||
let file = match options.open(path) {
|
||||
Ok(file) => file,
|
||||
Err(error) => {
|
||||
let Some(parent) = path
|
||||
.parent()
|
||||
.filter(|parent| !parent.as_os_str().is_empty())
|
||||
else {
|
||||
return Err(error);
|
||||
};
|
||||
fs::create_dir_all(parent)?;
|
||||
options.open(path)?
|
||||
}
|
||||
};
|
||||
let current_size = file.metadata()?.len();
|
||||
Ok((file, current_size))
|
||||
}
|
||||
|
||||
fn active_path_for(
|
||||
dir: &Path,
|
||||
base_name: &str,
|
||||
rotation: LogRotation,
|
||||
now: &DateTime<Utc>,
|
||||
) -> PathBuf {
|
||||
match rotation {
|
||||
LogRotation::Never => dir.join(base_name),
|
||||
LogRotation::Minutely | LogRotation::Hourly | LogRotation::Daily | LogRotation::Weekly => {
|
||||
dir.join(format!("{base_name}.{}", period_suffix_for(rotation, now)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn period_suffix_for(rotation: LogRotation, now: &DateTime<Utc>) -> String {
|
||||
match rotation {
|
||||
LogRotation::Never | LogRotation::Daily => now.format("%Y-%m-%d").to_string(),
|
||||
LogRotation::Hourly => now.format("%Y-%m-%d-%H").to_string(),
|
||||
LogRotation::Minutely => now.format("%Y-%m-%d-%H-%M").to_string(),
|
||||
LogRotation::Weekly => {
|
||||
let days_since_sunday = now.weekday().num_days_from_sunday() as i64;
|
||||
let week_start = now.date_naive() - ChronoDuration::days(days_since_sunday);
|
||||
week_start.format("%Y-%m-%d").to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn system_time_from_utc(now: &DateTime<Utc>) -> SystemTime {
|
||||
let duration = Duration::new(now.timestamp().unsigned_abs(), now.timestamp_subsec_nanos());
|
||||
if now.timestamp() >= 0 {
|
||||
UNIX_EPOCH + duration
|
||||
} else {
|
||||
UNIX_EPOCH - duration
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::io::Write;
|
||||
|
||||
use tempfile::tempdir;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn fixed_now() -> DateTime<Utc> {
|
||||
DateTime::<Utc>::from(UNIX_EPOCH + Duration::from_secs(10))
|
||||
}
|
||||
|
||||
fn options(path: PathBuf) -> FileLogOptions {
|
||||
FileLogOptions {
|
||||
path: path.to_string_lossy().to_string(),
|
||||
rotation: LogRotation::Never,
|
||||
max_size_bytes: 0,
|
||||
max_files: 0,
|
||||
max_age_secs: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn matching_logs(dir: &Path) -> Vec<PathBuf> {
|
||||
let mut files: Vec<_> = fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.map(|entry| entry.path())
|
||||
.filter(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.map(|name| name.starts_with("telemt.log"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
files
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_rotation_keeps_latest_write_in_active_file() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("telemt.log");
|
||||
let mut options = options(path.clone());
|
||||
options.max_size_bytes = 6;
|
||||
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
appender.write_all(b"abc\n").unwrap();
|
||||
appender.write_all(b"def\n").unwrap();
|
||||
appender.flush().unwrap();
|
||||
|
||||
assert_eq!(fs::read_to_string(path).unwrap(), "def\n");
|
||||
assert_eq!(matching_logs(dir.path()).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn max_files_retention_removes_oldest_archives() {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("telemt.log");
|
||||
let mut options = options(path);
|
||||
options.max_size_bytes = 4;
|
||||
options.max_files = 2;
|
||||
|
||||
let mut appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
for line in [b"aa\n", b"bb\n", b"cc\n", b"dd\n"] {
|
||||
appender.write_all(line).unwrap();
|
||||
}
|
||||
appender.flush().unwrap();
|
||||
|
||||
assert!(matching_logs(dir.path()).len() <= 2);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn max_age_retention_removes_old_archives() {
|
||||
use std::ffi::CString;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("telemt.log");
|
||||
let old_archive = dir.path().join("telemt.log.20000101000000.0");
|
||||
fs::write(&old_archive, "old").unwrap();
|
||||
|
||||
let c_path = CString::new(old_archive.as_os_str().as_bytes()).unwrap();
|
||||
let times = [
|
||||
libc::timespec {
|
||||
tv_sec: 0,
|
||||
tv_nsec: 0,
|
||||
},
|
||||
libc::timespec {
|
||||
tv_sec: 0,
|
||||
tv_nsec: 0,
|
||||
},
|
||||
];
|
||||
let rc = unsafe { libc::utimensat(libc::AT_FDCWD, c_path.as_ptr(), times.as_ptr(), 0) };
|
||||
assert_eq!(rc, 0);
|
||||
|
||||
let mut options = options(path);
|
||||
options.max_age_secs = 1;
|
||||
let _appender = BoundedFileAppender::with_now(options, Box::new(fixed_now)).unwrap();
|
||||
|
||||
assert!(!old_archive.exists());
|
||||
}
|
||||
}
|
||||
100
src/logging/tests.rs
Normal file
100
src/logging/tests.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_cli_options_default() {
|
||||
let args: Vec<String> = vec![];
|
||||
let options = parse_log_cli_options(&args).unwrap();
|
||||
assert_eq!(
|
||||
resolve_log_destination(&LoggingConfig::default(), &options).unwrap(),
|
||||
LogDestination::Stderr
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_cli_options_file() {
|
||||
let args = vec!["--log-file".to_string(), "/var/log/telemt.log".to_string()];
|
||||
let options = parse_log_cli_options(&args).unwrap();
|
||||
match resolve_log_destination(&LoggingConfig::default(), &options).unwrap() {
|
||||
LogDestination::File { options } => {
|
||||
assert_eq!(options.path, "/var/log/telemt.log");
|
||||
assert_eq!(options.rotation, LogRotation::Never);
|
||||
}
|
||||
_ => panic!("Expected File destination"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_cli_options_file_daily() {
|
||||
let args = vec!["--log-file-daily=/var/log/telemt".to_string()];
|
||||
let options = parse_log_cli_options(&args).unwrap();
|
||||
match resolve_log_destination(&LoggingConfig::default(), &options).unwrap() {
|
||||
LogDestination::File { options } => {
|
||||
assert_eq!(options.path, "/var/log/telemt");
|
||||
assert_eq!(options.rotation, LogRotation::Daily);
|
||||
}
|
||||
_ => panic!("Expected File destination"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_cli_options_bounds() {
|
||||
let args = vec![
|
||||
"--log-file=/var/log/telemt.log".to_string(),
|
||||
"--log-rotation=hourly".to_string(),
|
||||
"--log-max-size-bytes=1024".to_string(),
|
||||
"--log-max-files=3".to_string(),
|
||||
"--log-max-age-secs=60".to_string(),
|
||||
];
|
||||
let options = parse_log_cli_options(&args).unwrap();
|
||||
match resolve_log_destination(&LoggingConfig::default(), &options).unwrap() {
|
||||
LogDestination::File { options } => {
|
||||
assert_eq!(options.rotation, LogRotation::Hourly);
|
||||
assert_eq!(options.max_size_bytes, 1024);
|
||||
assert_eq!(options.max_files, 3);
|
||||
assert_eq!(options.max_age_secs, 60);
|
||||
}
|
||||
_ => panic!("Expected File destination"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_log_cli_options_rejects_bad_rotation() {
|
||||
let args = vec!["--log-rotation=yearly".to_string()];
|
||||
assert!(parse_log_cli_options(&args).is_err());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn test_parse_log_cli_options_syslog() {
|
||||
let args = vec!["--syslog".to_string()];
|
||||
let options = parse_log_cli_options(&args).unwrap();
|
||||
assert_eq!(
|
||||
resolve_log_destination(&LoggingConfig::default(), &options).unwrap(),
|
||||
LogDestination::Syslog
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn test_syslog_priority_for_level_mapping() {
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::ERROR),
|
||||
libc::LOG_ERR
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::WARN),
|
||||
libc::LOG_WARNING
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::INFO),
|
||||
libc::LOG_INFO
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::DEBUG),
|
||||
libc::LOG_DEBUG
|
||||
);
|
||||
assert_eq!(
|
||||
syslog_priority_for_level(&tracing::Level::TRACE),
|
||||
libc::LOG_DEBUG
|
||||
);
|
||||
}
|
||||
@@ -8,6 +8,8 @@ use crate::config::ProxyConfig;
|
||||
use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController};
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
use super::generation::RuntimeTaskScope;
|
||||
|
||||
const STARTUP_FALLBACK_AFTER: Duration = Duration::from_secs(80);
|
||||
const RUNTIME_FALLBACK_AFTER: Duration = Duration::from_secs(6);
|
||||
|
||||
@@ -19,6 +21,7 @@ pub(crate) async fn configure_admission_gate(
|
||||
admission_tx: &watch::Sender<bool>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
me_ready_rx: watch::Receiver<u64>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
) {
|
||||
if config.general.use_middle_proxy {
|
||||
if me_pool.is_some() || config.general.me2dc_fallback {
|
||||
@@ -64,7 +67,7 @@ pub(crate) async fn configure_admission_gate(
|
||||
let mut config_rx_gate = config_rx.clone();
|
||||
let mut me_ready_rx_gate = me_ready_rx;
|
||||
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
let mut gate_open = initial_gate_open;
|
||||
let mut route_mode = initial_route_mode;
|
||||
let mut ready_observed = initial_ready;
|
||||
|
||||
@@ -147,7 +147,7 @@ pub(crate) async fn run_startup_connectivity(
|
||||
.any(|r| r.rtt_ms.is_some());
|
||||
|
||||
if upstream_result.both_available {
|
||||
if prefer_ipv6 {
|
||||
if upstream_result.prefer_ipv6 {
|
||||
info!(" IPv6 in use / IPv4 is fallback");
|
||||
} else {
|
||||
info!(" IPv4 in use / IPv6 is fallback");
|
||||
|
||||
435
src/maestro/generation.rs
Normal file
435
src/maestro/generation.rs
Normal file
@@ -0,0 +1,435 @@
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
#[cfg(test)]
|
||||
use crate::proxy::route_mode::RelayRouteMode;
|
||||
use crate::proxy::route_mode::RouteRuntimeController;
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::tls_front::TlsFrontCache;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
const SESSION_STOP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const BACKGROUND_STOP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const SESSION_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
|
||||
const SESSION_REGISTRATION_COUNT: usize = SESSION_ADMISSION_CLOSED - 1;
|
||||
|
||||
struct SessionAdmission {
|
||||
state: AtomicUsize,
|
||||
}
|
||||
|
||||
struct SessionRegistration<'a> {
|
||||
admission: &'a SessionAdmission,
|
||||
}
|
||||
|
||||
impl SessionAdmission {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
state: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn try_register(&self) -> Option<SessionRegistration<'_>> {
|
||||
let mut state = self.state.load(Ordering::Acquire);
|
||||
loop {
|
||||
if state & SESSION_ADMISSION_CLOSED != 0
|
||||
|| state & SESSION_REGISTRATION_COUNT == SESSION_REGISTRATION_COUNT
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match self.state.compare_exchange_weak(
|
||||
state,
|
||||
state + 1,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
) {
|
||||
Ok(_) => return Some(SessionRegistration { admission: self }),
|
||||
Err(observed) => state = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn close(&self) {
|
||||
self.state
|
||||
.fetch_or(SESSION_ADMISSION_CLOSED, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
fn reopen(&self) {
|
||||
self.state
|
||||
.fetch_and(!SESSION_ADMISSION_CLOSED, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
async fn wait_for_registrations(&self) {
|
||||
while self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SessionRegistration<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.admission.state.fetch_sub(1, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
/// Process-visible control-plane receivers for one active runtime generation.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RuntimeWatchState {
|
||||
pub(crate) generation_id: u64,
|
||||
pub(crate) config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
pub(crate) admission_rx: watch::Receiver<bool>,
|
||||
}
|
||||
|
||||
/// Cancellation and join ownership for one generation's background tasks.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RuntimeTaskScope {
|
||||
tracker: TaskTracker,
|
||||
cancel: CancellationToken,
|
||||
}
|
||||
|
||||
impl RuntimeTaskScope {
|
||||
/// Creates an open generation-owned task scope.
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
tracker: TaskTracker::new(),
|
||||
cancel: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns one task that is cancelled when the generation stops.
|
||||
pub(crate) fn spawn<F>(&self, future: F)
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let cancel = self.cancel.clone();
|
||||
self.tracker.spawn(async move {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns the cancellation signal shared by generation-owned controllers.
|
||||
pub(crate) fn cancellation_token(&self) -> CancellationToken {
|
||||
self.cancel.clone()
|
||||
}
|
||||
|
||||
/// Cancels the scope and waits within the bounded background-task budget.
|
||||
pub(crate) async fn stop(&self) {
|
||||
self.cancel.cancel();
|
||||
self.tracker.close();
|
||||
let _ = tokio::time::timeout(BACKGROUND_STOP_TIMEOUT, self.tracker.wait()).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Runtime-owned data plane and control-plane dependencies for one generation.
|
||||
pub(crate) struct RuntimeGeneration {
|
||||
pub(crate) id: u64,
|
||||
pub(crate) config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
pub(crate) admission_rx: watch::Receiver<bool>,
|
||||
pub(crate) stats: Arc<Stats>,
|
||||
pub(crate) upstream_manager: Arc<UpstreamManager>,
|
||||
pub(crate) replay_checker: Arc<ReplayChecker>,
|
||||
pub(crate) buffer_pool: Arc<BufferPool>,
|
||||
pub(crate) rng: Arc<SecureRandom>,
|
||||
pub(crate) me_pool: Option<Arc<MePool>>,
|
||||
pub(crate) me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
pub(crate) route_runtime: Arc<RouteRuntimeController>,
|
||||
pub(crate) tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
pub(crate) ip_tracker: Arc<UserIpTracker>,
|
||||
pub(crate) beobachten: Arc<BeobachtenStore>,
|
||||
pub(crate) proxy_shared: Arc<ProxySharedState>,
|
||||
pub(crate) max_connections: Arc<Semaphore>,
|
||||
background_tasks: RuntimeTaskScope,
|
||||
sessions: TaskTracker,
|
||||
session_cancel: CancellationToken,
|
||||
session_admission: SessionAdmission,
|
||||
}
|
||||
|
||||
impl RuntimeGeneration {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Builds one fully owned runtime generation.
|
||||
pub(crate) fn new(
|
||||
id: u64,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
admission_rx: watch::Receiver<bool>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
proxy_shared: Arc<ProxySharedState>,
|
||||
max_connections: Arc<Semaphore>,
|
||||
background_tasks: RuntimeTaskScope,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
id,
|
||||
config_rx,
|
||||
admission_rx,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
proxy_shared,
|
||||
max_connections,
|
||||
background_tasks,
|
||||
sessions: TaskTracker::new(),
|
||||
session_cancel: CancellationToken::new(),
|
||||
session_admission: SessionAdmission::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the latest hot-reloaded configuration for this generation.
|
||||
pub(crate) fn config(&self) -> Arc<ProxyConfig> {
|
||||
self.config_rx.borrow().clone()
|
||||
}
|
||||
|
||||
/// Returns receivers used by process-scoped observers of this generation.
|
||||
pub(crate) fn watch_state(&self) -> RuntimeWatchState {
|
||||
RuntimeWatchState {
|
||||
generation_id: self.id,
|
||||
config_rx: self.config_rx.clone(),
|
||||
admission_rx: self.admission_rx.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the initial or asynchronously published Middle-End pool.
|
||||
pub(crate) async fn current_me_pool(&self) -> Option<Arc<MePool>> {
|
||||
if let Some(pool) = &self.me_pool {
|
||||
return Some(pool.clone());
|
||||
}
|
||||
self.me_pool_runtime.read().await.clone()
|
||||
}
|
||||
|
||||
/// Registers a session only while admission remains open.
|
||||
pub(crate) fn spawn_session<F>(&self, future: F) -> bool
|
||||
where
|
||||
F: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let Some(_registration) = self.session_admission.try_register() else {
|
||||
return false;
|
||||
};
|
||||
let cancel = self.session_cancel.clone();
|
||||
self.sessions.spawn(async move {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {}
|
||||
_ = future => {}
|
||||
}
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Closes admission while preserving already registered sessions.
|
||||
pub(crate) fn stop_accepting_sessions(&self) {
|
||||
self.session_admission.close();
|
||||
}
|
||||
|
||||
/// Reopens admission after a candidate activation rolls back.
|
||||
pub(crate) fn resume_accepting_sessions(&self) {
|
||||
self.session_admission.reopen();
|
||||
}
|
||||
|
||||
/// Waits for registered sessions and cancels them when the deadline expires.
|
||||
pub(crate) async fn drain_sessions(&self, timeout: Duration) -> bool {
|
||||
self.stop_accepting_sessions();
|
||||
self.session_admission.wait_for_registrations().await;
|
||||
self.sessions.close();
|
||||
if tokio::time::timeout(timeout, self.sessions.wait())
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
self.stop_sessions().await;
|
||||
false
|
||||
}
|
||||
|
||||
/// Cancels all sessions and waits within the bounded session-stop budget.
|
||||
pub(crate) async fn stop_sessions(&self) {
|
||||
self.stop_accepting_sessions();
|
||||
self.session_admission.wait_for_registrations().await;
|
||||
self.session_cancel.cancel();
|
||||
self.sessions.close();
|
||||
let _ = tokio::time::timeout(SESSION_STOP_TIMEOUT, self.sessions.wait()).await;
|
||||
}
|
||||
|
||||
/// Stops all background tasks owned by this generation.
|
||||
pub(crate) async fn stop_background_tasks(&self) {
|
||||
self.background_tasks.stop().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Builds a lightweight runtime generation without network startup tasks.
|
||||
pub(super) fn test_runtime_generation(id: u64, config: ProxyConfig) -> Arc<RuntimeGeneration> {
|
||||
let (config_tx, config_rx) = watch::channel(Arc::new(config.clone()));
|
||||
let (_admission_tx, admission_rx) = watch::channel(true);
|
||||
let stats = Arc::new(Stats::new());
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
config.upstreams,
|
||||
config.general.upstream_connect_retry_attempts,
|
||||
config.general.upstream_connect_retry_backoff_ms,
|
||||
config.general.upstream_connect_budget_ms,
|
||||
config.general.tg_connect,
|
||||
config.general.upstream_unhealthy_fail_threshold,
|
||||
config.general.upstream_connect_failfast_hard_errors,
|
||||
stats.clone(),
|
||||
));
|
||||
let _config_tx = config_tx;
|
||||
RuntimeGeneration::new(
|
||||
id,
|
||||
config_rx,
|
||||
admission_rx,
|
||||
stats,
|
||||
upstream_manager,
|
||||
Arc::new(ReplayChecker::new(128, Duration::from_secs(60))),
|
||||
Arc::new(BufferPool::with_config(4096, 16)),
|
||||
Arc::new(SecureRandom::new()),
|
||||
None,
|
||||
Arc::new(RwLock::new(None)),
|
||||
Arc::new(RouteRuntimeController::new(RelayRouteMode::Direct)),
|
||||
None,
|
||||
Arc::new(UserIpTracker::new()),
|
||||
Arc::new(BeobachtenStore::new()),
|
||||
ProxySharedState::new(),
|
||||
Arc::new(Semaphore::new(64)),
|
||||
RuntimeTaskScope::new(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn stop_sessions_cancels_tracked_future() {
|
||||
let generation = test_runtime_generation(1, ProxyConfig::default());
|
||||
let started = Arc::new(tokio::sync::Notify::new());
|
||||
let dropped = Arc::new(tokio::sync::Notify::new());
|
||||
let started_task = started.clone();
|
||||
let dropped_task = dropped.clone();
|
||||
assert!(generation.spawn_session(async move {
|
||||
struct DropSignal(Arc<tokio::sync::Notify>);
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
let _drop_signal = DropSignal(dropped_task);
|
||||
started_task.notify_one();
|
||||
std::future::pending::<()>().await;
|
||||
}));
|
||||
started.notified().await;
|
||||
|
||||
generation.stop_sessions().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!generation.spawn_session(async {}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_task_scope_joins_cancelled_background_task() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
scope.spawn(std::future::pending());
|
||||
tokio::time::timeout(Duration::from_secs(1), scope.stop())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_admission_waits_for_registration_started_before_cutover() {
|
||||
let admission = Arc::new(SessionAdmission::new());
|
||||
let registration = admission.try_register().unwrap();
|
||||
admission.close();
|
||||
assert!(admission.try_register().is_none());
|
||||
|
||||
let wait_admission = admission.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
wait_admission.wait_for_registrations().await;
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!waiter.is_finished());
|
||||
|
||||
drop(registration);
|
||||
tokio::time::timeout(Duration::from_secs(1), waiter)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
admission.reopen();
|
||||
assert!(admission.try_register().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cutover_never_leaves_late_session_registrations() {
|
||||
const ATTEMPTS: usize = 10_000;
|
||||
|
||||
let admission = Arc::new(SessionAdmission::new());
|
||||
let tracker = TaskTracker::new();
|
||||
let cancel = CancellationToken::new();
|
||||
let start = Arc::new(Barrier::new(ATTEMPTS + 1));
|
||||
let live = Arc::new(AtomicUsize::new(0));
|
||||
let mut attempts = tokio::task::JoinSet::new();
|
||||
|
||||
for _ in 0..ATTEMPTS {
|
||||
let admission = admission.clone();
|
||||
let tracker = tracker.clone();
|
||||
let cancel = cancel.clone();
|
||||
let start = start.clone();
|
||||
let live = live.clone();
|
||||
attempts.spawn(async move {
|
||||
start.wait().await;
|
||||
let Some(_registration) = admission.try_register() else {
|
||||
return;
|
||||
};
|
||||
tracker.spawn(async move {
|
||||
live.fetch_add(1, Ordering::AcqRel);
|
||||
cancel.cancelled().await;
|
||||
live.fetch_sub(1, Ordering::AcqRel);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
start.wait().await;
|
||||
admission.close();
|
||||
admission.wait_for_registrations().await;
|
||||
tracker.close();
|
||||
cancel.cancel();
|
||||
while attempts.join_next().await.is_some() {}
|
||||
tokio::time::timeout(Duration::from_secs(1), tracker.wait())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(live.load(Ordering::Acquire), 0);
|
||||
assert!(admission.try_register().is_none());
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::cli;
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::logging::LogDestination;
|
||||
use crate::logging::LogCliOptions;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::{
|
||||
ProxyConfigData, fetch_proxy_config_with_raw_via_upstream, load_proxy_config_cache,
|
||||
@@ -113,7 +113,7 @@ pub(crate) struct CliArgs {
|
||||
pub data_path: Option<PathBuf>,
|
||||
pub silent: bool,
|
||||
pub log_level: Option<String>,
|
||||
pub log_destination: LogDestination,
|
||||
pub log_cli_options: LogCliOptions,
|
||||
}
|
||||
|
||||
pub(crate) fn parse_cli() -> CliArgs {
|
||||
@@ -125,8 +125,13 @@ pub(crate) fn parse_cli() -> CliArgs {
|
||||
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
|
||||
// Parse log destination
|
||||
let log_destination = crate::logging::parse_log_destination(&args);
|
||||
let log_cli_options = match crate::logging::parse_log_cli_options(&args) {
|
||||
Ok(options) => options,
|
||||
Err(error) => {
|
||||
eprintln!("[telemt] {error}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
|
||||
// Check for --init first (handled before tokio)
|
||||
if let Some(init_opts) = cli::parse_init_args(&args) {
|
||||
@@ -180,6 +185,21 @@ pub(crate) fn parse_cli() -> CliArgs {
|
||||
s if s.starts_with("--log-level=") => {
|
||||
log_level = Some(s.trim_start_matches("--log-level=").to_string());
|
||||
}
|
||||
"--log-file" | "--log-file-daily" => {
|
||||
i += 1;
|
||||
}
|
||||
s if s.starts_with("--log-file=") || s.starts_with("--log-file-daily=") => {}
|
||||
"--log-rotation"
|
||||
| "--log-max-size-bytes"
|
||||
| "--log-max-files"
|
||||
| "--log-max-age-secs" => {
|
||||
i += 1;
|
||||
}
|
||||
s if s.starts_with("--log-rotation=")
|
||||
|| s.starts_with("--log-max-size-bytes=")
|
||||
|| s.starts_with("--log-max-files=")
|
||||
|| s.starts_with("--log-max-age-secs=") => {}
|
||||
"--syslog" => {}
|
||||
"--help" | "-h" => {
|
||||
print_help();
|
||||
std::process::exit(0);
|
||||
@@ -192,7 +212,8 @@ pub(crate) fn parse_cli() -> CliArgs {
|
||||
"--daemon" | "-d" | "--foreground" | "-f" => {}
|
||||
s if s.starts_with("--pid-file") => {
|
||||
if !s.contains('=') {
|
||||
i += 1; // skip value
|
||||
// Skip the pid-file value consumed by daemon argument parsing.
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
s if s.starts_with("--run-as-user") => {
|
||||
@@ -224,7 +245,7 @@ pub(crate) fn parse_cli() -> CliArgs {
|
||||
data_path,
|
||||
silent,
|
||||
log_level,
|
||||
log_destination,
|
||||
log_cli_options,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +275,10 @@ fn print_help() {
|
||||
eprintln!("Logging options:");
|
||||
eprintln!(" --log-file <PATH> Log to file (default: stderr)");
|
||||
eprintln!(" --log-file-daily <PATH> Log to file with daily rotation");
|
||||
eprintln!(" --log-rotation <MODE> never|minutely|hourly|daily|weekly");
|
||||
eprintln!(" --log-max-size-bytes N Rotate file logs when active file exceeds N bytes");
|
||||
eprintln!(" --log-max-files N Keep at most N matching file logs (0 disables)");
|
||||
eprintln!(" --log-max-age-secs N Remove rotated file logs older than N seconds");
|
||||
#[cfg(unix)]
|
||||
eprintln!(" --syslog Log to syslog (Unix only)");
|
||||
eprintln!();
|
||||
|
||||
@@ -3,35 +3,33 @@ use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::net::TcpListener;
|
||||
#[cfg(unix)]
|
||||
use tokio::net::UnixListener;
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use crate::config::{ProxyConfig, RstOnCloseMode};
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::proxy::ClientHandler;
|
||||
use crate::proxy::route_mode::RouteRuntimeController;
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::startup::{COMPONENT_LISTENERS_BIND, StartupTracker};
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::tls_front::TlsFrontCache;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use crate::transport::socket::set_linger_zero;
|
||||
use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes};
|
||||
use crate::transport::{ListenOptions, create_listener, find_listener_processes};
|
||||
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::helpers::{
|
||||
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
|
||||
print_proxy_links,
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
mod unix;
|
||||
#[cfg(unix)]
|
||||
pub(crate) use unix::spawn_unix_accept_loop;
|
||||
|
||||
pub(crate) struct BoundListeners {
|
||||
pub(crate) listeners: Vec<(TcpListener, bool)>,
|
||||
pub(crate) has_unix_listener: bool,
|
||||
#[cfg(unix)]
|
||||
pub(crate) unix_listener: Option<UnixListener>,
|
||||
}
|
||||
|
||||
fn listener_port_or_legacy(listener: &crate::config::ListenerConfig, config: &ProxyConfig) -> u16 {
|
||||
@@ -47,6 +45,10 @@ fn default_link_port(config: &ProxyConfig) -> u16 {
|
||||
.unwrap_or(config.server.port)
|
||||
}
|
||||
|
||||
fn mss_segment_multiplier(client_mss: u16) -> u16 {
|
||||
1460u16.div_ceil(client_mss)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn bind_listeners(
|
||||
config: &Arc<ProxyConfig>,
|
||||
@@ -55,21 +57,6 @@ pub(crate) async fn bind_listeners(
|
||||
detected_ip_v4: Option<IpAddr>,
|
||||
detected_ip_v6: Option<IpAddr>,
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
admission_rx: watch::Receiver<bool>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
max_connections: Arc<Semaphore>,
|
||||
) -> Result<BoundListeners, Box<dyn Error>> {
|
||||
startup_tracker
|
||||
.start_component(
|
||||
@@ -90,10 +77,22 @@ pub(crate) async fn bind_listeners(
|
||||
warn!(%addr, "Skipping IPv6 listener: IPv6 disabled by [network]");
|
||||
continue;
|
||||
}
|
||||
let client_mss = match listener_conf.effective_client_mss(&config.server) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
%addr,
|
||||
error = %error,
|
||||
"Invalid listener client MSS after config validation; using kernel default"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
let options = ListenOptions {
|
||||
reuse_port: listener_conf.reuse_allow,
|
||||
ipv6_only: listener_conf.ip.is_ipv6(),
|
||||
backlog: config.server.listen_backlog,
|
||||
client_mss,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -101,6 +100,14 @@ pub(crate) async fn bind_listeners(
|
||||
Ok(socket) => {
|
||||
let listener = TcpListener::from_std(socket.into())?;
|
||||
info!("Listening on {}", addr);
|
||||
if let Some(client_mss) = client_mss {
|
||||
info!(
|
||||
%addr,
|
||||
client_mss,
|
||||
segment_multiplier = mss_segment_multiplier(client_mss),
|
||||
"Client-facing TCP MSS configured"
|
||||
);
|
||||
}
|
||||
let listener_proxy_protocol = listener_conf
|
||||
.proxy_protocol
|
||||
.unwrap_or(config.server.proxy_protocol);
|
||||
@@ -194,7 +201,8 @@ pub(crate) async fn bind_listeners(
|
||||
print_proxy_links(&host, port, config);
|
||||
}
|
||||
|
||||
let mut has_unix_listener = false;
|
||||
#[cfg(unix)]
|
||||
let mut unix_listener_out = None;
|
||||
#[cfg(unix)]
|
||||
if let Some(ref unix_path) = config.server.listen_unix_sock {
|
||||
let _ = tokio::fs::remove_file(unix_path).await;
|
||||
@@ -227,123 +235,14 @@ pub(crate) async fn bind_listeners(
|
||||
info!("Listening on unix:{}", unix_path);
|
||||
}
|
||||
|
||||
has_unix_listener = true;
|
||||
|
||||
let mut config_rx_unix: watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
||||
let admission_rx_unix = admission_rx.clone();
|
||||
let stats = stats.clone();
|
||||
let upstream_manager = upstream_manager.clone();
|
||||
let replay_checker = replay_checker.clone();
|
||||
let buffer_pool = buffer_pool.clone();
|
||||
let rng = rng.clone();
|
||||
let me_pool = me_pool.clone();
|
||||
let me_pool_runtime = me_pool_runtime.clone();
|
||||
let route_runtime = route_runtime.clone();
|
||||
let tls_cache = tls_cache.clone();
|
||||
let ip_tracker = ip_tracker.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared = shared.clone();
|
||||
let max_connections_unix = max_connections.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let unix_conn_counter = Arc::new(std::sync::atomic::AtomicU64::new(1));
|
||||
|
||||
loop {
|
||||
match unix_listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
if !*admission_rx_unix.borrow() {
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let accept_permit_timeout_ms =
|
||||
config_rx_unix.borrow().server.accept_permit_timeout_ms;
|
||||
let permit = if accept_permit_timeout_ms == 0 {
|
||||
match max_connections_unix.clone().acquire_owned().await {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => {
|
||||
error!("Connection limiter is closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match tokio::time::timeout(
|
||||
Duration::from_millis(accept_permit_timeout_ms),
|
||||
max_connections_unix.clone().acquire_owned(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(_)) => {
|
||||
error!("Connection limiter is closed");
|
||||
break;
|
||||
}
|
||||
Err(_) => {
|
||||
stats.increment_accept_permit_timeout_total();
|
||||
debug!(
|
||||
timeout_ms = accept_permit_timeout_ms,
|
||||
"Dropping accepted unix connection: permit wait timeout"
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
let conn_id =
|
||||
unix_conn_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let fake_peer =
|
||||
SocketAddr::from(([127, 0, 0, 1], (conn_id % 65535) as u16));
|
||||
|
||||
let config = config_rx_unix.borrow_and_update().clone();
|
||||
let stats = stats.clone();
|
||||
let upstream_manager = upstream_manager.clone();
|
||||
let replay_checker = replay_checker.clone();
|
||||
let buffer_pool = buffer_pool.clone();
|
||||
let rng = rng.clone();
|
||||
let me_pool = me_pool.clone();
|
||||
let me_pool_runtime = me_pool_runtime.clone();
|
||||
let route_runtime = route_runtime.clone();
|
||||
let tls_cache = tls_cache.clone();
|
||||
let ip_tracker = ip_tracker.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared = shared.clone();
|
||||
let proxy_protocol_enabled = config.server.proxy_protocol;
|
||||
|
||||
tokio::spawn(async move {
|
||||
let _permit = permit;
|
||||
if let Err(e) =
|
||||
crate::proxy::client::handle_client_stream_with_shared_and_pool_runtime(
|
||||
stream,
|
||||
fake_peer,
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
Some(me_pool_runtime),
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
shared,
|
||||
proxy_protocol_enabled,
|
||||
)
|
||||
.await
|
||||
{
|
||||
debug!(error = %e, "Unix socket connection error");
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Unix socket accept error: {}", e);
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
unix_listener_out = Some(unix_listener);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
let has_unix_listener = unix_listener_out.is_some();
|
||||
#[cfg(not(unix))]
|
||||
let has_unix_listener = false;
|
||||
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_LISTENERS_BIND,
|
||||
@@ -357,51 +256,25 @@ pub(crate) async fn bind_listeners(
|
||||
|
||||
Ok(BoundListeners {
|
||||
listeners,
|
||||
has_unix_listener,
|
||||
#[cfg(unix)]
|
||||
unix_listener: unix_listener_out,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn spawn_tcp_accept_loops(
|
||||
listeners: Vec<(TcpListener, bool)>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
admission_rx: watch::Receiver<bool>,
|
||||
stats: Arc<Stats>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
replay_checker: Arc<ReplayChecker>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
route_runtime: Arc<RouteRuntimeController>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
max_connections: Arc<Semaphore>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
) {
|
||||
for (listener, listener_proxy_protocol) in listeners {
|
||||
let mut config_rx: watch::Receiver<Arc<ProxyConfig>> = config_rx.clone();
|
||||
let admission_rx_tcp = admission_rx.clone();
|
||||
let stats = stats.clone();
|
||||
let upstream_manager = upstream_manager.clone();
|
||||
let replay_checker = replay_checker.clone();
|
||||
let buffer_pool = buffer_pool.clone();
|
||||
let rng = rng.clone();
|
||||
let me_pool = me_pool.clone();
|
||||
let me_pool_runtime = me_pool_runtime.clone();
|
||||
let route_runtime = route_runtime.clone();
|
||||
let tls_cache = tls_cache.clone();
|
||||
let ip_tracker = ip_tracker.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared = shared.clone();
|
||||
let max_connections_tcp = max_connections.clone();
|
||||
let active_runtime = active_runtime.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, peer_addr)) => {
|
||||
let rst_mode = config_rx.borrow().general.rst_on_close;
|
||||
let runtime = active_runtime.load_full();
|
||||
let config = runtime.config();
|
||||
let rst_mode = config.general.rst_on_close;
|
||||
#[cfg(unix)]
|
||||
let raw_fd = {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
@@ -410,15 +283,14 @@ pub(crate) fn spawn_tcp_accept_loops(
|
||||
if matches!(rst_mode, RstOnCloseMode::Errors | RstOnCloseMode::Always) {
|
||||
let _ = set_linger_zero(&stream);
|
||||
}
|
||||
if !*admission_rx_tcp.borrow() {
|
||||
if !*runtime.admission_rx.borrow() {
|
||||
debug!(peer = %peer_addr, "Admission gate closed, dropping connection");
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
let accept_permit_timeout_ms =
|
||||
config_rx.borrow().server.accept_permit_timeout_ms;
|
||||
let accept_permit_timeout_ms = config.server.accept_permit_timeout_ms;
|
||||
let permit = if accept_permit_timeout_ms == 0 {
|
||||
match max_connections_tcp.clone().acquire_owned().await {
|
||||
match runtime.max_connections.clone().acquire_owned().await {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => {
|
||||
error!("Connection limiter is closed");
|
||||
@@ -428,7 +300,7 @@ pub(crate) fn spawn_tcp_accept_loops(
|
||||
} else {
|
||||
match tokio::time::timeout(
|
||||
Duration::from_millis(accept_permit_timeout_ms),
|
||||
max_connections_tcp.clone().acquire_owned(),
|
||||
runtime.max_connections.clone().acquire_owned(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -438,7 +310,7 @@ pub(crate) fn spawn_tcp_accept_loops(
|
||||
break;
|
||||
}
|
||||
Err(_) => {
|
||||
stats.increment_accept_permit_timeout_total();
|
||||
runtime.stats.increment_accept_permit_timeout_total();
|
||||
debug!(
|
||||
peer = %peer_addr,
|
||||
timeout_ms = accept_permit_timeout_ms,
|
||||
@@ -449,24 +321,23 @@ pub(crate) fn spawn_tcp_accept_loops(
|
||||
}
|
||||
}
|
||||
};
|
||||
let config = config_rx.borrow_and_update().clone();
|
||||
let stats = stats.clone();
|
||||
let upstream_manager = upstream_manager.clone();
|
||||
let replay_checker = replay_checker.clone();
|
||||
let buffer_pool = buffer_pool.clone();
|
||||
let rng = rng.clone();
|
||||
let me_pool = me_pool.clone();
|
||||
let me_pool_runtime = me_pool_runtime.clone();
|
||||
let route_runtime = route_runtime.clone();
|
||||
let tls_cache = tls_cache.clone();
|
||||
let ip_tracker = ip_tracker.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared = shared.clone();
|
||||
let stats = runtime.stats.clone();
|
||||
let upstream_manager = runtime.upstream_manager.clone();
|
||||
let replay_checker = runtime.replay_checker.clone();
|
||||
let buffer_pool = runtime.buffer_pool.clone();
|
||||
let rng = runtime.rng.clone();
|
||||
let me_pool = runtime.me_pool.clone();
|
||||
let me_pool_runtime = runtime.me_pool_runtime.clone();
|
||||
let route_runtime = runtime.route_runtime.clone();
|
||||
let tls_cache = runtime.tls_cache.clone();
|
||||
let ip_tracker = runtime.ip_tracker.clone();
|
||||
let beobachten = runtime.beobachten.clone();
|
||||
let shared = runtime.proxy_shared.clone();
|
||||
let proxy_protocol_enabled = listener_proxy_protocol;
|
||||
let real_peer_report = Arc::new(std::sync::Mutex::new(None));
|
||||
let real_peer_report_for_handler = real_peer_report.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let _ = runtime.spawn_session(async move {
|
||||
let _permit = permit;
|
||||
if let Err(e) = ClientHandler::new_with_shared(
|
||||
stream,
|
||||
|
||||
117
src/maestro/listeners/unix.rs
Normal file
117
src/maestro/listeners/unix.rs
Normal file
@@ -0,0 +1,117 @@
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::net::UnixListener;
|
||||
use tracing::{debug, error};
|
||||
|
||||
use super::RuntimeGeneration;
|
||||
|
||||
pub(crate) fn spawn_unix_accept_loop(
|
||||
listener: Option<UnixListener>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
) {
|
||||
let Some(listener) = listener else {
|
||||
return;
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
let connection_counter = AtomicU64::new(1);
|
||||
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, _)) => {
|
||||
let runtime = active_runtime.load_full();
|
||||
if !*runtime.admission_rx.borrow() {
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
|
||||
let config = runtime.config();
|
||||
let timeout_ms = config.server.accept_permit_timeout_ms;
|
||||
let permit = if timeout_ms == 0 {
|
||||
match runtime.max_connections.clone().acquire_owned().await {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => {
|
||||
error!("Connection limiter is closed");
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match tokio::time::timeout(
|
||||
Duration::from_millis(timeout_ms),
|
||||
runtime.max_connections.clone().acquire_owned(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(_)) => {
|
||||
error!("Connection limiter is closed");
|
||||
break;
|
||||
}
|
||||
Err(_) => {
|
||||
runtime.stats.increment_accept_permit_timeout_total();
|
||||
debug!(
|
||||
timeout_ms,
|
||||
"Dropping accepted unix connection: permit wait timeout"
|
||||
);
|
||||
drop(stream);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let connection_id = connection_counter.fetch_add(1, Ordering::Relaxed);
|
||||
let fake_peer =
|
||||
SocketAddr::from(([127, 0, 0, 1], (connection_id % 65535) as u16));
|
||||
let stats = runtime.stats.clone();
|
||||
let upstream_manager = runtime.upstream_manager.clone();
|
||||
let replay_checker = runtime.replay_checker.clone();
|
||||
let buffer_pool = runtime.buffer_pool.clone();
|
||||
let rng = runtime.rng.clone();
|
||||
let me_pool = runtime.me_pool.clone();
|
||||
let me_pool_runtime = runtime.me_pool_runtime.clone();
|
||||
let route_runtime = runtime.route_runtime.clone();
|
||||
let tls_cache = runtime.tls_cache.clone();
|
||||
let ip_tracker = runtime.ip_tracker.clone();
|
||||
let beobachten = runtime.beobachten.clone();
|
||||
let shared = runtime.proxy_shared.clone();
|
||||
let proxy_protocol_enabled = config.server.proxy_protocol;
|
||||
|
||||
let _ = runtime.spawn_session(async move {
|
||||
let _permit = permit;
|
||||
if let Err(error) =
|
||||
crate::proxy::client::handle_client_stream_with_shared_and_pool_runtime(
|
||||
stream,
|
||||
fake_peer,
|
||||
config,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
Some(me_pool_runtime),
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
shared,
|
||||
proxy_protocol_enabled,
|
||||
)
|
||||
.await
|
||||
{
|
||||
debug!(error = %error, "Unix socket connection error");
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(error) => {
|
||||
error!(error = %error, "Unix socket accept error");
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::{RwLock, watch};
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
@@ -17,8 +19,61 @@ use crate::stats::Stats;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
use super::generation::RuntimeTaskScope;
|
||||
use super::helpers::load_startup_proxy_config_snapshot;
|
||||
|
||||
async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
loop {
|
||||
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
||||
match result {
|
||||
Ok(()) => warn!(
|
||||
task = task_name,
|
||||
"Middle-End supervisor task exited unexpectedly, restarting"
|
||||
),
|
||||
Err(error) => {
|
||||
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_me_supervisors(
|
||||
task_scope: RuntimeTaskScope,
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
min_connections: usize,
|
||||
) {
|
||||
let health_pool = pool.clone();
|
||||
let health_rng = rng;
|
||||
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
||||
let pool = health_pool.clone();
|
||||
let rng = health_rng.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
||||
}
|
||||
}));
|
||||
|
||||
let drain_pool = pool.clone();
|
||||
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
||||
let pool = drain_pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
||||
}
|
||||
}));
|
||||
|
||||
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
||||
let pool = pool.clone();
|
||||
async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
pub(crate) async fn initialize_me_pool(
|
||||
use_middle_proxy: bool,
|
||||
config: &ProxyConfig,
|
||||
@@ -30,6 +85,7 @@ pub(crate) async fn initialize_me_pool(
|
||||
stats: Arc<Stats>,
|
||||
api_me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||
me_ready_tx: watch::Sender<u64>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
) -> Option<Arc<MePool>> {
|
||||
if !use_middle_proxy {
|
||||
return None;
|
||||
@@ -52,15 +108,8 @@ pub(crate) async fn initialize_me_pool(
|
||||
.as_ref()
|
||||
.map(|tag| hex::decode(tag).expect("general.ad_tag must be validated before startup"));
|
||||
|
||||
// =============================================================
|
||||
// CRITICAL: Download Telegram proxy-secret (NOT user secret!)
|
||||
//
|
||||
// C MTProxy uses TWO separate secrets:
|
||||
// -S flag = 16-byte user secret for client obfuscation
|
||||
// --aes-pwd = 32-512 byte binary file for ME RPC auth
|
||||
//
|
||||
// proxy-secret is from: https://core.telegram.org/getProxySecret
|
||||
// =============================================================
|
||||
// The Telegram proxy-secret authenticates ME RPC and is distinct from client secrets.
|
||||
// It corresponds to the C MTProxy --aes-pwd input and may be fetched from Telegram.
|
||||
let proxy_secret_path = config.general.proxy_secret_path.as_deref();
|
||||
let pool_size = config.general.middle_proxy_pool_size.max(1);
|
||||
let proxy_secret = loop {
|
||||
@@ -208,6 +257,8 @@ pub(crate) async fn initialize_me_pool(
|
||||
me_nat_probe,
|
||||
None,
|
||||
config.network.stun_servers.clone(),
|
||||
config.network.stun_tcp_fallback,
|
||||
config.network.http_ip_detect_urls.clone(),
|
||||
config.general.stun_nat_probe_concurrency,
|
||||
probe.detected_ipv6,
|
||||
config.timeouts.me_one_retry,
|
||||
@@ -277,6 +328,7 @@ pub(crate) async fn initialize_me_pool(
|
||||
config.general.me_writer_pick_sample_size,
|
||||
config.general.me_socks_kdf_policy,
|
||||
config.general.me_writer_cmd_channel_capacity,
|
||||
config.general.me_writer_byte_budget_bytes,
|
||||
config.general.me_route_channel_capacity,
|
||||
config.general.me_route_backpressure_enabled,
|
||||
config.general.me_route_fairshare_enabled,
|
||||
@@ -316,143 +368,70 @@ pub(crate) async fn initialize_me_pool(
|
||||
let rng_bg = rng.clone();
|
||||
let startup_tracker_bg = startup_tracker.clone();
|
||||
let me_ready_tx_bg = me_ready_tx.clone();
|
||||
let task_scope_bg = task_scope.clone();
|
||||
let retry_limit = if me_init_retry_attempts == 0 {
|
||||
String::from("unlimited")
|
||||
} else {
|
||||
me_init_retry_attempts.to_string()
|
||||
};
|
||||
std::thread::spawn(move || {
|
||||
let runtime = match tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
{
|
||||
Ok(runtime) => runtime,
|
||||
Err(error) => {
|
||||
error!(error = %error, "Failed to build background runtime for ME initialization");
|
||||
return;
|
||||
}
|
||||
};
|
||||
runtime.block_on(async move {
|
||||
let mut init_attempt: u32 = 0;
|
||||
loop {
|
||||
init_attempt = init_attempt.saturating_add(1);
|
||||
startup_tracker_bg.set_me_init_attempt(init_attempt).await;
|
||||
match pool_bg.init(pool_size, &rng_bg).await {
|
||||
Ok(()) => {
|
||||
startup_tracker_bg.set_me_last_error(None).await;
|
||||
startup_tracker_bg
|
||||
.complete_component(
|
||||
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||
Some("ME pool initialized".to_string()),
|
||||
)
|
||||
.await;
|
||||
startup_tracker_bg
|
||||
.set_me_status(StartupMeStatus::Ready, "ready")
|
||||
.await;
|
||||
me_ready_tx_bg.send_modify(|version| {
|
||||
*version = version.saturating_add(1);
|
||||
});
|
||||
info!(
|
||||
task_scope.spawn(async move {
|
||||
let mut init_attempt: u32 = 0;
|
||||
loop {
|
||||
init_attempt = init_attempt.saturating_add(1);
|
||||
startup_tracker_bg.set_me_init_attempt(init_attempt).await;
|
||||
match pool_bg.init(pool_size, &rng_bg).await {
|
||||
Ok(()) => {
|
||||
startup_tracker_bg.set_me_last_error(None).await;
|
||||
startup_tracker_bg
|
||||
.complete_component(
|
||||
COMPONENT_ME_POOL_INIT_STAGE1,
|
||||
Some("ME pool initialized".to_string()),
|
||||
)
|
||||
.await;
|
||||
startup_tracker_bg
|
||||
.set_me_status(StartupMeStatus::Ready, "ready")
|
||||
.await;
|
||||
me_ready_tx_bg.send_modify(|version| {
|
||||
*version = version.saturating_add(1);
|
||||
});
|
||||
info!(
|
||||
attempt = init_attempt,
|
||||
"Middle-End pool initialized successfully"
|
||||
);
|
||||
spawn_me_supervisors(
|
||||
task_scope_bg,
|
||||
pool_bg.clone(),
|
||||
rng_bg.clone(),
|
||||
pool_size,
|
||||
);
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
startup_tracker_bg
|
||||
.set_me_last_error(Some(e.to_string()))
|
||||
.await;
|
||||
if init_attempt >= me_init_warn_after_attempts {
|
||||
warn!(
|
||||
error = %e,
|
||||
attempt = init_attempt,
|
||||
"Middle-End pool initialized successfully"
|
||||
retry_limit = %retry_limit,
|
||||
retry_in_secs = 2,
|
||||
"ME pool is not ready yet; retrying background initialization"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
error = %e,
|
||||
attempt = init_attempt,
|
||||
retry_limit = %retry_limit,
|
||||
retry_in_secs = 2,
|
||||
"ME pool startup warmup: retrying background initialization"
|
||||
);
|
||||
|
||||
// ── Supervised background tasks ──────────────────
|
||||
// Each task runs inside a nested tokio::spawn so
|
||||
// that a panic is caught via JoinHandle and the
|
||||
// outer loop restarts the task automatically.
|
||||
let pool_health = pool_bg.clone();
|
||||
let rng_health = rng_bg.clone();
|
||||
let min_conns = pool_size;
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let p = pool_health.clone();
|
||||
let r = rng_health.clone();
|
||||
let res = tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(
|
||||
p, r, min_conns,
|
||||
)
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
match res {
|
||||
Ok(()) => warn!("me_health_monitor exited unexpectedly, restarting"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "me_health_monitor panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let pool_drain_enforcer = pool_bg.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let p = pool_drain_enforcer.clone();
|
||||
let res = tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(p).await;
|
||||
})
|
||||
.await;
|
||||
match res {
|
||||
Ok(()) => warn!("me_drain_timeout_enforcer exited unexpectedly, restarting"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "me_drain_timeout_enforcer panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let pool_watchdog = pool_bg.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let p = pool_watchdog.clone();
|
||||
let res = tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(p).await;
|
||||
})
|
||||
.await;
|
||||
match res {
|
||||
Ok(()) => warn!("me_zombie_writer_watchdog exited unexpectedly, restarting"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "me_zombie_writer_watchdog panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
// CRITICAL: keep the current-thread runtime
|
||||
// alive. Without this, block_on() returns,
|
||||
// the Runtime is dropped, and ALL spawned
|
||||
// background tasks (health monitor, drain
|
||||
// enforcer, zombie watchdog) are silently
|
||||
// cancelled — causing the draining-writer
|
||||
// leak that brought us here.
|
||||
std::future::pending::<()>().await;
|
||||
unreachable!();
|
||||
}
|
||||
Err(e) => {
|
||||
startup_tracker_bg.set_me_last_error(Some(e.to_string())).await;
|
||||
if init_attempt >= me_init_warn_after_attempts {
|
||||
warn!(
|
||||
error = %e,
|
||||
attempt = init_attempt,
|
||||
retry_limit = %retry_limit,
|
||||
retry_in_secs = 2,
|
||||
"ME pool is not ready yet; retrying background initialization"
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
error = %e,
|
||||
attempt = init_attempt,
|
||||
retry_limit = %retry_limit,
|
||||
retry_in_secs = 2,
|
||||
"ME pool startup warmup: retrying background initialization"
|
||||
);
|
||||
}
|
||||
pool_bg.reset_stun_state();
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
pool_bg.reset_stun_state();
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
startup_tracker
|
||||
.set_me_status(StartupMeStatus::Initializing, "background_init")
|
||||
@@ -487,70 +466,12 @@ pub(crate) async fn initialize_me_pool(
|
||||
"Middle-End pool initialized successfully"
|
||||
);
|
||||
|
||||
// ── Supervised background tasks ──────────────────
|
||||
let pool_clone = pool.clone();
|
||||
let rng_clone = rng.clone();
|
||||
let min_conns = pool_size;
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let p = pool_clone.clone();
|
||||
let r = rng_clone.clone();
|
||||
let res = tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_health_monitor(
|
||||
p, r, min_conns,
|
||||
)
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
match res {
|
||||
Ok(()) => warn!(
|
||||
"me_health_monitor exited unexpectedly, restarting"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(error = %e, "me_health_monitor panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let pool_drain_enforcer = pool.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let p = pool_drain_enforcer.clone();
|
||||
let res = tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(p).await;
|
||||
})
|
||||
.await;
|
||||
match res {
|
||||
Ok(()) => warn!(
|
||||
"me_drain_timeout_enforcer exited unexpectedly, restarting"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(error = %e, "me_drain_timeout_enforcer panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let pool_watchdog = pool.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let p = pool_watchdog.clone();
|
||||
let res = tokio::spawn(async move {
|
||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(p).await;
|
||||
})
|
||||
.await;
|
||||
match res {
|
||||
Ok(()) => warn!(
|
||||
"me_zombie_writer_watchdog exited unexpectedly, restarting"
|
||||
),
|
||||
Err(e) => {
|
||||
error!(error = %e, "me_zombie_writer_watchdog panicked, restarting in 1s");
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
spawn_me_supervisors(
|
||||
task_scope.clone(),
|
||||
pool.clone(),
|
||||
rng.clone(),
|
||||
pool_size,
|
||||
);
|
||||
|
||||
break Some(pool);
|
||||
}
|
||||
@@ -663,3 +584,69 @@ pub(crate) async fn initialize_me_pool(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
struct DropSignal(Arc<Notify>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_supervisor_aborts_its_current_child() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let dropped = Arc::new(Notify::new());
|
||||
let dropped_for_task = dropped.clone();
|
||||
scope.spawn(supervise_me_task("test", move || {
|
||||
let dropped = dropped_for_task.clone();
|
||||
async move {
|
||||
let _signal = DropSignal(dropped);
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
scope.stop().await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
||||
let scope = RuntimeTaskScope::new();
|
||||
let starts = Arc::new(AtomicUsize::new(0));
|
||||
let restarted = Arc::new(Notify::new());
|
||||
let starts_task = starts.clone();
|
||||
let restarted_task = restarted.clone();
|
||||
scope.spawn(supervise_me_task("restart_test", move || {
|
||||
let starts = starts_task.clone();
|
||||
let restarted = restarted_task.clone();
|
||||
async move {
|
||||
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
||||
restarted.notify_one();
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
scope.stop().await;
|
||||
let stopped_at = starts.load(Ordering::Acquire);
|
||||
for _ in 0..100 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
assert!(stopped_at >= 3);
|
||||
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,19 +13,24 @@
|
||||
// - shutdown: graceful shutdown sequence and uptime logging.
|
||||
mod admission;
|
||||
mod connectivity;
|
||||
pub(crate) mod generation;
|
||||
mod helpers;
|
||||
mod listeners;
|
||||
mod me_startup;
|
||||
pub(crate) mod reload;
|
||||
mod reload_supervisor;
|
||||
pub(crate) mod runtime_build;
|
||||
mod runtime_tasks;
|
||||
mod shutdown;
|
||||
mod tls_bootstrap;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
use tracing::{error, info, warn};
|
||||
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload};
|
||||
use tracing_subscriber::{EnvFilter, fmt, prelude::*, reload as tracing_reload};
|
||||
|
||||
use crate::api;
|
||||
use crate::config::{LogLevel, ProxyConfig};
|
||||
@@ -33,6 +38,9 @@ use crate::conntrack_control;
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::network::probe::{decide_network_capabilities, log_probe_result, run_probe};
|
||||
use crate::proxy::direct_buffer_budget::{
|
||||
DirectBufferBudget, resolve_direct_buffer_hard_limit, run_direct_buffer_budget_controller,
|
||||
};
|
||||
use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController};
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::startup::{
|
||||
@@ -43,8 +51,9 @@ use crate::startup::{
|
||||
};
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::stats::{QuotaStore, ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::synlimit_control;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
use helpers::{
|
||||
@@ -107,7 +116,7 @@ async fn run_telemt_core(
|
||||
let data_path = cli_args.data_path;
|
||||
let cli_silent = cli_args.silent;
|
||||
let cli_log_level = cli_args.log_level;
|
||||
let log_destination = cli_args.log_destination;
|
||||
let log_cli_options = cli_args.log_cli_options;
|
||||
let startup_cwd = match std::env::current_dir() {
|
||||
Ok(cwd) => cwd,
|
||||
Err(e) => {
|
||||
@@ -330,8 +339,16 @@ async fn run_telemt_core(
|
||||
};
|
||||
|
||||
let initial_filter_spec = runtime_tasks::log_filter_spec(has_rust_log, &effective_log_level);
|
||||
let log_destination =
|
||||
match crate::logging::resolve_log_destination(&config.logging, &log_cli_options) {
|
||||
Ok(destination) => destination,
|
||||
Err(error) => {
|
||||
eprintln!("[telemt] {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let (filter_layer, filter_handle) =
|
||||
reload::Layer::new(EnvFilter::new(initial_filter_spec.clone()));
|
||||
tracing_reload::Layer::new(EnvFilter::new(initial_filter_spec.clone()));
|
||||
startup_tracker
|
||||
.start_component(
|
||||
COMPONENT_TRACING_INIT,
|
||||
@@ -375,6 +392,7 @@ async fn run_telemt_core(
|
||||
_logging_guard = Some(guard);
|
||||
}
|
||||
}
|
||||
let runtime_log_filter = runtime_tasks::RuntimeLogFilter::new(filter_handle);
|
||||
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
@@ -421,21 +439,26 @@ async fn run_telemt_core(
|
||||
warn!("Using default tls_domain. Consider setting a custom domain.");
|
||||
}
|
||||
|
||||
let stats = Arc::new(Stats::new());
|
||||
let quota_store = Arc::new(QuotaStore::default());
|
||||
let stats = Arc::new(Stats::with_quota_store(quota_store.clone()));
|
||||
let runtime_task_scope = generation::RuntimeTaskScope::new();
|
||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||
let quota_state_path = config.general.quota_state_path.clone();
|
||||
crate::quota_state::load_quota_state("a_state_path, stats.as_ref()).await;
|
||||
|
||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||
config.upstreams.clone(),
|
||||
config.general.upstream_connect_retry_attempts,
|
||||
config.general.upstream_connect_retry_backoff_ms,
|
||||
config.general.upstream_connect_budget_ms,
|
||||
config.general.tg_connect,
|
||||
config.general.upstream_unhealthy_fail_threshold,
|
||||
config.general.upstream_connect_failfast_hard_errors,
|
||||
stats.clone(),
|
||||
));
|
||||
let upstream_manager = Arc::new(
|
||||
UpstreamManager::new(
|
||||
config.upstreams.clone(),
|
||||
config.general.upstream_connect_retry_attempts,
|
||||
config.general.upstream_connect_retry_backoff_ms,
|
||||
config.general.upstream_connect_budget_ms,
|
||||
config.general.tg_connect,
|
||||
config.general.upstream_unhealthy_fail_threshold,
|
||||
config.general.upstream_connect_failfast_hard_errors,
|
||||
stats.clone(),
|
||||
)
|
||||
.with_dns_overrides(&config.network.dns_overrides)?,
|
||||
);
|
||||
let ip_tracker = Arc::new(UserIpTracker::new());
|
||||
ip_tracker
|
||||
.load_limits(
|
||||
@@ -464,12 +487,31 @@ async fn run_telemt_core(
|
||||
config.network.dns_overrides.len()
|
||||
);
|
||||
}
|
||||
let direct_buffer_hard_limit =
|
||||
resolve_direct_buffer_hard_limit(config.general.direct_relay_buffer_budget_max_bytes).await;
|
||||
let direct_buffer_budget = DirectBufferBudget::new(direct_buffer_hard_limit);
|
||||
info!(
|
||||
hard_limit_bytes = direct_buffer_hard_limit,
|
||||
configured_override_bytes = config.general.direct_relay_buffer_budget_max_bytes,
|
||||
"Direct relay buffer budget initialized"
|
||||
);
|
||||
let shared_state =
|
||||
ProxySharedState::new_with_direct_buffer_budget(direct_buffer_budget.clone());
|
||||
shared_state.apply_user_enabled_config(&config.access.user_enabled);
|
||||
shared_state.traffic_limiter.apply_policy(
|
||||
config.access.user_rate_limits.clone(),
|
||||
config.access.cidr_rate_limits.clone(),
|
||||
);
|
||||
|
||||
let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone()));
|
||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||
let initial_admission_open = !config.general.use_middle_proxy || initial_direct_first;
|
||||
let (admission_tx, admission_rx) = watch::channel(initial_admission_open);
|
||||
let (reload_control, reload_commands) = reload::ReloadControl::channel(1);
|
||||
let (active_runtime_tx, active_runtime_rx) =
|
||||
watch::channel(None::<Arc<ArcSwap<generation::RuntimeGeneration>>>);
|
||||
let (runtime_watch_tx, runtime_watch_rx) =
|
||||
watch::channel(None::<generation::RuntimeWatchState>);
|
||||
let initial_route_mode = if !config.general.use_middle_proxy || initial_direct_first {
|
||||
RelayRouteMode::Direct
|
||||
} else {
|
||||
@@ -502,12 +544,14 @@ async fn run_telemt_core(
|
||||
let me_pool_api = api_me_pool.clone();
|
||||
let upstream_manager_api = upstream_manager.clone();
|
||||
let route_runtime_api = route_runtime.clone();
|
||||
let config_rx_api = api_config_rx.clone();
|
||||
let admission_rx_api = admission_rx.clone();
|
||||
let proxy_shared_api = shared_state.clone();
|
||||
let config_path_api = config_path.clone();
|
||||
let quota_state_path_api = quota_state_path.clone();
|
||||
let startup_tracker_api = startup_tracker.clone();
|
||||
let detected_ips_rx_api = detected_ips_rx.clone();
|
||||
let reload_control_api = reload_control.clone();
|
||||
let active_runtime_rx_api = active_runtime_rx.clone();
|
||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
api::serve(
|
||||
listen,
|
||||
@@ -515,14 +559,16 @@ async fn run_telemt_core(
|
||||
ip_tracker_api,
|
||||
me_pool_api,
|
||||
route_runtime_api,
|
||||
proxy_shared_api,
|
||||
upstream_manager_api,
|
||||
config_rx_api,
|
||||
admission_rx_api,
|
||||
config_path_api,
|
||||
quota_state_path_api,
|
||||
detected_ips_rx_api,
|
||||
process_started_at_epoch_secs,
|
||||
startup_tracker_api,
|
||||
reload_control_api,
|
||||
active_runtime_rx_api,
|
||||
runtime_watch_rx_api,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
@@ -562,8 +608,10 @@ async fn run_telemt_core(
|
||||
&tls_domains,
|
||||
upstream_manager.clone(),
|
||||
&startup_tracker,
|
||||
runtime_task_scope.clone(),
|
||||
tls_bootstrap::TlsBootstrapPolicy::BestEffort,
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
startup_tracker
|
||||
.start_component(
|
||||
@@ -689,6 +737,7 @@ async fn run_telemt_core(
|
||||
stats.clone(),
|
||||
api_me_pool.clone(),
|
||||
me_ready_tx.clone(),
|
||||
runtime_task_scope.clone(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -732,11 +781,6 @@ async fn run_telemt_core(
|
||||
));
|
||||
|
||||
let buffer_pool = Arc::new(BufferPool::with_config(64 * 1024, 4096));
|
||||
let shared_state = ProxySharedState::new();
|
||||
shared_state.traffic_limiter.apply_policy(
|
||||
config.access.user_rate_limits.clone(),
|
||||
config.access.cidr_rate_limits.clone(),
|
||||
);
|
||||
|
||||
if direct_first_startup {
|
||||
startup_tracker
|
||||
@@ -781,16 +825,22 @@ async fn run_telemt_core(
|
||||
rng.clone(),
|
||||
ip_tracker.clone(),
|
||||
beobachten.clone(),
|
||||
api_config_tx.clone(),
|
||||
me_pool.clone(),
|
||||
shared_state.clone(),
|
||||
me_ready_tx.clone(),
|
||||
runtime_task_scope.clone(),
|
||||
)
|
||||
.await;
|
||||
let config_rx = runtime_watches.config_rx;
|
||||
let log_level_rx = runtime_watches.log_level_rx;
|
||||
let detected_ip_v4 = runtime_watches.detected_ip_v4;
|
||||
let detected_ip_v6 = runtime_watches.detected_ip_v6;
|
||||
runtime_log_filter.start(
|
||||
has_rust_log,
|
||||
&effective_log_level,
|
||||
log_level_rx,
|
||||
runtime_task_scope.clone(),
|
||||
);
|
||||
|
||||
if direct_first_startup {
|
||||
let config_bg = config.clone();
|
||||
@@ -803,7 +853,8 @@ async fn run_telemt_core(
|
||||
let api_me_pool_bg = api_me_pool.clone();
|
||||
let me_ready_tx_bg = me_ready_tx.clone();
|
||||
let config_rx_bg = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
let task_scope_bg = runtime_task_scope.clone();
|
||||
runtime_task_scope.spawn(async move {
|
||||
let mut bootstrap_attempt: u32 = 0;
|
||||
loop {
|
||||
bootstrap_attempt = bootstrap_attempt.saturating_add(1);
|
||||
@@ -818,6 +869,7 @@ async fn run_telemt_core(
|
||||
stats_bg.clone(),
|
||||
api_me_pool_bg.clone(),
|
||||
me_ready_tx_bg.clone(),
|
||||
task_scope_bg.clone(),
|
||||
)
|
||||
.await;
|
||||
if let Some(pool) = pool {
|
||||
@@ -827,6 +879,7 @@ async fn run_telemt_core(
|
||||
pool,
|
||||
rng_bg,
|
||||
me_ready_tx_bg,
|
||||
task_scope_bg,
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -840,7 +893,7 @@ async fn run_telemt_core(
|
||||
let startup_tracker_ready = startup_tracker.clone();
|
||||
let api_me_pool_ready = api_me_pool.clone();
|
||||
let mut me_ready_rx_transport = me_ready_tx.subscribe();
|
||||
tokio::spawn(async move {
|
||||
runtime_task_scope.spawn(async move {
|
||||
if me_ready_rx_transport.changed().await.is_ok() {
|
||||
if let Some(pool) = api_me_pool_ready.read().await.as_ref() {
|
||||
pool.set_runtime_ready(true);
|
||||
@@ -862,13 +915,54 @@ async fn run_telemt_core(
|
||||
&admission_tx,
|
||||
config_rx.clone(),
|
||||
me_ready_rx,
|
||||
runtime_task_scope.clone(),
|
||||
)
|
||||
.await;
|
||||
let _admission_tx_hold = admission_tx;
|
||||
conntrack_control::spawn_conntrack_controller(
|
||||
let conntrack_scope = runtime_task_scope.clone();
|
||||
runtime_task_scope.spawn(conntrack_control::run_conntrack_controller(
|
||||
config_rx.clone(),
|
||||
stats.clone(),
|
||||
shared_state.clone(),
|
||||
conntrack_scope.cancellation_token(),
|
||||
));
|
||||
runtime_task_scope.spawn(run_direct_buffer_budget_controller(
|
||||
direct_buffer_budget,
|
||||
buffer_pool.clone(),
|
||||
stats.clone(),
|
||||
shared_state.clone(),
|
||||
config.server.max_connections,
|
||||
));
|
||||
|
||||
let runtime_generation = generation::RuntimeGeneration::new(
|
||||
1,
|
||||
config_rx.clone(),
|
||||
admission_rx.clone(),
|
||||
stats.clone(),
|
||||
upstream_manager.clone(),
|
||||
replay_checker.clone(),
|
||||
buffer_pool.clone(),
|
||||
rng.clone(),
|
||||
me_pool.clone(),
|
||||
api_me_pool.clone(),
|
||||
route_runtime.clone(),
|
||||
tls_cache.clone(),
|
||||
ip_tracker.clone(),
|
||||
beobachten.clone(),
|
||||
shared_state.clone(),
|
||||
max_connections.clone(),
|
||||
runtime_task_scope.clone(),
|
||||
);
|
||||
let active_runtime = Arc::new(ArcSwap::from(runtime_generation));
|
||||
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
||||
active_runtime.clone(),
|
||||
reload_control,
|
||||
reload_commands,
|
||||
config_path.clone(),
|
||||
quota_store,
|
||||
detected_ips_tx,
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx.clone(),
|
||||
);
|
||||
|
||||
let bound = listeners::bind_listeners(
|
||||
@@ -878,25 +972,15 @@ async fn run_telemt_core(
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
&startup_tracker,
|
||||
config_rx.clone(),
|
||||
admission_rx.clone(),
|
||||
stats.clone(),
|
||||
upstream_manager.clone(),
|
||||
replay_checker.clone(),
|
||||
buffer_pool.clone(),
|
||||
rng.clone(),
|
||||
me_pool.clone(),
|
||||
api_me_pool.clone(),
|
||||
route_runtime.clone(),
|
||||
tls_cache.clone(),
|
||||
ip_tracker.clone(),
|
||||
beobachten.clone(),
|
||||
shared_state.clone(),
|
||||
max_connections.clone(),
|
||||
)
|
||||
.await?;
|
||||
let listeners = bound.listeners;
|
||||
let has_unix_listener = bound.has_unix_listener;
|
||||
#[cfg(unix)]
|
||||
let unix_listener = bound.unix_listener;
|
||||
#[cfg(unix)]
|
||||
let has_unix_listener = unix_listener.is_some();
|
||||
#[cfg(not(unix))]
|
||||
let has_unix_listener = false;
|
||||
|
||||
if listeners.is_empty() && !has_unix_listener {
|
||||
error!("No listeners. Exiting.");
|
||||
@@ -906,51 +990,30 @@ async fn run_telemt_core(
|
||||
// On Unix, caller supplies privilege drop after bind (may require root for port < 1024).
|
||||
drop_after_bind();
|
||||
|
||||
runtime_tasks::apply_runtime_log_filter(
|
||||
has_rust_log,
|
||||
&effective_log_level,
|
||||
filter_handle,
|
||||
log_level_rx,
|
||||
)
|
||||
.await;
|
||||
let synlimit_controller = synlimit_control::spawn_synlimit_controller(runtime_watch_rx);
|
||||
|
||||
runtime_tasks::spawn_metrics_if_configured(
|
||||
&config,
|
||||
&startup_tracker,
|
||||
stats.clone(),
|
||||
beobachten.clone(),
|
||||
shared_state.clone(),
|
||||
ip_tracker.clone(),
|
||||
tls_cache.clone(),
|
||||
config_rx.clone(),
|
||||
)
|
||||
.await;
|
||||
runtime_tasks::spawn_metrics_if_configured(&config, &startup_tracker, active_runtime.clone())
|
||||
.await;
|
||||
|
||||
runtime_watch_tx.send_replace(Some(active_runtime.load_full().watch_state()));
|
||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||
|
||||
// Spawn signal handlers for SIGUSR1/SIGUSR2 (non-shutdown signals)
|
||||
shutdown::spawn_signal_handlers(stats.clone(), process_started_at);
|
||||
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
||||
|
||||
listeners::spawn_tcp_accept_loops(
|
||||
listeners,
|
||||
config_rx.clone(),
|
||||
admission_rx.clone(),
|
||||
stats.clone(),
|
||||
upstream_manager.clone(),
|
||||
replay_checker.clone(),
|
||||
buffer_pool.clone(),
|
||||
rng.clone(),
|
||||
me_pool.clone(),
|
||||
api_me_pool.clone(),
|
||||
route_runtime.clone(),
|
||||
tls_cache.clone(),
|
||||
ip_tracker.clone(),
|
||||
beobachten.clone(),
|
||||
shared_state,
|
||||
max_connections.clone(),
|
||||
);
|
||||
listeners::spawn_tcp_accept_loops(listeners, active_runtime.clone());
|
||||
#[cfg(unix)]
|
||||
listeners::spawn_unix_accept_loop(unix_listener, active_runtime.clone());
|
||||
|
||||
shutdown::wait_for_shutdown(process_started_at, me_pool, stats, quota_state_path).await;
|
||||
shutdown::wait_for_shutdown(
|
||||
process_started_at,
|
||||
active_runtime,
|
||||
quota_state_path,
|
||||
synlimit_controller,
|
||||
reload_supervisor,
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
449
src/maestro/reload.rs
Normal file
449
src/maestro/reload.rs
Normal file
@@ -0,0 +1,449 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
|
||||
const RELOAD_HISTORY_CAPACITY: usize = 32;
|
||||
const RELOAD_COMMAND_CAPACITY: usize = 1;
|
||||
const MAX_DRAIN_TIMEOUT_SECS: u64 = 3_600;
|
||||
|
||||
/// Session handling policy for an in-process runtime reload.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum ReloadMode {
|
||||
#[default]
|
||||
Instant,
|
||||
Drain,
|
||||
}
|
||||
|
||||
/// Failure policy applied during the activation barrier.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum ReloadFailurePolicy {
|
||||
#[default]
|
||||
KeepNew,
|
||||
Rollback,
|
||||
}
|
||||
|
||||
/// Request body accepted by the maestro reload endpoint.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct ReloadRequest {
|
||||
#[serde(default)]
|
||||
pub(crate) mode: ReloadMode,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) timeout_secs: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub(crate) failure_policy: ReloadFailurePolicy,
|
||||
}
|
||||
|
||||
impl ReloadRequest {
|
||||
/// Validates mode-specific request parameters.
|
||||
pub(crate) fn validate(&self) -> Result<(), &'static str> {
|
||||
match (self.mode, self.timeout_secs) {
|
||||
(ReloadMode::Instant, None) => Ok(()),
|
||||
(ReloadMode::Instant, Some(_)) => Err("timeout_secs is only valid when mode is drain"),
|
||||
(ReloadMode::Drain, Some(1..=MAX_DRAIN_TIMEOUT_SECS)) => Ok(()),
|
||||
(ReloadMode::Drain, Some(_)) => Err("timeout_secs must be within 1..=3600"),
|
||||
(ReloadMode::Drain, None) => Err("timeout_secs is required when mode is drain"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses optional PATCH query parameters into a reload request.
|
||||
pub(crate) fn from_query(query: Option<&str>) -> Result<Option<Self>, String> {
|
||||
let Some(query) = query.filter(|query| !query.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let mut mode = None;
|
||||
let mut timeout_secs = None;
|
||||
let mut failure_policy = None;
|
||||
for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||
match key.as_ref() {
|
||||
"reload" if mode.is_none() => {
|
||||
mode = Some(match value.as_ref() {
|
||||
"instant" => ReloadMode::Instant,
|
||||
"drain" => ReloadMode::Drain,
|
||||
_ => return Err("reload must be instant or drain".to_string()),
|
||||
});
|
||||
}
|
||||
"timeout_secs" if timeout_secs.is_none() => {
|
||||
timeout_secs = Some(
|
||||
value
|
||||
.parse::<u64>()
|
||||
.map_err(|_| "timeout_secs must be an integer".to_string())?,
|
||||
);
|
||||
}
|
||||
"failure_policy" if failure_policy.is_none() => {
|
||||
failure_policy = Some(match value.as_ref() {
|
||||
"keep_new" => ReloadFailurePolicy::KeepNew,
|
||||
"rollback" => ReloadFailurePolicy::Rollback,
|
||||
_ => {
|
||||
return Err("failure_policy must be keep_new or rollback".to_string());
|
||||
}
|
||||
});
|
||||
}
|
||||
"reload" | "timeout_secs" | "failure_policy" => {
|
||||
return Err(format!("duplicate query parameter: {}", key));
|
||||
}
|
||||
_ => return Err(format!("unknown query parameter: {}", key)),
|
||||
}
|
||||
}
|
||||
let mode = mode.ok_or_else(|| "reload query parameter is required".to_string())?;
|
||||
let request = Self {
|
||||
mode,
|
||||
timeout_secs,
|
||||
failure_policy: failure_policy.unwrap_or_default(),
|
||||
};
|
||||
request.validate().map_err(str::to_string)?;
|
||||
Ok(Some(request))
|
||||
}
|
||||
}
|
||||
|
||||
/// Observable phase of one reload operation.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(crate) enum ReloadPhase {
|
||||
Accepted,
|
||||
Preparing,
|
||||
Activating,
|
||||
Draining,
|
||||
Succeeded,
|
||||
RolledBack,
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl ReloadPhase {
|
||||
fn is_terminal(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
ReloadPhase::Succeeded | ReloadPhase::RolledBack | ReloadPhase::Failed
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded public status for one reload operation.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct ReloadStatus {
|
||||
pub(crate) reload_id: u64,
|
||||
pub(crate) target_generation: u64,
|
||||
pub(crate) config_revision: String,
|
||||
pub(crate) state: ReloadPhase,
|
||||
pub(crate) mode: ReloadMode,
|
||||
pub(crate) failure_policy: ReloadFailurePolicy,
|
||||
pub(crate) requested_at_epoch_secs: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) started_at_epoch_secs: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) finished_at_epoch_secs: Option<u64>,
|
||||
#[serde(
|
||||
rename = "deferred_process_fields",
|
||||
default,
|
||||
skip_serializing_if = "Vec::is_empty"
|
||||
)]
|
||||
pub(crate) deferred_fields: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub(crate) warnings: Vec<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) error: Option<String>,
|
||||
}
|
||||
|
||||
/// Accepted operation metadata returned before asynchronous preparation starts.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(crate) struct ReloadAccepted {
|
||||
pub(crate) reload_id: u64,
|
||||
pub(crate) target_generation: u64,
|
||||
pub(crate) config_revision: String,
|
||||
pub(crate) state: ReloadPhase,
|
||||
pub(crate) mode: ReloadMode,
|
||||
pub(crate) failure_policy: ReloadFailurePolicy,
|
||||
}
|
||||
|
||||
pub(crate) struct ReloadCommand {
|
||||
pub(crate) reload_id: u64,
|
||||
pub(crate) target_generation: u64,
|
||||
pub(crate) config: Arc<ProxyConfig>,
|
||||
pub(crate) config_revision: String,
|
||||
pub(crate) request: ReloadRequest,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ReloadSubmitError {
|
||||
InProgress(u64),
|
||||
MaestroUnavailable,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ReloadControl {
|
||||
command_tx: mpsc::Sender<ReloadCommand>,
|
||||
status_store: Arc<ReloadStatusStore>,
|
||||
active_generation: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
pub(crate) struct ReloadCommandReceiver {
|
||||
command_rx: mpsc::Receiver<ReloadCommand>,
|
||||
}
|
||||
|
||||
struct ReloadStatusState {
|
||||
next_reload_id: u64,
|
||||
active_reload_id: Option<u64>,
|
||||
statuses: VecDeque<ReloadStatus>,
|
||||
accepting_commands: bool,
|
||||
}
|
||||
|
||||
impl Default for ReloadStatusState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
next_reload_id: 0,
|
||||
active_reload_id: None,
|
||||
statuses: VecDeque::new(),
|
||||
accepting_commands: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReloadStatusStore {
|
||||
state: Mutex<ReloadStatusState>,
|
||||
}
|
||||
|
||||
impl ReloadControl {
|
||||
/// Creates the process-scoped coordinator channel and status store.
|
||||
pub(crate) fn channel(initial_generation: u64) -> (Self, ReloadCommandReceiver) {
|
||||
let (command_tx, command_rx) = mpsc::channel(RELOAD_COMMAND_CAPACITY);
|
||||
(
|
||||
Self {
|
||||
command_tx,
|
||||
status_store: Arc::new(ReloadStatusStore::default()),
|
||||
active_generation: Arc::new(AtomicU64::new(initial_generation)),
|
||||
},
|
||||
ReloadCommandReceiver { command_rx },
|
||||
)
|
||||
}
|
||||
|
||||
/// Atomically reserves and enqueues one reload operation.
|
||||
pub(crate) async fn submit(
|
||||
&self,
|
||||
config: Arc<ProxyConfig>,
|
||||
config_revision: String,
|
||||
request: ReloadRequest,
|
||||
) -> Result<ReloadAccepted, ReloadSubmitError> {
|
||||
let target_generation = self
|
||||
.active_generation
|
||||
.load(Ordering::Acquire)
|
||||
.saturating_add(1);
|
||||
let status = self
|
||||
.status_store
|
||||
.reserve(target_generation, config_revision, request.clone())
|
||||
.await?;
|
||||
let command = ReloadCommand {
|
||||
reload_id: status.reload_id,
|
||||
target_generation,
|
||||
config,
|
||||
config_revision: status.config_revision.clone(),
|
||||
request,
|
||||
};
|
||||
if self.command_tx.try_send(command).is_err() {
|
||||
self.status_store
|
||||
.finish(
|
||||
status.reload_id,
|
||||
ReloadPhase::Failed,
|
||||
Some("maestro command channel is closed".to_string()),
|
||||
)
|
||||
.await;
|
||||
return Err(ReloadSubmitError::MaestroUnavailable);
|
||||
}
|
||||
Ok(ReloadAccepted {
|
||||
reload_id: status.reload_id,
|
||||
target_generation,
|
||||
config_revision: status.config_revision,
|
||||
state: ReloadPhase::Accepted,
|
||||
mode: status.mode,
|
||||
failure_policy: status.failure_policy,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns a retained reload status by identifier.
|
||||
pub(crate) async fn status(&self, reload_id: u64) -> Option<ReloadStatus> {
|
||||
self.status_store.get(reload_id).await
|
||||
}
|
||||
|
||||
/// Returns the identifier of the currently active reload.
|
||||
pub(crate) async fn in_progress(&self) -> Option<u64> {
|
||||
self.status_store.state.lock().await.active_reload_id
|
||||
}
|
||||
|
||||
/// Rejects new commands while preserving an already accepted operation.
|
||||
pub(crate) async fn begin_shutdown(&self) {
|
||||
self.status_store.state.lock().await.accepting_commands = false;
|
||||
}
|
||||
|
||||
/// Records a non-terminal lifecycle phase.
|
||||
pub(crate) async fn mark_phase(&self, reload_id: u64, phase: ReloadPhase) {
|
||||
self.status_store.mark_phase(reload_id, phase).await;
|
||||
}
|
||||
|
||||
/// Records process-owned fields deferred until the next process restart.
|
||||
pub(crate) async fn set_deferred_fields(&self, reload_id: u64, fields: Vec<String>) {
|
||||
self.status_store
|
||||
.update(reload_id, |status| status.deferred_fields = fields)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Commits the active generation and completes the matching reload.
|
||||
pub(crate) async fn succeed(&self, reload_id: u64, generation: u64) {
|
||||
self.status_store
|
||||
.finish_success(reload_id, generation, &self.active_generation)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Marks the matching reload as failed.
|
||||
pub(crate) async fn fail(&self, reload_id: u64, error: impl Into<String>) {
|
||||
self.status_store
|
||||
.finish(reload_id, ReloadPhase::Failed, Some(error.into()))
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Marks the matching reload as rolled back.
|
||||
pub(crate) async fn rolled_back(&self, reload_id: u64, error: impl Into<String>) {
|
||||
self.status_store
|
||||
.finish(reload_id, ReloadPhase::RolledBack, Some(error.into()))
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Appends a non-fatal warning to the matching reload status.
|
||||
pub(crate) async fn add_warning(&self, reload_id: u64, warning: impl Into<String>) {
|
||||
let warning = warning.into();
|
||||
self.status_store
|
||||
.update(reload_id, |status| status.warnings.push(warning))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
impl ReloadCommandReceiver {
|
||||
/// Receives the next accepted reload command.
|
||||
pub(crate) async fn recv(&mut self) -> Option<ReloadCommand> {
|
||||
self.command_rx.recv().await
|
||||
}
|
||||
}
|
||||
|
||||
impl ReloadStatusStore {
|
||||
async fn reserve(
|
||||
&self,
|
||||
target_generation: u64,
|
||||
config_revision: String,
|
||||
request: ReloadRequest,
|
||||
) -> Result<ReloadStatus, ReloadSubmitError> {
|
||||
let mut state = self.state.lock().await;
|
||||
if !state.accepting_commands {
|
||||
return Err(ReloadSubmitError::MaestroUnavailable);
|
||||
}
|
||||
if let Some(reload_id) = state.active_reload_id {
|
||||
return Err(ReloadSubmitError::InProgress(reload_id));
|
||||
}
|
||||
state.next_reload_id = state.next_reload_id.saturating_add(1).max(1);
|
||||
let reload_id = state.next_reload_id;
|
||||
let status = ReloadStatus {
|
||||
reload_id,
|
||||
target_generation,
|
||||
config_revision,
|
||||
state: ReloadPhase::Accepted,
|
||||
mode: request.mode,
|
||||
failure_policy: request.failure_policy,
|
||||
requested_at_epoch_secs: now_epoch_secs(),
|
||||
started_at_epoch_secs: None,
|
||||
finished_at_epoch_secs: None,
|
||||
deferred_fields: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
error: None,
|
||||
};
|
||||
state.active_reload_id = Some(reload_id);
|
||||
state.statuses.push_back(status.clone());
|
||||
while state.statuses.len() > RELOAD_HISTORY_CAPACITY {
|
||||
state.statuses.pop_front();
|
||||
}
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
async fn get(&self, reload_id: u64) -> Option<ReloadStatus> {
|
||||
self.state
|
||||
.lock()
|
||||
.await
|
||||
.statuses
|
||||
.iter()
|
||||
.find(|status| status.reload_id == reload_id)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
async fn mark_phase(&self, reload_id: u64, phase: ReloadPhase) {
|
||||
self.update(reload_id, |status| {
|
||||
status.state = phase;
|
||||
if status.started_at_epoch_secs.is_none() && phase != ReloadPhase::Accepted {
|
||||
status.started_at_epoch_secs = Some(now_epoch_secs());
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn finish(&self, reload_id: u64, phase: ReloadPhase, error: Option<String>) {
|
||||
debug_assert!(phase.is_terminal());
|
||||
let mut state = self.state.lock().await;
|
||||
if let Some(status) = state
|
||||
.statuses
|
||||
.iter_mut()
|
||||
.find(|status| status.reload_id == reload_id)
|
||||
{
|
||||
status.state = phase;
|
||||
status.error = error;
|
||||
status.finished_at_epoch_secs = Some(now_epoch_secs());
|
||||
}
|
||||
if state.active_reload_id == Some(reload_id) {
|
||||
state.active_reload_id = None;
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_success(&self, reload_id: u64, generation: u64, active_generation: &AtomicU64) {
|
||||
let mut state = self.state.lock().await;
|
||||
if state.active_reload_id != Some(reload_id) {
|
||||
return;
|
||||
}
|
||||
let Some(status) = state
|
||||
.statuses
|
||||
.iter_mut()
|
||||
.find(|status| status.reload_id == reload_id)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
status.state = ReloadPhase::Succeeded;
|
||||
status.error = None;
|
||||
status.finished_at_epoch_secs = Some(now_epoch_secs());
|
||||
active_generation.store(generation, Ordering::Release);
|
||||
state.active_reload_id = None;
|
||||
}
|
||||
|
||||
async fn update(&self, reload_id: u64, update: impl FnOnce(&mut ReloadStatus)) {
|
||||
let mut state = self.state.lock().await;
|
||||
if let Some(status) = state
|
||||
.statuses
|
||||
.iter_mut()
|
||||
.find(|status| status.reload_id == reload_id)
|
||||
{
|
||||
update(status);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn now_epoch_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "reload_tests.rs"]
|
||||
mod tests;
|
||||
280
src/maestro/reload_supervisor.rs
Normal file
280
src/maestro/reload_supervisor.rs
Normal file
@@ -0,0 +1,280 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::watch;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::stats::QuotaStore;
|
||||
|
||||
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||
use super::reload::{
|
||||
ReloadCommand, ReloadCommandReceiver, ReloadControl, ReloadFailurePolicy, ReloadMode,
|
||||
ReloadPhase,
|
||||
};
|
||||
use super::runtime_build::{PreparedRuntime, deferred_process_fields, prepare_runtime};
|
||||
use super::runtime_tasks::RuntimeLogFilter;
|
||||
|
||||
pub(crate) struct ReloadSupervisor {
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
control: ReloadControl,
|
||||
commands: ReloadCommandReceiver,
|
||||
config_path: PathBuf,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
}
|
||||
|
||||
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
|
||||
pub(crate) struct ReloadSupervisorHandle {
|
||||
control: ReloadControl,
|
||||
shutdown: CancellationToken,
|
||||
join: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl ReloadSupervisorHandle {
|
||||
/// Stops new submissions and waits for the accepted reload to finish.
|
||||
pub(crate) async fn quiesce(self) {
|
||||
self.control.begin_shutdown().await;
|
||||
self.shutdown.cancel();
|
||||
if let Err(error) = self.join.await {
|
||||
warn!(error = %error, "Reload supervisor failed while quiescing");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum RevisionGateAction {
|
||||
Proceed,
|
||||
Warn(String),
|
||||
Rollback(String),
|
||||
}
|
||||
|
||||
fn revision_gate_action(
|
||||
accepted_revision: &str,
|
||||
current_revision: Result<String, String>,
|
||||
failure_policy: ReloadFailurePolicy,
|
||||
) -> RevisionGateAction {
|
||||
let warning = match current_revision {
|
||||
Ok(current) if current == accepted_revision => return RevisionGateAction::Proceed,
|
||||
Ok(current) => format!(
|
||||
"config revision changed during preparation: accepted={} current={}",
|
||||
accepted_revision, current
|
||||
),
|
||||
Err(error) => format!("config revision verification failed: {}", error),
|
||||
};
|
||||
match failure_policy {
|
||||
ReloadFailurePolicy::KeepNew => RevisionGateAction::Warn(warning),
|
||||
ReloadFailurePolicy::Rollback => RevisionGateAction::Rollback(warning),
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop_background_and_middle_end(generation: &RuntimeGeneration) -> bool {
|
||||
generation.stop_background_tasks().await;
|
||||
let Some(pool) = generation.current_me_pool().await else {
|
||||
return false;
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
||||
.await
|
||||
.is_err()
|
||||
}
|
||||
|
||||
async fn cleanup_candidate(generation: &RuntimeGeneration) -> bool {
|
||||
generation.stop_sessions().await;
|
||||
stop_background_and_middle_end(generation).await
|
||||
}
|
||||
|
||||
impl ReloadSupervisor {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Starts the process-scoped reload supervisor and returns its shutdown owner.
|
||||
pub(crate) fn spawn(
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
control: ReloadControl,
|
||||
commands: ReloadCommandReceiver,
|
||||
config_path: PathBuf,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||
) -> ReloadSupervisorHandle {
|
||||
let supervisor = Self {
|
||||
active_runtime,
|
||||
control,
|
||||
commands,
|
||||
config_path,
|
||||
quota_store,
|
||||
detected_ips_tx,
|
||||
runtime_log_filter,
|
||||
runtime_watch_tx,
|
||||
};
|
||||
let control = supervisor.control.clone();
|
||||
let shutdown = CancellationToken::new();
|
||||
let join = tokio::spawn(supervisor.run(shutdown.clone()));
|
||||
ReloadSupervisorHandle {
|
||||
control,
|
||||
shutdown,
|
||||
join,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(mut self, shutdown: CancellationToken) {
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = shutdown.cancelled() => {
|
||||
if self.control.in_progress().await.is_some()
|
||||
&& let Some(command) = self.commands.recv().await
|
||||
{
|
||||
self.reload(command).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
command = self.commands.recv() => {
|
||||
let Some(command) = command else {
|
||||
break;
|
||||
};
|
||||
self.reload(command).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reload(&self, command: ReloadCommand) {
|
||||
self.control
|
||||
.mark_phase(command.reload_id, ReloadPhase::Preparing)
|
||||
.await;
|
||||
let old_runtime = self.active_runtime.load_full();
|
||||
let deferred = deferred_process_fields(&old_runtime.config(), &command.config);
|
||||
self.control
|
||||
.set_deferred_fields(command.reload_id, deferred)
|
||||
.await;
|
||||
|
||||
let prepared = match prepare_runtime(
|
||||
command.target_generation,
|
||||
command.config.as_ref().clone(),
|
||||
&self.config_path,
|
||||
self.quota_store.clone(),
|
||||
self.runtime_log_filter.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => {
|
||||
self.control.fail(command.reload_id, error).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let revision_action = revision_gate_action(
|
||||
&command.config_revision,
|
||||
crate::api::config_store::current_revision_for_maestro(&self.config_path).await,
|
||||
command.request.failure_policy,
|
||||
);
|
||||
self.activate_prepared(command, old_runtime, prepared, revision_action, |entries| {
|
||||
crate::network::dns_overrides::install_entries(entries)
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn activate_prepared<InstallDns>(
|
||||
&self,
|
||||
command: ReloadCommand,
|
||||
old_runtime: Arc<RuntimeGeneration>,
|
||||
prepared: PreparedRuntime,
|
||||
revision_action: RevisionGateAction,
|
||||
install_dns: InstallDns,
|
||||
) where
|
||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
||||
{
|
||||
match revision_action {
|
||||
RevisionGateAction::Proceed => {}
|
||||
RevisionGateAction::Warn(warning) => {
|
||||
self.control.add_warning(command.reload_id, warning).await;
|
||||
}
|
||||
RevisionGateAction::Rollback(warning) => {
|
||||
let _ = cleanup_candidate(&prepared.generation).await;
|
||||
self.runtime_log_filter
|
||||
.apply_reload(&old_runtime.config().general.log_level);
|
||||
self.control.rolled_back(command.reload_id, warning).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
self.control
|
||||
.mark_phase(command.reload_id, ReloadPhase::Activating)
|
||||
.await;
|
||||
let new_runtime = prepared.generation;
|
||||
old_runtime.stop_accepting_sessions();
|
||||
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
||||
let message = format!("runtime DNS activation failed: {}", error);
|
||||
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
||||
old_runtime.resume_accepting_sessions();
|
||||
let _ = cleanup_candidate(&new_runtime).await;
|
||||
self.runtime_log_filter
|
||||
.apply_reload(&old_runtime.config().general.log_level);
|
||||
self.control.rolled_back(command.reload_id, message).await;
|
||||
return;
|
||||
}
|
||||
self.control.add_warning(command.reload_id, message).await;
|
||||
}
|
||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
||||
self.detected_ips_tx.send_replace(prepared.detected_ips);
|
||||
self.runtime_log_filter
|
||||
.apply_reload(&new_runtime.config().general.log_level);
|
||||
self.runtime_watch_tx
|
||||
.send_replace(Some(new_runtime.watch_state()));
|
||||
|
||||
info!(
|
||||
reload_id = command.reload_id,
|
||||
old_generation = replaced.id,
|
||||
new_generation = new_runtime.id,
|
||||
config_revision = %command.config_revision,
|
||||
"Runtime generation activated"
|
||||
);
|
||||
|
||||
match command.request.mode {
|
||||
ReloadMode::Instant => {
|
||||
replaced.stop_sessions().await;
|
||||
}
|
||||
ReloadMode::Drain => {
|
||||
self.control
|
||||
.mark_phase(command.reload_id, ReloadPhase::Draining)
|
||||
.await;
|
||||
let timeout = Duration::from_secs(
|
||||
command
|
||||
.request
|
||||
.timeout_secs
|
||||
.expect("validated drain request must carry timeout_secs"),
|
||||
);
|
||||
if !replaced.drain_sessions(timeout).await {
|
||||
let warning = format!(
|
||||
"generation {} exceeded drain timeout; remaining sessions were cancelled",
|
||||
replaced.id
|
||||
);
|
||||
warn!(reload_id = command.reload_id, warning = %warning);
|
||||
self.control.add_warning(command.reload_id, warning).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if stop_background_and_middle_end(&replaced).await {
|
||||
let warning = format!(
|
||||
"generation {} Middle-End close broadcast timed out",
|
||||
replaced.id
|
||||
);
|
||||
warn!(reload_id = command.reload_id, warning = %warning);
|
||||
self.control.add_warning(command.reload_id, warning).await;
|
||||
}
|
||||
self.control
|
||||
.succeed(command.reload_id, new_runtime.id)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "reload_supervisor_tests.rs"]
|
||||
mod tests;
|
||||
320
src/maestro/reload_supervisor_tests.rs
Normal file
320
src/maestro/reload_supervisor_tests.rs
Normal file
@@ -0,0 +1,320 @@
|
||||
use super::*;
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
use crate::maestro::reload::{ReloadRequest, ReloadSubmitError};
|
||||
use crate::stats::QuotaStore;
|
||||
use tokio::sync::Notify;
|
||||
use tracing_subscriber::{EnvFilter, Registry};
|
||||
|
||||
struct ReloadFixture {
|
||||
supervisor: Arc<ReloadSupervisor>,
|
||||
control: ReloadControl,
|
||||
command: ReloadCommand,
|
||||
old_runtime: Arc<RuntimeGeneration>,
|
||||
new_runtime: Arc<RuntimeGeneration>,
|
||||
runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||
}
|
||||
|
||||
fn runtime_log_filter() -> RuntimeLogFilter {
|
||||
let (_layer, handle) =
|
||||
tracing_subscriber::reload::Layer::<EnvFilter, Registry>::new(EnvFilter::new("info"));
|
||||
RuntimeLogFilter::new(handle)
|
||||
}
|
||||
|
||||
async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||
let old_runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||
let new_config = Arc::new(ProxyConfig::default());
|
||||
let new_runtime = test_runtime_generation(2, new_config.as_ref().clone());
|
||||
let active_runtime = Arc::new(ArcSwap::from(old_runtime.clone()));
|
||||
let (control, commands) = ReloadControl::channel(old_runtime.id);
|
||||
let accepted = control
|
||||
.submit(new_config.clone(), "revision".to_string(), request.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
|
||||
let supervisor = Arc::new(ReloadSupervisor {
|
||||
active_runtime,
|
||||
control: control.clone(),
|
||||
commands,
|
||||
config_path: PathBuf::new(),
|
||||
quota_store: Arc::new(QuotaStore::default()),
|
||||
detected_ips_tx,
|
||||
runtime_log_filter: runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
});
|
||||
let command = ReloadCommand {
|
||||
reload_id: accepted.reload_id,
|
||||
target_generation: accepted.target_generation,
|
||||
config: new_config,
|
||||
config_revision: accepted.config_revision,
|
||||
request,
|
||||
};
|
||||
ReloadFixture {
|
||||
supervisor,
|
||||
control,
|
||||
command,
|
||||
old_runtime,
|
||||
new_runtime,
|
||||
runtime_watch_rx,
|
||||
}
|
||||
}
|
||||
|
||||
struct DropSignal(Arc<Notify>);
|
||||
|
||||
impl Drop for DropSignal {
|
||||
fn drop(&mut self) {
|
||||
self.0.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_gate_proceeds_only_on_verified_match() {
|
||||
assert_eq!(
|
||||
revision_gate_action(
|
||||
"accepted",
|
||||
Ok("accepted".to_string()),
|
||||
ReloadFailurePolicy::Rollback,
|
||||
),
|
||||
RevisionGateAction::Proceed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_gate_applies_failure_policy_to_mismatch_and_read_error() {
|
||||
for result in [Ok("changed".to_string()), Err("read failed".to_string())] {
|
||||
assert!(matches!(
|
||||
revision_gate_action("accepted", result.clone(), ReloadFailurePolicy::KeepNew,),
|
||||
RevisionGateAction::Warn(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
revision_gate_action("accepted", result, ReloadFailurePolicy::Rollback),
|
||||
RevisionGateAction::Rollback(_)
|
||||
));
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
||||
let fixture = fixture(ReloadRequest {
|
||||
failure_policy: ReloadFailurePolicy::Rollback,
|
||||
..ReloadRequest::default()
|
||||
})
|
||||
.await;
|
||||
let candidate_dropped = Arc::new(Notify::new());
|
||||
let candidate_drop = candidate_dropped.clone();
|
||||
assert!(fixture.new_runtime.spawn_session(async move {
|
||||
let _drop_signal = DropSignal(candidate_drop);
|
||||
std::future::pending::<()>().await;
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
fixture
|
||||
.supervisor
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
fixture.old_runtime.clone(),
|
||||
PreparedRuntime {
|
||||
generation: fixture.new_runtime,
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
RevisionGateAction::Rollback("revision changed".to_string()),
|
||||
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
||||
)
|
||||
.await;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), candidate_dropped.notified())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 1);
|
||||
assert_eq!(
|
||||
fixture
|
||||
.runtime_watch_rx
|
||||
.borrow()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.generation_id,
|
||||
1
|
||||
);
|
||||
assert!(fixture.old_runtime.spawn_session(async {}));
|
||||
let status = fixture.control.status(1).await.unwrap();
|
||||
assert_eq!(status.state, ReloadPhase::RolledBack);
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dns_failure_policy_controls_rollback_or_keep_new() {
|
||||
for policy in [ReloadFailurePolicy::Rollback, ReloadFailurePolicy::KeepNew] {
|
||||
let fixture = fixture(ReloadRequest {
|
||||
failure_policy: policy,
|
||||
..ReloadRequest::default()
|
||||
})
|
||||
.await;
|
||||
fixture
|
||||
.supervisor
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
fixture.old_runtime.clone(),
|
||||
PreparedRuntime {
|
||||
generation: fixture.new_runtime.clone(),
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Err("invalid DNS entry".to_string()),
|
||||
)
|
||||
.await;
|
||||
|
||||
let status = fixture.control.status(1).await.unwrap();
|
||||
match policy {
|
||||
ReloadFailurePolicy::Rollback => {
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 1);
|
||||
assert_eq!(status.state, ReloadPhase::RolledBack);
|
||||
assert!(fixture.old_runtime.spawn_session(async {}));
|
||||
fixture.old_runtime.stop_sessions().await;
|
||||
}
|
||||
ReloadFailurePolicy::KeepNew => {
|
||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 2);
|
||||
assert_eq!(status.state, ReloadPhase::Succeeded);
|
||||
assert_eq!(status.warnings.len(), 1);
|
||||
assert!(!fixture.old_runtime.spawn_session(async {}));
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||
let mut fixture = fixture(ReloadRequest {
|
||||
mode: ReloadMode::Drain,
|
||||
timeout_secs: Some(30),
|
||||
..ReloadRequest::default()
|
||||
})
|
||||
.await;
|
||||
let old_started = Arc::new(Notify::new());
|
||||
let old_release = Arc::new(Notify::new());
|
||||
let started = old_started.clone();
|
||||
let release = old_release.clone();
|
||||
assert!(fixture.old_runtime.spawn_session(async move {
|
||||
started.notify_one();
|
||||
release.notified().await;
|
||||
}));
|
||||
old_started.notified().await;
|
||||
|
||||
let supervisor = fixture.supervisor.clone();
|
||||
let old_runtime = fixture.old_runtime.clone();
|
||||
let new_runtime = fixture.new_runtime.clone();
|
||||
let activation = tokio::spawn(async move {
|
||||
supervisor
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
old_runtime,
|
||||
PreparedRuntime {
|
||||
generation: new_runtime,
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
|
||||
fixture.runtime_watch_rx.changed().await.unwrap();
|
||||
assert_eq!(
|
||||
fixture
|
||||
.runtime_watch_rx
|
||||
.borrow()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.generation_id,
|
||||
2
|
||||
);
|
||||
assert!(!activation.is_finished());
|
||||
assert!(!fixture.old_runtime.spawn_session(async {}));
|
||||
|
||||
old_release.notify_one();
|
||||
activation.await.unwrap();
|
||||
assert_eq!(
|
||||
fixture.control.status(1).await.unwrap().state,
|
||||
ReloadPhase::Succeeded
|
||||
);
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
||||
let mut fixture = fixture(ReloadRequest {
|
||||
mode: ReloadMode::Drain,
|
||||
timeout_secs: Some(1),
|
||||
..ReloadRequest::default()
|
||||
})
|
||||
.await;
|
||||
let dropped = Arc::new(Notify::new());
|
||||
let drop_signal = dropped.clone();
|
||||
assert!(fixture.old_runtime.spawn_session(async move {
|
||||
let _drop_signal = DropSignal(drop_signal);
|
||||
std::future::pending::<()>().await;
|
||||
}));
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let supervisor = fixture.supervisor.clone();
|
||||
let old_runtime = fixture.old_runtime.clone();
|
||||
let new_runtime = fixture.new_runtime.clone();
|
||||
let activation = tokio::spawn(async move {
|
||||
supervisor
|
||||
.activate_prepared(
|
||||
fixture.command,
|
||||
old_runtime,
|
||||
PreparedRuntime {
|
||||
generation: new_runtime,
|
||||
detected_ips: (None, None),
|
||||
},
|
||||
RevisionGateAction::Proceed,
|
||||
|_| Ok(()),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
fixture.runtime_watch_rx.changed().await.unwrap();
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
activation.await.unwrap();
|
||||
|
||||
dropped.notified().await;
|
||||
let status = fixture.control.status(1).await.unwrap();
|
||||
assert_eq!(status.state, ReloadPhase::Succeeded);
|
||||
assert_eq!(status.warnings.len(), 1);
|
||||
assert!(status.warnings[0].contains("exceeded drain timeout"));
|
||||
fixture.new_runtime.stop_sessions().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
||||
let runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||
let active_runtime = Arc::new(ArcSwap::from(runtime.clone()));
|
||||
let (control, commands) = ReloadControl::channel(runtime.id);
|
||||
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
||||
let (runtime_watch_tx, _runtime_watch_rx) = watch::channel(Some(runtime.watch_state()));
|
||||
let handle = ReloadSupervisor::spawn(
|
||||
active_runtime,
|
||||
control.clone(),
|
||||
commands,
|
||||
PathBuf::new(),
|
||||
Arc::new(QuotaStore::default()),
|
||||
detected_ips_tx,
|
||||
runtime_log_filter(),
|
||||
runtime_watch_tx,
|
||||
);
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
|
||||
.await
|
||||
.unwrap();
|
||||
let result = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"revision".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result, Err(ReloadSubmitError::MaestroUnavailable));
|
||||
runtime.stop_sessions().await;
|
||||
}
|
||||
256
src/maestro/reload_tests.rs
Normal file
256
src/maestro/reload_tests.rs
Normal file
@@ -0,0 +1,256 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn request_defaults_to_instant_keep_new() {
|
||||
let request: ReloadRequest = serde_json::from_str("{}").unwrap();
|
||||
assert_eq!(request, ReloadRequest::default());
|
||||
assert_eq!(request.validate(), Ok(()));
|
||||
}
|
||||
#[test]
|
||||
fn drain_requires_bounded_timeout() {
|
||||
let missing = ReloadRequest {
|
||||
mode: ReloadMode::Drain,
|
||||
..ReloadRequest::default()
|
||||
};
|
||||
assert!(missing.validate().is_err());
|
||||
let valid = ReloadRequest {
|
||||
mode: ReloadMode::Drain,
|
||||
timeout_secs: Some(30),
|
||||
..ReloadRequest::default()
|
||||
};
|
||||
assert_eq!(valid.validate(), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn patch_query_parses_reload_policy() {
|
||||
let request =
|
||||
ReloadRequest::from_query(Some("reload=drain&timeout_secs=30&failure_policy=rollback"))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(request.mode, ReloadMode::Drain);
|
||||
assert_eq!(request.timeout_secs, Some(30));
|
||||
assert_eq!(request.failure_policy, ReloadFailurePolicy::Rollback);
|
||||
assert!(ReloadRequest::from_query(Some("timeout_secs=30")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_uses_documented_deferred_process_fields_key() {
|
||||
let status = ReloadStatus {
|
||||
reload_id: 1,
|
||||
target_generation: 2,
|
||||
config_revision: "revision".to_string(),
|
||||
state: ReloadPhase::Succeeded,
|
||||
mode: ReloadMode::Instant,
|
||||
failure_policy: ReloadFailurePolicy::KeepNew,
|
||||
requested_at_epoch_secs: 10,
|
||||
started_at_epoch_secs: Some(11),
|
||||
finished_at_epoch_secs: Some(12),
|
||||
deferred_fields: vec!["server.listeners".to_string()],
|
||||
warnings: Vec::new(),
|
||||
error: None,
|
||||
};
|
||||
let value = serde_json::to_value(status).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
value["deferred_process_fields"],
|
||||
serde_json::json!(["server.listeners"])
|
||||
);
|
||||
assert!(value.get("deferred_fields").is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn coordinator_rejects_concurrent_reload_and_releases_terminal_slot() {
|
||||
let (control, mut receiver) = ReloadControl::channel(1);
|
||||
let first = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-1".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
let second = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-2".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(second, Err(ReloadSubmitError::InProgress(first.reload_id)));
|
||||
control
|
||||
.succeed(first.reload_id, first.target_generation)
|
||||
.await;
|
||||
let third = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-3".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(third.reload_id, first.reload_id + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_outcomes_release_slot_and_only_success_advances_generation() {
|
||||
let (control, mut receiver) = ReloadControl::channel(7);
|
||||
|
||||
let failed = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-failed".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
control
|
||||
.mark_phase(failed.reload_id, ReloadPhase::Preparing)
|
||||
.await;
|
||||
control.fail(failed.reload_id, "prepare failed").await;
|
||||
let failed_status = control.status(failed.reload_id).await.unwrap();
|
||||
assert_eq!(failed_status.state, ReloadPhase::Failed);
|
||||
assert_eq!(failed_status.error.as_deref(), Some("prepare failed"));
|
||||
assert!(failed_status.started_at_epoch_secs.is_some());
|
||||
assert!(failed_status.finished_at_epoch_secs.is_some());
|
||||
|
||||
let rolled_back = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-rollback".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
assert_eq!(rolled_back.target_generation, 8);
|
||||
control
|
||||
.rolled_back(rolled_back.reload_id, "revision changed")
|
||||
.await;
|
||||
|
||||
let succeeded = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-success".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
assert_eq!(succeeded.target_generation, 8);
|
||||
control
|
||||
.succeed(succeeded.reload_id, succeeded.target_generation)
|
||||
.await;
|
||||
|
||||
let next = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-next".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(next.target_generation, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_success_cannot_advance_generation_or_release_active_reload() {
|
||||
let (control, mut receiver) = ReloadControl::channel(3);
|
||||
let active = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-active".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
|
||||
control.succeed(active.reload_id + 100, 99).await;
|
||||
|
||||
assert_eq!(control.in_progress().await, Some(active.reload_id));
|
||||
control.fail(active.reload_id, "expected failure").await;
|
||||
let next = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-next".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(next.target_generation, 4);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn status_history_retains_only_the_latest_entries() {
|
||||
let (control, mut receiver) = ReloadControl::channel(1);
|
||||
let mut reload_ids = Vec::new();
|
||||
for index in 0..=RELOAD_HISTORY_CAPACITY {
|
||||
let accepted = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
format!("rev-{index}"),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
reload_ids.push(accepted.reload_id);
|
||||
control.fail(accepted.reload_id, "expected failure").await;
|
||||
}
|
||||
|
||||
assert!(control.status(reload_ids[0]).await.is_none());
|
||||
assert!(control.status(reload_ids[1]).await.is_some());
|
||||
assert!(control.status(*reload_ids.last().unwrap()).await.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_command_channel_marks_reload_failed_and_releases_slot() {
|
||||
let (control, receiver) = ReloadControl::channel(1);
|
||||
drop(receiver);
|
||||
|
||||
let result = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-closed".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result, Err(ReloadSubmitError::MaestroUnavailable));
|
||||
assert_eq!(control.in_progress().await, None);
|
||||
let status = control.status(1).await.unwrap();
|
||||
assert_eq!(status.state, ReloadPhase::Failed);
|
||||
assert_eq!(
|
||||
status.error.as_deref(),
|
||||
Some("maestro command channel is closed")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_gate_rejects_new_commands_without_disturbing_active_status() {
|
||||
let (control, mut receiver) = ReloadControl::channel(4);
|
||||
let active = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-active".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let _command = receiver.recv().await.unwrap();
|
||||
|
||||
control.begin_shutdown().await;
|
||||
let rejected = control
|
||||
.submit(
|
||||
Arc::new(ProxyConfig::default()),
|
||||
"rev-rejected".to_string(),
|
||||
ReloadRequest::default(),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(rejected, Err(ReloadSubmitError::MaestroUnavailable));
|
||||
assert_eq!(control.in_progress().await, Some(active.reload_id));
|
||||
control.fail(active.reload_id, "shutdown test").await;
|
||||
}
|
||||
383
src/maestro/runtime_build.rs
Normal file
383
src/maestro/runtime_build.rs
Normal file
@@ -0,0 +1,383 @@
|
||||
use std::net::IpAddr;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tokio::sync::{RwLock, Semaphore, watch};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::network::probe::{decide_network_capabilities, run_probe};
|
||||
use crate::proxy::direct_buffer_budget::{
|
||||
DirectBufferBudget, resolve_direct_buffer_hard_limit, run_direct_buffer_budget_controller,
|
||||
};
|
||||
use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController};
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::startup::StartupTracker;
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{QuotaStore, ReplayChecker, Stats};
|
||||
use crate::stream::BufferPool;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
use super::admission;
|
||||
use super::generation::{RuntimeGeneration, RuntimeTaskScope};
|
||||
use super::runtime_tasks::RuntimeLogFilter;
|
||||
use super::{me_startup, runtime_tasks, tls_bootstrap};
|
||||
|
||||
pub(crate) struct PreparedRuntime {
|
||||
pub(crate) generation: Arc<RuntimeGeneration>,
|
||||
pub(crate) detected_ips: (Option<IpAddr>, Option<IpAddr>),
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_runtime(
|
||||
generation_id: u64,
|
||||
config: ProxyConfig,
|
||||
config_path: &Path,
|
||||
quota_store: Arc<QuotaStore>,
|
||||
runtime_log_filter: RuntimeLogFilter,
|
||||
) -> Result<PreparedRuntime, String> {
|
||||
let started_at_epoch_secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
let startup_tracker = Arc::new(StartupTracker::new(started_at_epoch_secs));
|
||||
let task_scope = RuntimeTaskScope::new();
|
||||
let stats = Arc::new(Stats::with_quota_store(quota_store));
|
||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||
|
||||
let upstream_manager = Arc::new(
|
||||
UpstreamManager::new(
|
||||
config.upstreams.clone(),
|
||||
config.general.upstream_connect_retry_attempts,
|
||||
config.general.upstream_connect_retry_backoff_ms,
|
||||
config.general.upstream_connect_budget_ms,
|
||||
config.general.tg_connect,
|
||||
config.general.upstream_unhealthy_fail_threshold,
|
||||
config.general.upstream_connect_failfast_hard_errors,
|
||||
stats.clone(),
|
||||
)
|
||||
.with_dns_overrides(&config.network.dns_overrides)
|
||||
.map_err(|error| format!("DNS override preparation failed: {}", error))?,
|
||||
);
|
||||
let ip_tracker = Arc::new(UserIpTracker::new());
|
||||
ip_tracker
|
||||
.load_limits(
|
||||
config.access.user_max_unique_ips_global_each,
|
||||
&config.access.user_max_unique_ips,
|
||||
)
|
||||
.await;
|
||||
ip_tracker
|
||||
.set_limit_policy(
|
||||
config.access.user_max_unique_ips_mode,
|
||||
config.access.user_max_unique_ips_window_secs,
|
||||
)
|
||||
.await;
|
||||
|
||||
let hard_limit =
|
||||
resolve_direct_buffer_hard_limit(config.general.direct_relay_buffer_budget_max_bytes).await;
|
||||
let direct_buffer_budget = DirectBufferBudget::new(hard_limit);
|
||||
let proxy_shared =
|
||||
ProxySharedState::new_with_direct_buffer_budget(direct_buffer_budget.clone());
|
||||
proxy_shared.apply_user_enabled_config(&config.access.user_enabled);
|
||||
proxy_shared.traffic_limiter.apply_policy(
|
||||
config.access.user_rate_limits.clone(),
|
||||
config.access.cidr_rate_limits.clone(),
|
||||
);
|
||||
|
||||
let probe = run_probe(
|
||||
&config.network,
|
||||
&config.upstreams,
|
||||
config.general.middle_proxy_nat_probe,
|
||||
config.general.stun_nat_probe_concurrency,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| format!("network probe failed: {}", error))?;
|
||||
let decision =
|
||||
decide_network_capabilities(&config.network, &probe, config.general.middle_proxy_nat_ip);
|
||||
let prefer_ipv6 = decision.prefer_ipv6();
|
||||
|
||||
let mut tls_domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
||||
tls_domains.push(config.censorship.tls_domain.clone());
|
||||
for domain in &config.censorship.tls_domains {
|
||||
if !tls_domains.contains(domain) {
|
||||
tls_domains.push(domain.clone());
|
||||
}
|
||||
}
|
||||
let tls_cache = tls_bootstrap::bootstrap_tls_front(
|
||||
&config,
|
||||
&tls_domains,
|
||||
upstream_manager.clone(),
|
||||
&startup_tracker,
|
||||
task_scope.clone(),
|
||||
tls_bootstrap::TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
||||
let beobachten = Arc::new(BeobachtenStore::new());
|
||||
let rng = Arc::new(SecureRandom::new());
|
||||
let route_mode = if !config.general.use_middle_proxy || config.general.me2dc_fallback {
|
||||
RelayRouteMode::Direct
|
||||
} else {
|
||||
RelayRouteMode::Middle
|
||||
};
|
||||
let route_runtime = Arc::new(RouteRuntimeController::new(route_mode));
|
||||
let me_pool_runtime = Arc::new(RwLock::new(None::<Arc<MePool>>));
|
||||
let (me_ready_tx, me_ready_rx) = watch::channel(0_u64);
|
||||
let direct_first_startup = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||
let me_pool = if direct_first_startup {
|
||||
None
|
||||
} else {
|
||||
me_startup::initialize_me_pool(
|
||||
config.general.use_middle_proxy,
|
||||
&config,
|
||||
&decision,
|
||||
&probe,
|
||||
&startup_tracker,
|
||||
upstream_manager.clone(),
|
||||
rng.clone(),
|
||||
stats.clone(),
|
||||
me_pool_runtime.clone(),
|
||||
me_ready_tx.clone(),
|
||||
task_scope.clone(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
if strict_middle_proxy_unavailable(
|
||||
config.general.use_middle_proxy,
|
||||
direct_first_startup,
|
||||
me_pool.is_some(),
|
||||
) {
|
||||
task_scope.stop().await;
|
||||
return Err(
|
||||
"Middle-End pool is required but did not become ready during reload preparation"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let config = Arc::new(config);
|
||||
let replay_checker = Arc::new(ReplayChecker::new(
|
||||
config.access.replay_check_len,
|
||||
Duration::from_secs(config.access.replay_window_secs),
|
||||
));
|
||||
let buffer_pool = Arc::new(BufferPool::with_config(64 * 1024, 4096));
|
||||
let max_connections_limit = if config.server.max_connections == 0 {
|
||||
Semaphore::MAX_PERMITS
|
||||
} else {
|
||||
config.server.max_connections as usize
|
||||
};
|
||||
let max_connections = Arc::new(Semaphore::new(max_connections_limit));
|
||||
let watches = runtime_tasks::spawn_runtime_tasks(
|
||||
&config,
|
||||
config_path,
|
||||
&probe,
|
||||
prefer_ipv6,
|
||||
decision.ipv4_dc,
|
||||
decision.ipv6_dc,
|
||||
&startup_tracker,
|
||||
stats.clone(),
|
||||
upstream_manager.clone(),
|
||||
replay_checker.clone(),
|
||||
me_pool.clone(),
|
||||
rng.clone(),
|
||||
ip_tracker.clone(),
|
||||
beobachten.clone(),
|
||||
me_pool.clone(),
|
||||
proxy_shared.clone(),
|
||||
me_ready_tx.clone(),
|
||||
task_scope.clone(),
|
||||
)
|
||||
.await;
|
||||
let config_rx = watches.config_rx;
|
||||
runtime_log_filter.spawn_watcher(watches.log_level_rx, task_scope.clone());
|
||||
let initial_admission_open = !config.general.use_middle_proxy || me_pool.is_some();
|
||||
let (admission_tx, admission_rx) = watch::channel(initial_admission_open);
|
||||
admission::configure_admission_gate(
|
||||
&config,
|
||||
me_pool.clone(),
|
||||
me_pool_runtime.clone(),
|
||||
route_runtime.clone(),
|
||||
&admission_tx,
|
||||
config_rx.clone(),
|
||||
me_ready_rx,
|
||||
task_scope.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
if direct_first_startup {
|
||||
let config_bg = config.clone();
|
||||
let decision_bg = decision.clone();
|
||||
let probe_bg = probe.clone();
|
||||
let startup_tracker_bg = startup_tracker.clone();
|
||||
let upstream_manager_bg = upstream_manager.clone();
|
||||
let rng_bg = rng.clone();
|
||||
let stats_bg = stats.clone();
|
||||
let me_pool_runtime_bg = me_pool_runtime.clone();
|
||||
let me_ready_tx_bg = me_ready_tx.clone();
|
||||
let config_rx_bg = config_rx.clone();
|
||||
let task_scope_bg = task_scope.clone();
|
||||
let retry_limit = config.general.me_init_retry_attempts;
|
||||
task_scope.spawn(async move {
|
||||
let mut attempt = 0_u32;
|
||||
loop {
|
||||
attempt = attempt.saturating_add(1);
|
||||
let pool = me_startup::initialize_me_pool(
|
||||
true,
|
||||
config_bg.as_ref(),
|
||||
&decision_bg,
|
||||
&probe_bg,
|
||||
&startup_tracker_bg,
|
||||
upstream_manager_bg.clone(),
|
||||
rng_bg.clone(),
|
||||
stats_bg.clone(),
|
||||
me_pool_runtime_bg.clone(),
|
||||
me_ready_tx_bg.clone(),
|
||||
task_scope_bg.clone(),
|
||||
)
|
||||
.await;
|
||||
if let Some(pool) = pool {
|
||||
runtime_tasks::spawn_middle_proxy_runtime_tasks(
|
||||
config_bg.as_ref(),
|
||||
config_rx_bg,
|
||||
pool,
|
||||
rng_bg,
|
||||
me_ready_tx_bg,
|
||||
task_scope_bg,
|
||||
);
|
||||
break;
|
||||
}
|
||||
if retry_limit > 0 && attempt >= retry_limit {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let conntrack_scope = task_scope.clone();
|
||||
task_scope.spawn(crate::conntrack_control::run_conntrack_controller(
|
||||
config_rx.clone(),
|
||||
stats.clone(),
|
||||
proxy_shared.clone(),
|
||||
conntrack_scope.cancellation_token(),
|
||||
));
|
||||
task_scope.spawn(run_direct_buffer_budget_controller(
|
||||
direct_buffer_budget,
|
||||
buffer_pool.clone(),
|
||||
stats.clone(),
|
||||
proxy_shared.clone(),
|
||||
config.server.max_connections,
|
||||
));
|
||||
let generation = RuntimeGeneration::new(
|
||||
generation_id,
|
||||
config_rx,
|
||||
admission_rx,
|
||||
stats,
|
||||
upstream_manager,
|
||||
replay_checker,
|
||||
buffer_pool,
|
||||
rng,
|
||||
me_pool,
|
||||
me_pool_runtime,
|
||||
route_runtime,
|
||||
tls_cache,
|
||||
ip_tracker,
|
||||
beobachten,
|
||||
proxy_shared,
|
||||
max_connections,
|
||||
task_scope,
|
||||
);
|
||||
drop(admission_tx);
|
||||
|
||||
Ok(PreparedRuntime {
|
||||
generation,
|
||||
detected_ips: (
|
||||
probe.detected_ipv4.map(IpAddr::V4),
|
||||
probe.detected_ipv6.map(IpAddr::V6),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
fn strict_middle_proxy_unavailable(
|
||||
use_middle_proxy: bool,
|
||||
direct_first_startup: bool,
|
||||
pool_available: bool,
|
||||
) -> bool {
|
||||
use_middle_proxy && !direct_first_startup && !pool_available
|
||||
}
|
||||
|
||||
pub(crate) fn deferred_process_fields(old: &ProxyConfig, new: &ProxyConfig) -> Vec<String> {
|
||||
let mut fields = Vec::new();
|
||||
if old.server.port != new.server.port
|
||||
|| old.server.proxy_protocol != new.server.proxy_protocol
|
||||
|| old.server.listen_backlog != new.server.listen_backlog
|
||||
|| serde_json::to_value(&old.server.listeners).ok()
|
||||
!= serde_json::to_value(&new.server.listeners).ok()
|
||||
{
|
||||
fields.push("server.listeners".to_string());
|
||||
}
|
||||
if old.server.listen_unix_sock != new.server.listen_unix_sock
|
||||
|| old.server.listen_unix_sock_perm != new.server.listen_unix_sock_perm
|
||||
{
|
||||
fields.push("server.listen_unix_sock".to_string());
|
||||
}
|
||||
if old.server.api.listen != new.server.api.listen
|
||||
|| old.server.api.enabled != new.server.api.enabled
|
||||
{
|
||||
fields.push("server.api.listen".to_string());
|
||||
}
|
||||
if old.server.metrics_listen != new.server.metrics_listen
|
||||
|| old.server.metrics_port != new.server.metrics_port
|
||||
{
|
||||
fields.push("server.metrics_listen".to_string());
|
||||
}
|
||||
if old.general.quota_state_path != new.general.quota_state_path {
|
||||
fields.push("general.quota_state_path".to_string());
|
||||
}
|
||||
if old.general.disable_colors != new.general.disable_colors {
|
||||
fields.push("general.disable_colors".to_string());
|
||||
}
|
||||
if old.general.data_path != new.general.data_path {
|
||||
fields.push("general.data_path".to_string());
|
||||
}
|
||||
if serde_json::to_value(&old.logging).ok() != serde_json::to_value(&new.logging).ok() {
|
||||
fields.push("logging".to_string());
|
||||
}
|
||||
fields
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn process_socket_and_logging_changes_are_deferred() {
|
||||
let old = ProxyConfig::default();
|
||||
let mut new = old.clone();
|
||||
new.server.listen_backlog = new.server.listen_backlog.saturating_add(1);
|
||||
new.general.disable_colors = !new.general.disable_colors;
|
||||
|
||||
let fields = deferred_process_fields(&old, &new);
|
||||
assert!(fields.contains(&"server.listeners".to_string()));
|
||||
assert!(fields.contains(&"general.disable_colors".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_only_change_does_not_require_process_rebind() {
|
||||
let old = ProxyConfig::default();
|
||||
let mut new = old.clone();
|
||||
new.censorship.tls_domain = "reload.example".to_string();
|
||||
assert!(deferred_process_fields(&old, &new).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_middle_proxy_requires_a_prepared_pool() {
|
||||
assert!(strict_middle_proxy_unavailable(true, false, false));
|
||||
assert!(!strict_middle_proxy_unavailable(true, false, true));
|
||||
assert!(!strict_middle_proxy_unavailable(true, true, false));
|
||||
assert!(!strict_middle_proxy_unavailable(false, false, false));
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,11 @@ use std::net::IpAddr;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::Registry;
|
||||
use tracing_subscriber::reload;
|
||||
|
||||
use crate::config::hot_reload::spawn_config_watcher;
|
||||
@@ -21,10 +23,11 @@ use crate::startup::{
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::telemetry::TelemetryPolicy;
|
||||
use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::tls_front::TlsFrontCache;
|
||||
use crate::transport::UpstreamManager;
|
||||
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
||||
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::generation::RuntimeTaskScope;
|
||||
use super::helpers::write_beobachten_snapshot;
|
||||
|
||||
pub(crate) struct RuntimeWatches {
|
||||
@@ -34,6 +37,56 @@ pub(crate) struct RuntimeWatches {
|
||||
pub(crate) detected_ip_v6: Option<IpAddr>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct RuntimeLogFilter {
|
||||
handle: reload::Handle<EnvFilter, Registry>,
|
||||
}
|
||||
|
||||
impl RuntimeLogFilter {
|
||||
pub(crate) fn new(handle: reload::Handle<EnvFilter, Registry>) -> Self {
|
||||
Self { handle }
|
||||
}
|
||||
|
||||
pub(crate) fn start(
|
||||
&self,
|
||||
has_rust_log: bool,
|
||||
effective_log_level: &LogLevel,
|
||||
log_level_rx: watch::Receiver<LogLevel>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
) {
|
||||
self.apply(effective_log_level, has_rust_log);
|
||||
self.spawn_watcher(log_level_rx, task_scope);
|
||||
}
|
||||
|
||||
pub(crate) fn apply_reload(&self, level: &LogLevel) {
|
||||
self.apply(level, false);
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_watcher(
|
||||
&self,
|
||||
mut log_level_rx: watch::Receiver<LogLevel>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
) {
|
||||
let filter = self.clone();
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
if log_level_rx.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
let level = log_level_rx.borrow_and_update().clone();
|
||||
filter.apply_reload(&level);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn apply(&self, level: &LogLevel, has_rust_log: bool) {
|
||||
let runtime_filter = EnvFilter::new(log_filter_spec(has_rust_log, level));
|
||||
if let Err(error) = self.handle.reload(runtime_filter) {
|
||||
tracing::error!(error = %error, "Failed to update runtime log filter");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn spawn_runtime_tasks(
|
||||
config: &Arc<ProxyConfig>,
|
||||
@@ -50,14 +103,14 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
rng: Arc<SecureRandom>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
|
||||
me_pool_for_policy: Option<Arc<MePool>>,
|
||||
shared_state: Arc<ProxySharedState>,
|
||||
me_ready_tx: watch::Sender<u64>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
) -> RuntimeWatches {
|
||||
let um_clone = upstream_manager.clone();
|
||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
um_clone
|
||||
.run_health_checks(
|
||||
prefer_ipv6,
|
||||
@@ -69,19 +122,19 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
});
|
||||
|
||||
let rc_clone = replay_checker.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
rc_clone.run_periodic_cleanup().await;
|
||||
});
|
||||
|
||||
let stats_maintenance = stats.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
stats_maintenance
|
||||
.run_periodic_user_stats_maintenance()
|
||||
.await;
|
||||
});
|
||||
|
||||
let ip_tracker_maintenance = ip_tracker.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
ip_tracker_maintenance.run_periodic_maintenance().await;
|
||||
});
|
||||
|
||||
@@ -104,6 +157,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
config.clone(),
|
||||
detected_ip_v4,
|
||||
detected_ip_v6,
|
||||
task_scope.cancellation_token(),
|
||||
);
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
@@ -111,21 +165,10 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
Some("config hot-reload watcher started".to_string()),
|
||||
)
|
||||
.await;
|
||||
let mut config_rx_api_bridge = config_rx.clone();
|
||||
let api_config_tx_bridge = api_config_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
if config_rx_api_bridge.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
let cfg = config_rx_api_bridge.borrow_and_update().clone();
|
||||
api_config_tx_bridge.send_replace(cfg);
|
||||
}
|
||||
});
|
||||
|
||||
let stats_policy = stats.clone();
|
||||
let upstream_policy = upstream_manager.clone();
|
||||
let mut config_rx_policy = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
if config_rx_policy.changed().await.is_err() {
|
||||
break;
|
||||
@@ -133,6 +176,9 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
let cfg = config_rx_policy.borrow_and_update().clone();
|
||||
stats_policy
|
||||
.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
|
||||
if let Err(error) = upstream_policy.update_dns_overrides(&cfg.network.dns_overrides) {
|
||||
warn!(error = %error, "Failed to update generation DNS overrides");
|
||||
}
|
||||
if let Some(pool) = &me_pool_for_policy {
|
||||
pool.update_runtime_transport_policy(
|
||||
cfg.general.me_socks_kdf_policy,
|
||||
@@ -149,7 +195,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
|
||||
let ip_tracker_policy = ip_tracker.clone();
|
||||
let mut config_rx_ip_limits = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
let mut prev_limits = config_rx_ip_limits
|
||||
.borrow()
|
||||
.access
|
||||
@@ -205,7 +251,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
config.access.cidr_rate_limits.clone(),
|
||||
);
|
||||
let mut config_rx_rate_limits = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
let mut prev_user_limits = config_rx_rate_limits
|
||||
.borrow()
|
||||
.access
|
||||
@@ -234,9 +280,30 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
}
|
||||
});
|
||||
|
||||
let shared_user_enabled = shared_state.clone();
|
||||
let mut config_rx_user_enabled = config_rx.clone();
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
if config_rx_user_enabled.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
let cfg = config_rx_user_enabled.borrow_and_update().clone();
|
||||
for user in shared_user_enabled.apply_user_enabled_config(&cfg.access.user_enabled) {
|
||||
let cancelled = shared_user_enabled.cancel_user_sessions(&user);
|
||||
if cancelled > 0 {
|
||||
info!(
|
||||
user = %user,
|
||||
cancelled,
|
||||
"Disabled user sessions cancelled after config reload"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let beobachten_writer = beobachten.clone();
|
||||
let config_rx_beobachten = config_rx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
let cfg = config_rx_beobachten.borrow().clone();
|
||||
let sleep_secs = cfg.general.beobachten_flush_secs.max(1);
|
||||
@@ -257,7 +324,14 @@ pub(crate) async fn spawn_runtime_tasks(
|
||||
});
|
||||
|
||||
if let Some(pool) = me_pool {
|
||||
spawn_middle_proxy_runtime_tasks(config, config_rx.clone(), pool, rng, me_ready_tx);
|
||||
spawn_middle_proxy_runtime_tasks(
|
||||
config,
|
||||
config_rx.clone(),
|
||||
pool,
|
||||
rng,
|
||||
me_ready_tx,
|
||||
task_scope,
|
||||
);
|
||||
}
|
||||
|
||||
RuntimeWatches {
|
||||
@@ -274,6 +348,7 @@ pub(crate) fn spawn_middle_proxy_runtime_tasks(
|
||||
pool: Arc<MePool>,
|
||||
rng: Arc<SecureRandom>,
|
||||
me_ready_tx: watch::Sender<u64>,
|
||||
task_scope: RuntimeTaskScope,
|
||||
) {
|
||||
let reinit_trigger_capacity = config.general.me_reinit_trigger_channel.max(1);
|
||||
let (reinit_tx, reinit_rx) = mpsc::channel::<MeReinitTrigger>(reinit_trigger_capacity);
|
||||
@@ -282,7 +357,7 @@ pub(crate) fn spawn_middle_proxy_runtime_tasks(
|
||||
let rng_clone_sched = rng.clone();
|
||||
let config_rx_clone_sched = config_rx.clone();
|
||||
let me_ready_tx_sched = me_ready_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
crate::transport::middle_proxy::me_reinit_scheduler(
|
||||
pool_clone_sched,
|
||||
rng_clone_sched,
|
||||
@@ -296,7 +371,7 @@ pub(crate) fn spawn_middle_proxy_runtime_tasks(
|
||||
let pool_clone = pool.clone();
|
||||
let config_rx_clone = config_rx.clone();
|
||||
let reinit_tx_updater = reinit_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
crate::transport::middle_proxy::me_config_updater(
|
||||
pool_clone,
|
||||
config_rx_clone,
|
||||
@@ -307,37 +382,12 @@ pub(crate) fn spawn_middle_proxy_runtime_tasks(
|
||||
|
||||
let config_rx_clone_rot = config_rx.clone();
|
||||
let reinit_tx_rotation = reinit_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
task_scope.spawn(async move {
|
||||
crate::transport::middle_proxy::me_rotation_task(config_rx_clone_rot, reinit_tx_rotation)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_runtime_log_filter(
|
||||
has_rust_log: bool,
|
||||
effective_log_level: &LogLevel,
|
||||
filter_handle: reload::Handle<EnvFilter, tracing_subscriber::Registry>,
|
||||
mut log_level_rx: watch::Receiver<LogLevel>,
|
||||
) {
|
||||
let runtime_filter = EnvFilter::new(log_filter_spec(has_rust_log, effective_log_level));
|
||||
filter_handle
|
||||
.reload(runtime_filter)
|
||||
.expect("Failed to switch log filter");
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
if log_level_rx.changed().await.is_err() {
|
||||
break;
|
||||
}
|
||||
let level = log_level_rx.borrow_and_update().clone();
|
||||
let new_filter = tracing_subscriber::EnvFilter::new(log_filter_spec(false, &level));
|
||||
if let Err(e) = filter_handle.reload(new_filter) {
|
||||
tracing::error!("config reload: failed to update log filter: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn log_filter_spec(has_rust_log: bool, effective_log_level: &LogLevel) -> String {
|
||||
if has_rust_log {
|
||||
std::env::var("RUST_LOG")
|
||||
@@ -352,12 +402,7 @@ pub(crate) fn log_filter_spec(has_rust_log: bool, effective_log_level: &LogLevel
|
||||
pub(crate) async fn spawn_metrics_if_configured(
|
||||
config: &Arc<ProxyConfig>,
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
stats: Arc<Stats>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared_state: Arc<ProxySharedState>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
) {
|
||||
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
||||
let metrics_target: Option<(u16, Option<String>)> =
|
||||
@@ -387,28 +432,10 @@ pub(crate) async fn spawn_metrics_if_configured(
|
||||
Some(format!("spawn metrics endpoint on {}", label)),
|
||||
)
|
||||
.await;
|
||||
let stats = stats.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared_state = shared_state.clone();
|
||||
let config_rx_metrics = config_rx.clone();
|
||||
let ip_tracker_metrics = ip_tracker.clone();
|
||||
let tls_cache_metrics = tls_cache.clone();
|
||||
let whitelist = config.server.metrics_whitelist.clone();
|
||||
let active_runtime = active_runtime.clone();
|
||||
let listen_backlog = config.server.listen_backlog;
|
||||
tokio::spawn(async move {
|
||||
metrics::serve(
|
||||
port,
|
||||
listen,
|
||||
listen_backlog,
|
||||
stats,
|
||||
beobachten,
|
||||
shared_state,
|
||||
ip_tracker_metrics,
|
||||
tls_cache_metrics,
|
||||
config_rx_metrics,
|
||||
whitelist,
|
||||
)
|
||||
.await;
|
||||
metrics::serve(port, listen, listen_backlog, active_runtime).await;
|
||||
});
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
|
||||
@@ -12,16 +12,18 @@ use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
#[cfg(not(unix))]
|
||||
use tokio::signal;
|
||||
#[cfg(unix)]
|
||||
use tokio::signal::unix::{SignalKind, signal};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::stats::Stats;
|
||||
use crate::transport::middle_proxy::MePool;
|
||||
|
||||
use super::generation::RuntimeGeneration;
|
||||
use super::helpers::{format_uptime, unit_label};
|
||||
use super::reload_supervisor::ReloadSupervisorHandle;
|
||||
use crate::stats::Stats;
|
||||
use crate::synlimit_control;
|
||||
|
||||
/// Signal that triggered shutdown.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -47,17 +49,19 @@ impl std::fmt::Display for ShutdownSignal {
|
||||
/// Waits for a shutdown signal and performs graceful shutdown.
|
||||
pub(crate) async fn wait_for_shutdown(
|
||||
process_started_at: Instant,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
stats: Arc<Stats>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
quota_state_path: PathBuf,
|
||||
synlimit_controller: synlimit_control::SynlimitController,
|
||||
reload_supervisor: ReloadSupervisorHandle,
|
||||
) {
|
||||
let signal = wait_for_shutdown_signal().await;
|
||||
perform_shutdown(
|
||||
signal,
|
||||
process_started_at,
|
||||
me_pool,
|
||||
&stats,
|
||||
active_runtime,
|
||||
quota_state_path,
|
||||
synlimit_controller,
|
||||
reload_supervisor,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -86,13 +90,18 @@ async fn wait_for_shutdown_signal() -> ShutdownSignal {
|
||||
async fn perform_shutdown(
|
||||
signal: ShutdownSignal,
|
||||
process_started_at: Instant,
|
||||
me_pool: Option<Arc<MePool>>,
|
||||
stats: &Stats,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
quota_state_path: PathBuf,
|
||||
synlimit_controller: synlimit_control::SynlimitController,
|
||||
reload_supervisor: ReloadSupervisorHandle,
|
||||
) {
|
||||
let shutdown_started_at = Instant::now();
|
||||
info!(signal = %signal, "Received shutdown signal");
|
||||
|
||||
reload_supervisor.quiesce().await;
|
||||
let runtime = active_runtime.load_full();
|
||||
let stats = runtime.stats.as_ref();
|
||||
|
||||
// Dump stats if SIGQUIT
|
||||
if signal == ShutdownSignal::Quit {
|
||||
dump_stats(stats, process_started_at);
|
||||
@@ -103,7 +112,9 @@ async fn perform_shutdown(
|
||||
info!("Uptime: {}", format_uptime(uptime_secs));
|
||||
|
||||
// Graceful ME pool shutdown
|
||||
if let Some(pool) = &me_pool {
|
||||
runtime.stop_sessions().await;
|
||||
runtime.stop_background_tasks().await;
|
||||
if let Some(pool) = runtime.current_me_pool().await {
|
||||
match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
||||
.await
|
||||
{
|
||||
@@ -119,6 +130,11 @@ async fn perform_shutdown(
|
||||
}
|
||||
}
|
||||
|
||||
synlimit_controller.shutdown().await;
|
||||
if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
|
||||
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
|
||||
}
|
||||
|
||||
match crate::quota_state::save_quota_state("a_state_path, stats).await {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
@@ -186,7 +202,10 @@ fn dump_stats(stats: &Stats, process_started_at: Instant) {
|
||||
/// - SIGUSR1: Log rotation acknowledgment (for external log rotation tools)
|
||||
/// - SIGUSR2: Dump runtime status to log
|
||||
#[cfg(unix)]
|
||||
pub(crate) fn spawn_signal_handlers(stats: Arc<Stats>, process_started_at: Instant) {
|
||||
pub(crate) fn spawn_signal_handlers(
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
process_started_at: Instant,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
let mut sigusr1 =
|
||||
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
|
||||
@@ -199,7 +218,8 @@ pub(crate) fn spawn_signal_handlers(stats: Arc<Stats>, process_started_at: Insta
|
||||
handle_sigusr1();
|
||||
}
|
||||
_ = sigusr2.recv() => {
|
||||
handle_sigusr2(&stats, process_started_at);
|
||||
let runtime = active_runtime.load_full();
|
||||
handle_sigusr2(runtime.stats.as_ref(), process_started_at);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -208,7 +228,10 @@ pub(crate) fn spawn_signal_handlers(stats: Arc<Stats>, process_started_at: Insta
|
||||
|
||||
/// No-op on non-Unix platforms.
|
||||
#[cfg(not(unix))]
|
||||
pub(crate) fn spawn_signal_handlers(_stats: Arc<Stats>, _process_started_at: Instant) {
|
||||
pub(crate) fn spawn_signal_handlers(
|
||||
_active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
_process_started_at: Instant,
|
||||
) {
|
||||
// No SIGUSR1/SIGUSR2 on non-Unix
|
||||
}
|
||||
|
||||
|
||||
@@ -5,11 +5,86 @@ use rand::RngExt;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::error::{ProxyError, Result};
|
||||
use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker};
|
||||
use crate::tls_front::TlsFrontCache;
|
||||
use crate::tls_front::fetcher::TlsFetchStrategy;
|
||||
use crate::transport::UpstreamManager;
|
||||
|
||||
use super::generation::RuntimeTaskScope;
|
||||
|
||||
/// Readiness requirement for TLS-front cache initialization.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum TlsBootstrapPolicy {
|
||||
BestEffort,
|
||||
RequireReady,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TlsFetchContext {
|
||||
cache: Arc<TlsFrontCache>,
|
||||
domains: Vec<String>,
|
||||
mask_host: String,
|
||||
primary_domain: String,
|
||||
mask_unix_sock: Option<String>,
|
||||
tls_fetch_scope: Option<String>,
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
strategy: TlsFetchStrategy,
|
||||
port: u16,
|
||||
proxy_protocol: u8,
|
||||
}
|
||||
|
||||
impl TlsFetchContext {
|
||||
async fn fetch_all(&self, failure_message: &'static str) {
|
||||
let mut join = tokio::task::JoinSet::new();
|
||||
for domain in self.domains.clone() {
|
||||
let cache = self.cache.clone();
|
||||
let host = tls_fetch_host_for_domain(&self.mask_host, &self.primary_domain, &domain);
|
||||
let unix_sock = self.mask_unix_sock.clone();
|
||||
let scope = self.tls_fetch_scope.clone();
|
||||
let upstream = self.upstream_manager.clone();
|
||||
let strategy = self.strategy.clone();
|
||||
let port = self.port;
|
||||
let proxy_protocol = self.proxy_protocol;
|
||||
join.spawn(async move {
|
||||
match crate::tls_front::fetcher::fetch_real_tls_with_strategy(
|
||||
&host,
|
||||
port,
|
||||
&domain,
|
||||
&strategy,
|
||||
Some(upstream),
|
||||
scope.as_deref(),
|
||||
proxy_protocol,
|
||||
unix_sock.as_deref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => cache.update_from_fetch(&domain, result).await,
|
||||
Err(error) => warn!(domain = %domain, error = %error, failure_message),
|
||||
}
|
||||
});
|
||||
}
|
||||
while let Some(result) = join.join_next().await {
|
||||
if let Err(error) = result {
|
||||
warn!(error = %error, "TLS emulation fetch task join failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_all_with_budget(&self, phase: &'static str) {
|
||||
if tokio::time::timeout(self.strategy.total_budget, self.fetch_all(phase))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
warn!(
|
||||
phase,
|
||||
timeout_ms = self.strategy.total_budget.as_millis(),
|
||||
"TLS emulation fetch budget exhausted"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tls_fetch_host_for_domain(mask_host: &str, primary_tls_domain: &str, domain: &str) -> String {
|
||||
if mask_host.eq_ignore_ascii_case(primary_tls_domain) {
|
||||
domain.to_string()
|
||||
@@ -18,12 +93,24 @@ fn tls_fetch_host_for_domain(mask_host: &str, primary_tls_domain: &str, domain:
|
||||
}
|
||||
}
|
||||
|
||||
fn readiness_error(default_domains: &[String]) -> Option<String> {
|
||||
(!default_domains.is_empty()).then(|| {
|
||||
format!(
|
||||
"TLS-front profiles are not ready for domains: {}",
|
||||
default_domains.join(", ")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Initializes the TLS-front cache and generation-owned refresh tasks.
|
||||
pub(crate) async fn bootstrap_tls_front(
|
||||
config: &ProxyConfig,
|
||||
tls_domains: &[String],
|
||||
upstream_manager: Arc<UpstreamManager>,
|
||||
startup_tracker: &Arc<StartupTracker>,
|
||||
) -> Option<Arc<TlsFrontCache>> {
|
||||
task_scope: RuntimeTaskScope,
|
||||
policy: TlsBootstrapPolicy,
|
||||
) -> Result<Option<Arc<TlsFrontCache>>> {
|
||||
startup_tracker
|
||||
.start_component(
|
||||
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||
@@ -31,26 +118,38 @@ pub(crate) async fn bootstrap_tls_front(
|
||||
)
|
||||
.await;
|
||||
|
||||
let tls_cache: Option<Arc<TlsFrontCache>> = if config.censorship.tls_emulation {
|
||||
let cache = Arc::new(TlsFrontCache::new(
|
||||
tls_domains,
|
||||
config.censorship.fake_cert_len,
|
||||
&config.censorship.tls_front_dir,
|
||||
));
|
||||
cache.load_from_disk().await;
|
||||
if !config.censorship.tls_emulation {
|
||||
startup_tracker
|
||||
.skip_component(
|
||||
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||
Some("censorship.tls_emulation is false".to_string()),
|
||||
)
|
||||
.await;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let port = config.censorship.mask_port;
|
||||
let proxy_protocol = config.censorship.mask_proxy_protocol;
|
||||
let mask_host = config
|
||||
let cache = Arc::new(TlsFrontCache::new(
|
||||
tls_domains,
|
||||
config.censorship.fake_cert_len,
|
||||
&config.censorship.tls_front_dir,
|
||||
));
|
||||
cache.load_from_disk().await;
|
||||
|
||||
let tls_fetch = config.censorship.tls_fetch.clone();
|
||||
let fetch_context = TlsFetchContext {
|
||||
cache: cache.clone(),
|
||||
domains: tls_domains.to_vec(),
|
||||
mask_host: config
|
||||
.censorship
|
||||
.mask_host
|
||||
.clone()
|
||||
.unwrap_or_else(|| config.censorship.tls_domain.clone());
|
||||
let mask_unix_sock = config.censorship.mask_unix_sock.clone();
|
||||
let tls_fetch_scope = (!config.censorship.tls_fetch_scope.is_empty())
|
||||
.then(|| config.censorship.tls_fetch_scope.clone());
|
||||
let tls_fetch = config.censorship.tls_fetch.clone();
|
||||
let fetch_strategy = TlsFetchStrategy {
|
||||
.unwrap_or_else(|| config.censorship.tls_domain.clone()),
|
||||
primary_domain: config.censorship.tls_domain.clone(),
|
||||
mask_unix_sock: config.censorship.mask_unix_sock.clone(),
|
||||
tls_fetch_scope: (!config.censorship.tls_fetch_scope.is_empty())
|
||||
.then(|| config.censorship.tls_fetch_scope.clone()),
|
||||
upstream_manager,
|
||||
strategy: TlsFetchStrategy {
|
||||
profiles: tls_fetch.profiles,
|
||||
strict_route: tls_fetch.strict_route,
|
||||
attempt_timeout: Duration::from_millis(tls_fetch.attempt_timeout_ms.max(1)),
|
||||
@@ -58,150 +157,110 @@ pub(crate) async fn bootstrap_tls_front(
|
||||
grease_enabled: tls_fetch.grease_enabled,
|
||||
deterministic: tls_fetch.deterministic,
|
||||
profile_cache_ttl: Duration::from_secs(tls_fetch.profile_cache_ttl_secs),
|
||||
};
|
||||
let fetch_timeout = fetch_strategy.total_budget;
|
||||
},
|
||||
port: config.censorship.mask_port,
|
||||
proxy_protocol: config.censorship.mask_proxy_protocol,
|
||||
};
|
||||
|
||||
let cache_initial = cache.clone();
|
||||
let domains_initial = tls_domains.to_vec();
|
||||
let host_initial = mask_host.clone();
|
||||
let primary_initial = config.censorship.tls_domain.clone();
|
||||
let unix_sock_initial = mask_unix_sock.clone();
|
||||
let scope_initial = tls_fetch_scope.clone();
|
||||
let upstream_initial = upstream_manager.clone();
|
||||
let strategy_initial = fetch_strategy.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut join = tokio::task::JoinSet::new();
|
||||
for domain in domains_initial {
|
||||
let cache_domain = cache_initial.clone();
|
||||
let host_domain =
|
||||
tls_fetch_host_for_domain(&host_initial, &primary_initial, &domain);
|
||||
let unix_sock_domain = unix_sock_initial.clone();
|
||||
let scope_domain = scope_initial.clone();
|
||||
let upstream_domain = upstream_initial.clone();
|
||||
let strategy_domain = strategy_initial.clone();
|
||||
join.spawn(async move {
|
||||
match crate::tls_front::fetcher::fetch_real_tls_with_strategy(
|
||||
&host_domain,
|
||||
port,
|
||||
&domain,
|
||||
&strategy_domain,
|
||||
Some(upstream_domain),
|
||||
scope_domain.as_deref(),
|
||||
proxy_protocol,
|
||||
unix_sock_domain.as_deref(),
|
||||
)
|
||||
match policy {
|
||||
TlsBootstrapPolicy::BestEffort => {
|
||||
let initial_fetch = fetch_context.clone();
|
||||
let fake_cert_len = config.censorship.fake_cert_len;
|
||||
task_scope.spawn(async move {
|
||||
initial_fetch
|
||||
.fetch_all_with_budget("TLS emulation initial fetch failed")
|
||||
.await;
|
||||
for domain in initial_fetch
|
||||
.cache
|
||||
.default_profile_domains(&initial_fetch.domains)
|
||||
.await
|
||||
{
|
||||
Ok(res) => cache_domain.update_from_fetch(&domain, res).await,
|
||||
Err(e) => {
|
||||
warn!(domain = %domain, error = %e, "TLS emulation initial fetch failed")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
while let Some(res) = join.join_next().await {
|
||||
if let Err(e) = res {
|
||||
warn!(error = %e, "TLS emulation initial fetch task join failed");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let cache_timeout = cache.clone();
|
||||
let domains_timeout = tls_domains.to_vec();
|
||||
let fake_cert_len = config.censorship.fake_cert_len;
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(fetch_timeout).await;
|
||||
for domain in domains_timeout {
|
||||
let cached = cache_timeout.get(&domain).await;
|
||||
if cached.domain == "default" {
|
||||
{
|
||||
warn!(
|
||||
domain = %domain,
|
||||
timeout_secs = fetch_timeout.as_secs(),
|
||||
timeout_ms = initial_fetch.strategy.total_budget.as_millis(),
|
||||
fake_cert_len,
|
||||
"TLS-front fetch not ready within timeout; using cache/default fake cert fallback"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
TlsBootstrapPolicy::RequireReady => {
|
||||
fetch_context
|
||||
.fetch_all_with_budget("TLS emulation initial fetch failed")
|
||||
.await;
|
||||
let default_domains = cache.default_profile_domains(tls_domains).await;
|
||||
if let Some(error) = readiness_error(&default_domains) {
|
||||
startup_tracker
|
||||
.fail_component(COMPONENT_TLS_FRONT_BOOTSTRAP, Some(error.clone()))
|
||||
.await;
|
||||
return Err(ProxyError::Proxy(error));
|
||||
}
|
||||
});
|
||||
|
||||
let cache_refresh = cache.clone();
|
||||
let domains_refresh = tls_domains.to_vec();
|
||||
let host_refresh = mask_host.clone();
|
||||
let primary_refresh = config.censorship.tls_domain.clone();
|
||||
let unix_sock_refresh = mask_unix_sock.clone();
|
||||
let scope_refresh = tls_fetch_scope.clone();
|
||||
let upstream_refresh = upstream_manager.clone();
|
||||
let strategy_refresh = fetch_strategy.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let base_secs = rand::rng().random_range(4 * 3600..=6 * 3600);
|
||||
let jitter_secs = rand::rng().random_range(0..=7200);
|
||||
tokio::time::sleep(Duration::from_secs(base_secs + jitter_secs)).await;
|
||||
|
||||
let mut join = tokio::task::JoinSet::new();
|
||||
for domain in domains_refresh.clone() {
|
||||
let cache_domain = cache_refresh.clone();
|
||||
let host_domain =
|
||||
tls_fetch_host_for_domain(&host_refresh, &primary_refresh, &domain);
|
||||
let unix_sock_domain = unix_sock_refresh.clone();
|
||||
let scope_domain = scope_refresh.clone();
|
||||
let upstream_domain = upstream_refresh.clone();
|
||||
let strategy_domain = strategy_refresh.clone();
|
||||
join.spawn(async move {
|
||||
match crate::tls_front::fetcher::fetch_real_tls_with_strategy(
|
||||
&host_domain,
|
||||
port,
|
||||
&domain,
|
||||
&strategy_domain,
|
||||
Some(upstream_domain),
|
||||
scope_domain.as_deref(),
|
||||
proxy_protocol,
|
||||
unix_sock_domain.as_deref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(res) => cache_domain.update_from_fetch(&domain, res).await,
|
||||
Err(e) => {
|
||||
warn!(domain = %domain, error = %e, "TLS emulation refresh failed")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(res) = join.join_next().await {
|
||||
if let Err(e) = res {
|
||||
warn!(error = %e, "TLS emulation refresh task join failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Some(cache)
|
||||
} else {
|
||||
startup_tracker
|
||||
.skip_component(
|
||||
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||
Some("censorship.tls_emulation is false".to_string()),
|
||||
)
|
||||
.await;
|
||||
None
|
||||
};
|
||||
|
||||
if tls_cache.is_some() {
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||
Some("tls front cache is initialized".to_string()),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
tls_cache
|
||||
let refresh_context = fetch_context;
|
||||
task_scope.spawn(async move {
|
||||
loop {
|
||||
let base_secs = rand::rng().random_range(4 * 3600..=6 * 3600);
|
||||
let jitter_secs = rand::rng().random_range(0..=7200);
|
||||
tokio::time::sleep(Duration::from_secs(base_secs + jitter_secs)).await;
|
||||
refresh_context
|
||||
.fetch_all_with_budget("TLS emulation refresh failed")
|
||||
.await;
|
||||
}
|
||||
});
|
||||
|
||||
startup_tracker
|
||||
.complete_component(
|
||||
COMPONENT_TLS_FRONT_BOOTSTRAP,
|
||||
Some("tls front cache is initialized".to_string()),
|
||||
)
|
||||
.await;
|
||||
Ok(Some(cache))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::tls_fetch_host_for_domain;
|
||||
use super::*;
|
||||
use crate::startup::StartupComponentStatus;
|
||||
use crate::stats::Stats;
|
||||
|
||||
fn test_config(cache_dir: &std::path::Path) -> ProxyConfig {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.tls_emulation = true;
|
||||
config.censorship.tls_domain = "front.example".to_string();
|
||||
config.censorship.mask_host = Some("127.0.0.1".to_string());
|
||||
config.censorship.mask_port = 1;
|
||||
config.censorship.tls_front_dir = cache_dir.display().to_string();
|
||||
config.censorship.tls_fetch.profiles.truncate(1);
|
||||
config.censorship.tls_fetch.attempt_timeout_ms = 10;
|
||||
config.censorship.tls_fetch.total_budget_ms = 20;
|
||||
config
|
||||
}
|
||||
|
||||
fn upstream_manager(config: &ProxyConfig) -> Arc<UpstreamManager> {
|
||||
Arc::new(UpstreamManager::new(
|
||||
Vec::new(),
|
||||
config.general.upstream_connect_retry_attempts,
|
||||
config.general.upstream_connect_retry_backoff_ms,
|
||||
config.general.upstream_connect_budget_ms,
|
||||
config.general.tg_connect,
|
||||
config.general.upstream_unhealthy_fail_threshold,
|
||||
config.general.upstream_connect_failfast_hard_errors,
|
||||
Arc::new(Stats::new()),
|
||||
))
|
||||
}
|
||||
|
||||
async fn tls_component_status(tracker: &StartupTracker) -> StartupComponentStatus {
|
||||
tracker
|
||||
.snapshot()
|
||||
.await
|
||||
.components
|
||||
.into_iter()
|
||||
.find(|component| component.id == COMPONENT_TLS_FRONT_BOOTSTRAP)
|
||||
.unwrap()
|
||||
.status
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tls_fetch_host_uses_each_domain_when_mask_host_is_primary_default() {
|
||||
@@ -218,4 +277,105 @@ mod tests {
|
||||
"origin.example"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn readiness_rejects_only_default_profiles() {
|
||||
assert!(readiness_error(&[]).is_none());
|
||||
assert_eq!(
|
||||
readiness_error(&["front.example".to_string()]),
|
||||
Some("TLS-front profiles are not ready for domains: front.example".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn require_ready_rejects_default_cache_after_bounded_fetch_failure() {
|
||||
let cache_dir = tempfile::tempdir().unwrap();
|
||||
let config = test_config(cache_dir.path());
|
||||
let domains = vec![config.censorship.tls_domain.clone()];
|
||||
let tracker = Arc::new(StartupTracker::new(1));
|
||||
let scope = RuntimeTaskScope::new();
|
||||
|
||||
let result = bootstrap_tls_front(
|
||||
&config,
|
||||
&domains,
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert_eq!(
|
||||
tls_component_status(&tracker).await,
|
||||
StartupComponentStatus::Failed
|
||||
);
|
||||
scope.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn require_ready_accepts_non_default_disk_cache_when_refresh_fails() {
|
||||
let cache_dir = tempfile::tempdir().unwrap();
|
||||
let config = test_config(cache_dir.path());
|
||||
let domains = vec![config.censorship.tls_domain.clone()];
|
||||
let seed = TlsFrontCache::new(&domains, config.censorship.fake_cert_len, cache_dir.path());
|
||||
let mut cached = seed.default_entry().as_ref().clone();
|
||||
cached.domain = domains[0].clone();
|
||||
tokio::fs::write(
|
||||
cache_dir.path().join("front.example.json"),
|
||||
serde_json::to_vec(&cached).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let tracker = Arc::new(StartupTracker::new(1));
|
||||
let scope = RuntimeTaskScope::new();
|
||||
|
||||
let cache = bootstrap_tls_front(
|
||||
&config,
|
||||
&domains,
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
TlsBootstrapPolicy::RequireReady,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert!(cache.default_profile_domains(&domains).await.is_empty());
|
||||
assert_eq!(
|
||||
tls_component_status(&tracker).await,
|
||||
StartupComponentStatus::Ready
|
||||
);
|
||||
scope.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn best_effort_returns_ready_and_refresh_tasks_are_scope_owned() {
|
||||
let cache_dir = tempfile::tempdir().unwrap();
|
||||
let config = test_config(cache_dir.path());
|
||||
let domains = vec![config.censorship.tls_domain.clone()];
|
||||
let tracker = Arc::new(StartupTracker::new(1));
|
||||
let scope = RuntimeTaskScope::new();
|
||||
|
||||
let cache = bootstrap_tls_front(
|
||||
&config,
|
||||
&domains,
|
||||
upstream_manager(&config),
|
||||
&tracker,
|
||||
scope.clone(),
|
||||
TlsBootstrapPolicy::BestEffort,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(cache.is_some());
|
||||
assert_eq!(
|
||||
tls_component_status(&tracker).await,
|
||||
StartupComponentStatus::Ready
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(1), scope.stop())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ mod service;
|
||||
mod startup;
|
||||
mod stats;
|
||||
mod stream;
|
||||
mod synlimit_control;
|
||||
mod tls_front;
|
||||
mod transport;
|
||||
mod util;
|
||||
|
||||
349
src/metrics.rs
349
src/metrics.rs
@@ -4,12 +4,12 @@ use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use http_body_util::Full;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
use ipnetwork::IpNetwork;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::time::timeout;
|
||||
@@ -17,6 +17,7 @@ use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::maestro::generation::RuntimeGeneration;
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::stats::Stats;
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
@@ -36,16 +37,8 @@ pub async fn serve(
|
||||
port: u16,
|
||||
listen: Option<String>,
|
||||
listen_backlog: u32,
|
||||
stats: Arc<Stats>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared_state: Arc<ProxySharedState>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
||||
whitelist: Vec<IpNetwork>,
|
||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||
) {
|
||||
let whitelist = Arc::new(whitelist);
|
||||
|
||||
// If `metrics_listen` is set, bind on that single address only.
|
||||
if let Some(ref listen_addr) = listen {
|
||||
let addr: SocketAddr = match listen_addr.parse() {
|
||||
@@ -55,21 +48,13 @@ pub async fn serve(
|
||||
return;
|
||||
}
|
||||
};
|
||||
let is_ipv6 = addr.is_ipv6();
|
||||
match bind_metrics_listener(addr, is_ipv6, listen_backlog) {
|
||||
// Match `server.api.listen`: `[::]:port` is a dual-stack wildcard
|
||||
// on Linux when `net.ipv6.bindv6only=0`.
|
||||
let ipv6_only = addr.is_ipv6() && !addr.ip().is_unspecified();
|
||||
match bind_metrics_listener(addr, ipv6_only, listen_backlog) {
|
||||
Ok(listener) => {
|
||||
info!("Metrics endpoint: http://{}/metrics and /beobachten", addr);
|
||||
serve_listener(
|
||||
listener,
|
||||
stats,
|
||||
beobachten,
|
||||
shared_state,
|
||||
ip_tracker,
|
||||
tls_cache,
|
||||
config_rx,
|
||||
whitelist,
|
||||
)
|
||||
.await;
|
||||
serve_listener(listener, active_runtime).await;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "Failed to bind metrics on {}", addr);
|
||||
@@ -115,50 +100,14 @@ pub async fn serve(
|
||||
warn!("Metrics listener is unavailable on both IPv4 and IPv6");
|
||||
}
|
||||
(Some(listener), None) | (None, Some(listener)) => {
|
||||
serve_listener(
|
||||
listener,
|
||||
stats,
|
||||
beobachten,
|
||||
shared_state,
|
||||
ip_tracker,
|
||||
tls_cache,
|
||||
config_rx,
|
||||
whitelist,
|
||||
)
|
||||
.await;
|
||||
serve_listener(listener, active_runtime).await;
|
||||
}
|
||||
(Some(listener4), Some(listener6)) => {
|
||||
let stats_v6 = stats.clone();
|
||||
let beobachten_v6 = beobachten.clone();
|
||||
let shared_state_v6 = shared_state.clone();
|
||||
let ip_tracker_v6 = ip_tracker.clone();
|
||||
let tls_cache_v6 = tls_cache.clone();
|
||||
let config_rx_v6 = config_rx.clone();
|
||||
let whitelist_v6 = whitelist.clone();
|
||||
let active_runtime_v6 = active_runtime.clone();
|
||||
tokio::spawn(async move {
|
||||
serve_listener(
|
||||
listener6,
|
||||
stats_v6,
|
||||
beobachten_v6,
|
||||
shared_state_v6,
|
||||
ip_tracker_v6,
|
||||
tls_cache_v6,
|
||||
config_rx_v6,
|
||||
whitelist_v6,
|
||||
)
|
||||
.await;
|
||||
serve_listener(listener6, active_runtime_v6).await;
|
||||
});
|
||||
serve_listener(
|
||||
listener4,
|
||||
stats,
|
||||
beobachten,
|
||||
shared_state,
|
||||
ip_tracker,
|
||||
tls_cache,
|
||||
config_rx,
|
||||
whitelist,
|
||||
)
|
||||
.await;
|
||||
serve_listener(listener4, active_runtime).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -178,16 +127,7 @@ fn bind_metrics_listener(
|
||||
TcpListener::from_std(socket.into())
|
||||
}
|
||||
|
||||
async fn serve_listener(
|
||||
listener: TcpListener,
|
||||
stats: Arc<Stats>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared_state: Arc<ProxySharedState>,
|
||||
ip_tracker: Arc<UserIpTracker>,
|
||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
||||
whitelist: Arc<Vec<IpNetwork>>,
|
||||
) {
|
||||
async fn serve_listener(listener: TcpListener, active_runtime: Arc<ArcSwap<RuntimeGeneration>>) {
|
||||
let connection_permits = Arc::new(Semaphore::new(METRICS_MAX_CONTROL_CONNECTIONS));
|
||||
|
||||
loop {
|
||||
@@ -199,7 +139,15 @@ async fn serve_listener(
|
||||
}
|
||||
};
|
||||
|
||||
if !whitelist.is_empty() && !whitelist.iter().any(|net| net.contains(peer.ip())) {
|
||||
let runtime = active_runtime.load_full();
|
||||
let config = runtime.config();
|
||||
if !config.server.metrics_whitelist.is_empty()
|
||||
&& !config
|
||||
.server
|
||||
.metrics_whitelist
|
||||
.iter()
|
||||
.any(|net| net.contains(peer.ip()))
|
||||
{
|
||||
debug!(peer = %peer, "Metrics request denied by whitelist");
|
||||
continue;
|
||||
}
|
||||
@@ -216,21 +164,17 @@ async fn serve_listener(
|
||||
}
|
||||
};
|
||||
|
||||
let stats = stats.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared_state = shared_state.clone();
|
||||
let ip_tracker = ip_tracker.clone();
|
||||
let tls_cache = tls_cache.clone();
|
||||
let config_rx_conn = config_rx.clone();
|
||||
let active_runtime = active_runtime.clone();
|
||||
tokio::spawn(async move {
|
||||
let _connection_permit = connection_permit;
|
||||
let svc = service_fn(move |req| {
|
||||
let stats = stats.clone();
|
||||
let beobachten = beobachten.clone();
|
||||
let shared_state = shared_state.clone();
|
||||
let ip_tracker = ip_tracker.clone();
|
||||
let tls_cache = tls_cache.clone();
|
||||
let config = config_rx_conn.borrow().clone();
|
||||
let runtime = active_runtime.load_full();
|
||||
let stats = runtime.stats.clone();
|
||||
let beobachten = runtime.beobachten.clone();
|
||||
let shared_state = runtime.proxy_shared.clone();
|
||||
let ip_tracker = runtime.ip_tracker.clone();
|
||||
let tls_cache = runtime.tls_cache.clone();
|
||||
let config = runtime.config();
|
||||
async move {
|
||||
handle(
|
||||
req,
|
||||
@@ -286,7 +230,7 @@ async fn handle<B>(
|
||||
}
|
||||
|
||||
if req.uri().path() == "/beobachten" {
|
||||
let body = render_beobachten(beobachten, config);
|
||||
let body = render_beobachten(stats, beobachten, config);
|
||||
let resp = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("content-type", "text/plain; charset=utf-8")
|
||||
@@ -302,13 +246,22 @@ async fn handle<B>(
|
||||
Ok(resp)
|
||||
}
|
||||
|
||||
fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> String {
|
||||
fn render_beobachten(stats: &Stats, beobachten: &BeobachtenStore, config: &ProxyConfig) -> String {
|
||||
if !config.general.beobachten {
|
||||
return "beobachten disabled\n".to_string();
|
||||
}
|
||||
|
||||
let ttl = Duration::from_secs(config.general.beobachten_minutes.saturating_mul(60));
|
||||
beobachten.snapshot_text(ttl)
|
||||
let mut body = beobachten.snapshot_text(ttl);
|
||||
let tls_text = stats.tls_fingerprint_snapshot_text(ttl, 20);
|
||||
if !tls_text.is_empty() {
|
||||
if !body.ends_with('\n') {
|
||||
body.push('\n');
|
||||
}
|
||||
body.push('\n');
|
||||
body.push_str(&tls_text);
|
||||
}
|
||||
body
|
||||
}
|
||||
|
||||
fn tls_front_domains(config: &ProxyConfig) -> Vec<String> {
|
||||
@@ -370,11 +323,32 @@ async fn render_tls_front_profile_health(
|
||||
"# HELP telemt_tls_front_profile_info TLS front profile source and feature flags per configured domain"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_info gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_profile_quality_info TLS front profile quality and key-share group per configured domain"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_quality_info gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_profile_age_seconds Age of cached TLS front profile data per configured domain"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_age_seconds gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_profile_server_hello_bytes TLS front cached ServerHello record body bytes per configured domain"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_front_profile_server_hello_bytes gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_profile_server_hello_extensions TLS front cached visible ServerHello extension count per configured domain"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_tls_front_profile_server_hello_extensions gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_tls_front_profile_app_data_records TLS front cached app-data record count per configured domain"
|
||||
@@ -409,11 +383,26 @@ async fn render_tls_front_profile_health(
|
||||
"telemt_tls_front_profile_info{{domain=\"{}\",source=\"{}\",is_default=\"{}\",has_cert_info=\"{}\",has_cert_payload=\"{}\"}} 1",
|
||||
domain, item.source, item.is_default, item.has_cert_info, item.has_cert_payload
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_profile_quality_info{{domain=\"{}\",quality=\"{}\",key_share_group=\"{}\"}} 1",
|
||||
domain, item.quality, item.key_share_group
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_profile_age_seconds{{domain=\"{}\"}} {}",
|
||||
domain, item.age_seconds
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_profile_server_hello_bytes{{domain=\"{}\"}} {}",
|
||||
domain, item.server_hello_record_len
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_profile_server_hello_extensions{{domain=\"{}\"}} {}",
|
||||
domain, item.server_hello_extensions
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_tls_front_profile_app_data_records{{domain=\"{}\"}} {}",
|
||||
@@ -548,6 +537,81 @@ async fn render_metrics(
|
||||
"telemt_buffer_pool_buffers_total{{kind=\"in_use\"}} {}",
|
||||
stats.get_buffer_pool_in_use_gauge()
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_buffer_pool_events_total Buffer-pool allocation lifecycle events"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_buffer_pool_events_total counter");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_buffer_pool_events_total{{event=\"replaced_nonstandard\"}} {}",
|
||||
stats.get_buffer_pool_replaced_nonstandard_total()
|
||||
);
|
||||
|
||||
let direct_budget = shared_state.direct_buffer_budget.snapshot();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_budget_bytes Direct relay copy-buffer budget and memory inputs"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_direct_relay_buffer_budget_bytes gauge");
|
||||
for (kind, value) in [
|
||||
("hard_limit", direct_budget.hard_limit_bytes),
|
||||
("target", direct_budget.target_bytes),
|
||||
("reserved", direct_budget.reserved_bytes),
|
||||
("memory_total", direct_budget.memory_total_bytes),
|
||||
("memory_available", direct_budget.memory_available_bytes),
|
||||
("process_rss", direct_budget.process_rss_bytes),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_budget_bytes{{kind=\"{}\"}} {}",
|
||||
kind, value
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_budget_events_total Direct relay buffer-budget lifecycle events"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_direct_relay_buffer_budget_events_total counter"
|
||||
);
|
||||
for (result, value) in [
|
||||
("promotion", direct_budget.promotion_total),
|
||||
("promotion_denied", direct_budget.promotion_denied_total),
|
||||
("minimum_fallback", direct_budget.minimum_fallback_total),
|
||||
("admission_rejected", direct_budget.admission_rejected_total),
|
||||
("quiet_demotion", direct_budget.quiet_demotion_total),
|
||||
(
|
||||
"write_pressure_demotion",
|
||||
direct_budget.write_pressure_demotion_total,
|
||||
),
|
||||
(
|
||||
"global_pressure_demotion",
|
||||
direct_budget.global_pressure_demotion_total,
|
||||
),
|
||||
] {
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_budget_events_total{{result=\"{}\"}} {}",
|
||||
result, value
|
||||
);
|
||||
}
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_direct_relay_buffer_sessions Current Direct relay sessions by adaptive tier"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_direct_relay_buffer_sessions gauge");
|
||||
for (tier, value) in ["base", "tier1", "tier2", "tier3"]
|
||||
.into_iter()
|
||||
.zip(direct_budget.tier_sessions)
|
||||
{
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_direct_relay_buffer_sessions{{tier=\"{}\"}} {}",
|
||||
tier, value
|
||||
);
|
||||
}
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
@@ -2388,6 +2452,82 @@ async fn render_metrics(
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_limit_bytes Configured resident-memory budget per ME writer"
|
||||
);
|
||||
let _ = writeln!(out, "# TYPE telemt_me_writer_byte_budget_limit_bytes gauge");
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_limit_bytes {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_limit_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_reserved_bytes Aggregate ME writer memory reservations by lifecycle state"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_byte_budget_reserved_bytes gauge"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_reserved_bytes{{state=\"queued\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_queued_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_reserved_bytes{{state=\"inflight\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_inflight_bytes_gauge()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_byte_budget_events_total ME writer byte-budget outcomes"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# TYPE telemt_me_writer_byte_budget_events_total counter"
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"wait\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_wait_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"timeout\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_timeout_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"telemt_me_writer_byte_budget_events_total{{result=\"oversize\"}} {}",
|
||||
if me_allows_normal {
|
||||
stats.get_me_writer_byte_budget_oversize_total()
|
||||
} else {
|
||||
0
|
||||
}
|
||||
);
|
||||
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"# HELP telemt_me_writer_pick_total ME writer-pick outcomes by mode and result"
|
||||
@@ -3890,7 +4030,20 @@ mod tests {
|
||||
session_id: Vec::new(),
|
||||
cipher_suite: [0x13, 0x01],
|
||||
compression: 0,
|
||||
extensions: Vec::new(),
|
||||
extensions: {
|
||||
let mut key_share = vec![0x00, 0x1d, 0x00, 0x20];
|
||||
key_share.resize(36, 0x42);
|
||||
vec![
|
||||
crate::tls_front::types::TlsExtension {
|
||||
ext_type: 0x002b,
|
||||
data: vec![0x03, 0x04],
|
||||
},
|
||||
crate::tls_front::types::TlsExtension {
|
||||
ext_type: 0x0033,
|
||||
data: key_share,
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
cert_info: None,
|
||||
cert_payload: Some(TlsCertPayload {
|
||||
@@ -3904,6 +4057,7 @@ mod tests {
|
||||
app_data_record_sizes: vec![1024, 512],
|
||||
ticket_record_sizes: vec![69],
|
||||
source: TlsProfileSource::Merged,
|
||||
..TlsBehaviorProfile::default()
|
||||
},
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: "primary.example".to_string(),
|
||||
@@ -3922,6 +4076,18 @@ mod tests {
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_quality_info{domain=\"primary.example\",quality=\"raw_strict\",key_share_group=\"x25519\"} 1")
|
||||
);
|
||||
assert!(
|
||||
output.contains("telemt_tls_front_profile_quality_info{domain=\"fallback.example\",quality=\"fallback\",key_share_group=\"none\"} 1")
|
||||
);
|
||||
assert!(output.contains(
|
||||
"telemt_tls_front_profile_server_hello_bytes{domain=\"primary.example\"} 90"
|
||||
));
|
||||
assert!(output.contains(
|
||||
"telemt_tls_front_profile_server_hello_extensions{domain=\"primary.example\"} 2"
|
||||
));
|
||||
assert!(
|
||||
output.contains(
|
||||
"telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2"
|
||||
@@ -4034,7 +4200,10 @@ mod tests {
|
||||
);
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_quality_info gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_server_hello_bytes gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_server_hello_extensions gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge"));
|
||||
assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge"));
|
||||
assert!(
|
||||
|
||||
@@ -8,6 +8,33 @@ use crate::error::{ProxyError, Result};
|
||||
|
||||
type OverrideMap = HashMap<(String, u16), IpAddr>;
|
||||
|
||||
/// Immutable DNS override snapshot owned by one runtime generation.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DnsOverrides {
|
||||
entries: std::sync::Arc<OverrideMap>,
|
||||
}
|
||||
|
||||
impl DnsOverrides {
|
||||
/// Parses a validated generation-local override snapshot.
|
||||
pub fn from_entries(entries: &[String]) -> Result<Self> {
|
||||
Ok(Self {
|
||||
entries: std::sync::Arc::new(parse_entries(entries)?),
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolves a generation-local hostname override.
|
||||
pub fn resolve(&self, host: &str, port: u16) -> Option<IpAddr> {
|
||||
self.entries
|
||||
.get(&(host.to_ascii_lowercase(), port))
|
||||
.copied()
|
||||
}
|
||||
|
||||
/// Resolves a generation-local override as a socket address.
|
||||
pub fn resolve_socket_addr(&self, host: &str, port: u16) -> Option<SocketAddr> {
|
||||
self.resolve(host, port).map(|ip| SocketAddr::new(ip, port))
|
||||
}
|
||||
}
|
||||
|
||||
static DNS_OVERRIDES: OnceLock<RwLock<OverrideMap>> = OnceLock::new();
|
||||
|
||||
fn overrides_store() -> &'static RwLock<OverrideMap> {
|
||||
@@ -180,6 +207,22 @@ mod tests {
|
||||
assert_eq!(resolved, Some("127.0.0.1".parse().unwrap()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generation_snapshots_do_not_observe_each_other() {
|
||||
let first = DnsOverrides::from_entries(&["example.com:443:127.0.0.1".to_string()]).unwrap();
|
||||
let second =
|
||||
DnsOverrides::from_entries(&["example.com:443:127.0.0.2".to_string()]).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
first.resolve("example.com", 443),
|
||||
Some("127.0.0.1".parse().unwrap())
|
||||
);
|
||||
assert_eq!(
|
||||
second.resolve("example.com", 443),
|
||||
Some("127.0.0.2".parse().unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_host_port_parses_supported_shapes() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -12,7 +12,7 @@ use tracing::{debug, info, warn};
|
||||
use crate::config::{NetworkConfig, UpstreamConfig, UpstreamType};
|
||||
use crate::error::Result;
|
||||
use crate::network::stun::{
|
||||
DualStunResult, IpFamily, StunProbeResult, stun_probe_family_with_bind,
|
||||
DualStunResult, IpFamily, StunProbeResult, stun_probe_family_with_bind_and_tcp_fallback,
|
||||
};
|
||||
use crate::transport::UpstreamManager;
|
||||
|
||||
@@ -58,6 +58,7 @@ impl NetworkDecision {
|
||||
}
|
||||
|
||||
const STUN_BATCH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const STUN_BATCH_TCP_FALLBACK_TIMEOUT: Duration = Duration::from_secs(12);
|
||||
|
||||
pub async fn run_probe(
|
||||
config: &NetworkConfig,
|
||||
@@ -81,8 +82,14 @@ pub async fn run_probe(
|
||||
warn!("STUN probe is enabled but network.stun_servers is empty");
|
||||
DualStunResult::default()
|
||||
} else {
|
||||
probe_stun_servers_parallel(&servers, stun_nat_probe_concurrency.max(1), None, None)
|
||||
.await
|
||||
probe_stun_servers_parallel(
|
||||
&servers,
|
||||
stun_nat_probe_concurrency.max(1),
|
||||
None,
|
||||
None,
|
||||
config.stun_tcp_fallback,
|
||||
)
|
||||
.await
|
||||
}
|
||||
} else if nat_probe {
|
||||
info!("STUN probe is disabled by network.stun_use=false");
|
||||
@@ -163,6 +170,7 @@ pub async fn run_probe(
|
||||
stun_nat_probe_concurrency.max(1),
|
||||
bind_v4,
|
||||
bind_v6,
|
||||
config.stun_tcp_fallback,
|
||||
)
|
||||
.await;
|
||||
if let Some(reflected) = direct_stun_res.v4.map(|r| r.reflected_addr) {
|
||||
@@ -234,7 +242,7 @@ pub async fn run_probe(
|
||||
Ok(probe)
|
||||
}
|
||||
|
||||
async fn detect_public_ipv4_http(urls: &[String]) -> Option<Ipv4Addr> {
|
||||
pub(crate) async fn detect_public_ipv4_http(urls: &[String]) -> Option<Ipv4Addr> {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()
|
||||
@@ -277,6 +285,7 @@ async fn probe_stun_servers_parallel(
|
||||
concurrency: usize,
|
||||
bind_v4: Option<IpAddr>,
|
||||
bind_v6: Option<IpAddr>,
|
||||
tcp_fallback: bool,
|
||||
) -> DualStunResult {
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut next_idx = 0usize;
|
||||
@@ -288,9 +297,26 @@ async fn probe_stun_servers_parallel(
|
||||
let stun_addr = servers[next_idx].clone();
|
||||
next_idx += 1;
|
||||
join_set.spawn(async move {
|
||||
let res = timeout(STUN_BATCH_TIMEOUT, async {
|
||||
let v4 = stun_probe_family_with_bind(&stun_addr, IpFamily::V4, bind_v4).await?;
|
||||
let v6 = stun_probe_family_with_bind(&stun_addr, IpFamily::V6, bind_v6).await?;
|
||||
let batch_timeout = if tcp_fallback {
|
||||
STUN_BATCH_TCP_FALLBACK_TIMEOUT
|
||||
} else {
|
||||
STUN_BATCH_TIMEOUT
|
||||
};
|
||||
let res = timeout(batch_timeout, async {
|
||||
let v4 = stun_probe_family_with_bind_and_tcp_fallback(
|
||||
&stun_addr,
|
||||
IpFamily::V4,
|
||||
bind_v4,
|
||||
tcp_fallback,
|
||||
)
|
||||
.await?;
|
||||
let v6 = stun_probe_family_with_bind_and_tcp_fallback(
|
||||
&stun_addr,
|
||||
IpFamily::V6,
|
||||
bind_v6,
|
||||
tcp_fallback,
|
||||
)
|
||||
.await?;
|
||||
Ok::<DualStunResult, crate::error::ProxyError>(DualStunResult { v4, v6 })
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use tokio::net::{UdpSocket, lookup_host};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpSocket, UdpSocket, lookup_host};
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
|
||||
use crate::crypto::SecureRandom;
|
||||
@@ -36,9 +37,16 @@ pub struct DualStunResult {
|
||||
}
|
||||
|
||||
pub async fn stun_probe_dual(stun_addr: &str) -> Result<DualStunResult> {
|
||||
stun_probe_dual_with_tcp_fallback(stun_addr, false).await
|
||||
}
|
||||
|
||||
pub async fn stun_probe_dual_with_tcp_fallback(
|
||||
stun_addr: &str,
|
||||
tcp_fallback: bool,
|
||||
) -> Result<DualStunResult> {
|
||||
let (v4, v6) = tokio::join!(
|
||||
stun_probe_family(stun_addr, IpFamily::V4),
|
||||
stun_probe_family(stun_addr, IpFamily::V6),
|
||||
stun_probe_family_with_tcp_fallback(stun_addr, IpFamily::V4, tcp_fallback),
|
||||
stun_probe_family_with_tcp_fallback(stun_addr, IpFamily::V6, tcp_fallback),
|
||||
);
|
||||
|
||||
Ok(DualStunResult { v4: v4?, v6: v6? })
|
||||
@@ -48,13 +56,44 @@ pub async fn stun_probe_family(
|
||||
stun_addr: &str,
|
||||
family: IpFamily,
|
||||
) -> Result<Option<StunProbeResult>> {
|
||||
stun_probe_family_with_bind(stun_addr, family, None).await
|
||||
stun_probe_family_with_tcp_fallback(stun_addr, family, false).await
|
||||
}
|
||||
|
||||
pub async fn stun_probe_family_with_tcp_fallback(
|
||||
stun_addr: &str,
|
||||
family: IpFamily,
|
||||
tcp_fallback: bool,
|
||||
) -> Result<Option<StunProbeResult>> {
|
||||
stun_probe_family_with_bind_and_tcp_fallback(stun_addr, family, None, tcp_fallback).await
|
||||
}
|
||||
|
||||
pub async fn stun_probe_family_with_bind(
|
||||
stun_addr: &str,
|
||||
family: IpFamily,
|
||||
bind_ip: Option<IpAddr>,
|
||||
) -> Result<Option<StunProbeResult>> {
|
||||
stun_probe_family_with_bind_and_tcp_fallback(stun_addr, family, bind_ip, false).await
|
||||
}
|
||||
|
||||
pub async fn stun_probe_family_with_bind_and_tcp_fallback(
|
||||
stun_addr: &str,
|
||||
family: IpFamily,
|
||||
bind_ip: Option<IpAddr>,
|
||||
tcp_fallback: bool,
|
||||
) -> Result<Option<StunProbeResult>> {
|
||||
let udp_attempts = if tcp_fallback { 1 } else { 3 };
|
||||
let udp_result = stun_probe_family_udp(stun_addr, family, bind_ip, udp_attempts).await?;
|
||||
if udp_result.is_some() || !tcp_fallback {
|
||||
return Ok(udp_result);
|
||||
}
|
||||
stun_probe_family_tcp(stun_addr, family, bind_ip).await
|
||||
}
|
||||
|
||||
async fn stun_probe_family_udp(
|
||||
stun_addr: &str,
|
||||
family: IpFamily,
|
||||
bind_ip: Option<IpAddr>,
|
||||
max_attempts: u8,
|
||||
) -> Result<Option<StunProbeResult>> {
|
||||
let bind_addr = match (family, bind_ip) {
|
||||
(IpFamily::V4, Some(IpAddr::V4(ip))) => SocketAddr::new(IpAddr::V4(ip), 0),
|
||||
@@ -94,12 +133,7 @@ pub async fn stun_probe_family_with_bind(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut req = [0u8; 20];
|
||||
req[0..2].copy_from_slice(&0x0001u16.to_be_bytes()); // Binding Request
|
||||
req[2..4].copy_from_slice(&0u16.to_be_bytes()); // length
|
||||
req[4..8].copy_from_slice(&0x2112A442u32.to_be_bytes()); // magic cookie
|
||||
stun_rng().fill(&mut req[8..20]); // transaction ID
|
||||
|
||||
let req = build_binding_request();
|
||||
let mut buf = [0u8; 256];
|
||||
let mut attempt = 0;
|
||||
let mut backoff = Duration::from_secs(1);
|
||||
@@ -115,7 +149,7 @@ pub async fn stun_probe_family_with_bind(
|
||||
Ok(Err(e)) => return Err(ProxyError::Proxy(format!("STUN recv failed: {e}"))),
|
||||
Err(_) => {
|
||||
attempt += 1;
|
||||
if attempt >= 3 {
|
||||
if attempt >= max_attempts {
|
||||
return Ok(None);
|
||||
}
|
||||
sleep(backoff).await;
|
||||
@@ -128,19 +162,139 @@ pub async fn stun_probe_family_with_bind(
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let magic = 0x2112A442u32.to_be_bytes();
|
||||
let txid = &req[8..20];
|
||||
let mut idx = 20;
|
||||
while idx + 4 <= n {
|
||||
let atype = u16::from_be_bytes(buf[idx..idx + 2].try_into().unwrap());
|
||||
let alen = u16::from_be_bytes(buf[idx + 2..idx + 4].try_into().unwrap()) as usize;
|
||||
idx += 4;
|
||||
if idx + alen > n {
|
||||
break;
|
||||
}
|
||||
if let Some(reflected_addr) = parse_reflected_addr(&buf[..n], txid) {
|
||||
let local_addr = socket
|
||||
.local_addr()
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN local_addr failed: {e}")))?;
|
||||
return Ok(Some(StunProbeResult {
|
||||
local_addr,
|
||||
reflected_addr,
|
||||
family,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
match atype {
|
||||
0x0020 /* XOR-MAPPED-ADDRESS */ | 0x0001 /* MAPPED-ADDRESS */ => {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn stun_probe_family_tcp(
|
||||
stun_addr: &str,
|
||||
family: IpFamily,
|
||||
bind_ip: Option<IpAddr>,
|
||||
) -> Result<Option<StunProbeResult>> {
|
||||
let target_addr = match resolve_stun_addr(stun_addr, family).await? {
|
||||
Some(addr) => addr,
|
||||
None => return Ok(None),
|
||||
};
|
||||
let socket = match family {
|
||||
IpFamily::V4 => TcpSocket::new_v4(),
|
||||
IpFamily::V6 => TcpSocket::new_v6(),
|
||||
}
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN TCP socket failed: {e}")))?;
|
||||
match (family, bind_ip) {
|
||||
(IpFamily::V4, Some(IpAddr::V4(ip))) => {
|
||||
if socket.bind(SocketAddr::new(IpAddr::V4(ip), 0)).is_err() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
(IpFamily::V6, Some(IpAddr::V6(ip))) => {
|
||||
if socket.bind(SocketAddr::new(IpAddr::V6(ip), 0)).is_err() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
(IpFamily::V4, Some(IpAddr::V6(_))) | (IpFamily::V6, Some(IpAddr::V4(_))) => {
|
||||
return Ok(None);
|
||||
}
|
||||
(_, None) => {}
|
||||
}
|
||||
|
||||
let connect_res = timeout(Duration::from_secs(3), socket.connect(target_addr)).await;
|
||||
let mut stream = match connect_res {
|
||||
Ok(Ok(stream)) => stream,
|
||||
Ok(Err(e))
|
||||
if family == IpFamily::V6
|
||||
&& matches!(
|
||||
e.kind(),
|
||||
std::io::ErrorKind::NetworkUnreachable
|
||||
| std::io::ErrorKind::HostUnreachable
|
||||
| std::io::ErrorKind::Unsupported
|
||||
| std::io::ErrorKind::NetworkDown
|
||||
) =>
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Err(e)) => return Err(ProxyError::Proxy(format!("STUN TCP connect failed: {e}"))),
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
|
||||
let req = build_binding_request();
|
||||
timeout(Duration::from_secs(3), stream.write_all(&req))
|
||||
.await
|
||||
.map_err(|_| ProxyError::Proxy("STUN TCP send timeout".to_string()))?
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN TCP send failed: {e}")))?;
|
||||
|
||||
let mut header = [0u8; 20];
|
||||
timeout(Duration::from_secs(3), stream.read_exact(&mut header))
|
||||
.await
|
||||
.map_err(|_| ProxyError::Proxy("STUN TCP header timeout".to_string()))?
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN TCP header read failed: {e}")))?;
|
||||
let body_len = u16::from_be_bytes([header[2], header[3]]) as usize;
|
||||
if body_len > 236 {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut buf = [0u8; 256];
|
||||
buf[..20].copy_from_slice(&header);
|
||||
if body_len > 0 {
|
||||
timeout(
|
||||
Duration::from_secs(3),
|
||||
stream.read_exact(&mut buf[20..20 + body_len]),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| ProxyError::Proxy("STUN TCP body timeout".to_string()))?
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN TCP body read failed: {e}")))?;
|
||||
}
|
||||
|
||||
let txid = &req[8..20];
|
||||
let Some(reflected_addr) = parse_reflected_addr(&buf[..20 + body_len], txid) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let local_addr = stream
|
||||
.local_addr()
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN TCP local_addr failed: {e}")))?;
|
||||
Ok(Some(StunProbeResult {
|
||||
local_addr,
|
||||
reflected_addr,
|
||||
family,
|
||||
}))
|
||||
}
|
||||
|
||||
fn build_binding_request() -> [u8; 20] {
|
||||
let mut req = [0u8; 20];
|
||||
req[0..2].copy_from_slice(&0x0001u16.to_be_bytes());
|
||||
req[2..4].copy_from_slice(&0u16.to_be_bytes());
|
||||
req[4..8].copy_from_slice(&0x2112A442u32.to_be_bytes());
|
||||
stun_rng().fill(&mut req[8..20]);
|
||||
req
|
||||
}
|
||||
|
||||
fn parse_reflected_addr(buf: &[u8], txid: &[u8]) -> Option<SocketAddr> {
|
||||
if buf.len() < 20 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let magic = 0x2112A442u32.to_be_bytes();
|
||||
let mut idx = 20;
|
||||
while idx + 4 <= buf.len() {
|
||||
let atype = u16::from_be_bytes(buf[idx..idx + 2].try_into().ok()?);
|
||||
let alen = u16::from_be_bytes(buf[idx + 2..idx + 4].try_into().ok()?) as usize;
|
||||
idx += 4;
|
||||
if idx + alen > buf.len() {
|
||||
break;
|
||||
}
|
||||
|
||||
match atype {
|
||||
0x0020 | 0x0001 => {
|
||||
if alen < 8 {
|
||||
break;
|
||||
}
|
||||
@@ -157,7 +311,6 @@ pub async fn stun_probe_family_with_bind(
|
||||
|
||||
let raw_ip = &buf[idx + 4..idx + 4 + len_check];
|
||||
let mut port = u16::from_be_bytes(port_bytes);
|
||||
|
||||
let reflected_ip = if atype == 0x0020 {
|
||||
port ^= ((magic[0] as u16) << 8) | magic[1] as u16;
|
||||
match family_byte {
|
||||
@@ -172,7 +325,9 @@ pub async fn stun_probe_family_with_bind(
|
||||
}
|
||||
0x02 => {
|
||||
let mut ip = [0u8; 16];
|
||||
let xor_key = [magic.as_slice(), txid].concat();
|
||||
let mut xor_key = [0u8; 16];
|
||||
xor_key[..4].copy_from_slice(&magic);
|
||||
xor_key[4..].copy_from_slice(txid.get(..12)?);
|
||||
for (i, b) in raw_ip.iter().enumerate().take(16) {
|
||||
ip[i] = *b ^ xor_key[i];
|
||||
}
|
||||
@@ -185,34 +340,24 @@ pub async fn stun_probe_family_with_bind(
|
||||
}
|
||||
} else {
|
||||
match family_byte {
|
||||
0x01 => IpAddr::V4(Ipv4Addr::new(raw_ip[0], raw_ip[1], raw_ip[2], raw_ip[3])),
|
||||
0x02 => IpAddr::V6(Ipv6Addr::from(<[u8; 16]>::try_from(raw_ip).unwrap())),
|
||||
0x01 => {
|
||||
IpAddr::V4(Ipv4Addr::new(raw_ip[0], raw_ip[1], raw_ip[2], raw_ip[3]))
|
||||
}
|
||||
0x02 => IpAddr::V6(Ipv6Addr::from(<[u8; 16]>::try_from(raw_ip).ok()?)),
|
||||
_ => {
|
||||
idx += (alen + 3) & !3;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let reflected_addr = SocketAddr::new(reflected_ip, port);
|
||||
let local_addr = socket
|
||||
.local_addr()
|
||||
.map_err(|e| ProxyError::Proxy(format!("STUN local_addr failed: {e}")))?;
|
||||
|
||||
return Ok(Some(StunProbeResult {
|
||||
local_addr,
|
||||
reflected_addr,
|
||||
family,
|
||||
}));
|
||||
return Some(SocketAddr::new(reflected_ip, port));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
idx += (alen + 3) & !3;
|
||||
}
|
||||
idx += (alen + 3) & !3;
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
None
|
||||
}
|
||||
|
||||
async fn resolve_stun_addr(stun_addr: &str, family: IpFamily) -> Result<Option<SocketAddr>> {
|
||||
@@ -245,3 +390,58 @@ async fn resolve_stun_addr(stun_addr: &str, family: IpFamily) -> Result<Option<S
|
||||
});
|
||||
Ok(target)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_reflected_addr_reads_mapped_ipv4() {
|
||||
let txid = [0u8; 12];
|
||||
let mut response = [0u8; 32];
|
||||
response[0..2].copy_from_slice(&0x0101u16.to_be_bytes());
|
||||
response[2..4].copy_from_slice(&12u16.to_be_bytes());
|
||||
response[4..8].copy_from_slice(&0x2112A442u32.to_be_bytes());
|
||||
response[20..22].copy_from_slice(&0x0001u16.to_be_bytes());
|
||||
response[22..24].copy_from_slice(&8u16.to_be_bytes());
|
||||
response[25] = 0x01;
|
||||
response[26..28].copy_from_slice(&443u16.to_be_bytes());
|
||||
response[28..32].copy_from_slice(&[203, 0, 113, 9]);
|
||||
|
||||
let reflected = parse_reflected_addr(&response, &txid).unwrap();
|
||||
assert_eq!(
|
||||
reflected,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 9)), 443)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_reflected_addr_reads_xor_mapped_ipv4() {
|
||||
let txid = [0u8; 12];
|
||||
let magic = 0x2112A442u32.to_be_bytes();
|
||||
let port = 443u16;
|
||||
let ip = [203u8, 0, 113, 9];
|
||||
let xport = port ^ (((magic[0] as u16) << 8) | magic[1] as u16);
|
||||
let xip = [
|
||||
ip[0] ^ magic[0],
|
||||
ip[1] ^ magic[1],
|
||||
ip[2] ^ magic[2],
|
||||
ip[3] ^ magic[3],
|
||||
];
|
||||
let mut response = [0u8; 32];
|
||||
response[0..2].copy_from_slice(&0x0101u16.to_be_bytes());
|
||||
response[2..4].copy_from_slice(&12u16.to_be_bytes());
|
||||
response[4..8].copy_from_slice(&0x2112A442u32.to_be_bytes());
|
||||
response[20..22].copy_from_slice(&0x0020u16.to_be_bytes());
|
||||
response[22..24].copy_from_slice(&8u16.to_be_bytes());
|
||||
response[25] = 0x01;
|
||||
response[26..28].copy_from_slice(&xport.to_be_bytes());
|
||||
response[28..32].copy_from_slice(&xip);
|
||||
|
||||
let reflected = parse_reflected_addr(&response, &txid).unwrap();
|
||||
assert_eq!(
|
||||
reflected,
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 9)), 443)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
use std::net::{IpAddr, Ipv4Addr};
|
||||
|
||||
use crate::crypto::SecureRandom;
|
||||
use crate::protocol::framing::{
|
||||
secure_version_d_body_len_from_wire_len, secure_version_d_padding_len,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
// ============= Telegram Datacenters =============
|
||||
@@ -236,22 +239,20 @@ pub fn is_valid_secure_payload_len(data_len: usize) -> bool {
|
||||
}
|
||||
|
||||
/// Compute Secure Intermediate payload length from wire length.
|
||||
/// Secure mode strips up to 3 random tail bytes by truncating to 4-byte boundary.
|
||||
/// Secure mode cannot distinguish full-word padding from payload, so only the
|
||||
/// non-aligned tail bytes are stripped.
|
||||
pub fn secure_payload_len_from_wire_len(wire_len: usize) -> Option<usize> {
|
||||
if wire_len < 4 {
|
||||
return None;
|
||||
}
|
||||
Some(wire_len - (wire_len % 4))
|
||||
secure_version_d_body_len_from_wire_len(wire_len)
|
||||
}
|
||||
|
||||
/// Generate padding length for Secure Intermediate protocol.
|
||||
/// Data must be 4-byte aligned; padding is 1..=3 so total is never divisible by 4.
|
||||
/// Outbound padding is 1..=3 so a receiver can strip it by 4-byte alignment.
|
||||
pub fn secure_padding_len(data_len: usize, rng: &SecureRandom) -> usize {
|
||||
debug_assert!(
|
||||
is_valid_secure_payload_len(data_len),
|
||||
"Secure payload must be 4-byte aligned, got {data_len}"
|
||||
);
|
||||
rng.range(3) + 1
|
||||
secure_version_d_padding_len(rng)
|
||||
}
|
||||
|
||||
// ============= Timeouts =============
|
||||
@@ -430,7 +431,7 @@ mod tests {
|
||||
for _ in 0..100 {
|
||||
let padding = secure_padding_len(data_len, &rng);
|
||||
assert!(
|
||||
padding <= 3,
|
||||
(1..=3).contains(&padding),
|
||||
"padding out of range: data_len={data_len}, padding={padding}"
|
||||
);
|
||||
assert_ne!(
|
||||
@@ -454,6 +455,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secure_wire_len_preserves_full_word_tail() {
|
||||
let payload_len = 64;
|
||||
for padding in [4usize, 8, 12] {
|
||||
let wire_len = payload_len + padding;
|
||||
let recovered = secure_payload_len_from_wire_len(wire_len);
|
||||
assert_eq!(recovered, Some(wire_len));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secure_wire_len_rejects_too_short_frames() {
|
||||
assert_eq!(secure_payload_len_from_wire_len(0), None);
|
||||
|
||||
92
src/protocol/framing.rs
Normal file
92
src/protocol/framing.rs
Normal file
@@ -0,0 +1,92 @@
|
||||
//! Shared MTProto transport framing helpers.
|
||||
|
||||
use crate::crypto::SecureRandom;
|
||||
|
||||
/// QuickACK marker bit used by Intermediate and Secure Intermediate headers.
|
||||
pub(crate) const INTERMEDIATE_QUICKACK_FLAG: u32 = 0x8000_0000;
|
||||
|
||||
/// Payload length mask used by Intermediate and Secure Intermediate headers.
|
||||
pub(crate) const INTERMEDIATE_WIRE_LEN_MASK: u32 = 0x7fff_ffff;
|
||||
|
||||
/// Maximum outbound Secure tail length that keeps wire lengths non-aligned.
|
||||
pub(crate) const SECURE_VERSION_D_PADDING_MAX: usize = 3;
|
||||
|
||||
/// Parsed Intermediate/Secure Intermediate length header.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct IntermediateHeader {
|
||||
/// Payload length on the wire, excluding the four-byte header.
|
||||
pub(crate) wire_len: usize,
|
||||
/// Whether the QuickACK marker bit was set in the length header.
|
||||
pub(crate) quickack: bool,
|
||||
}
|
||||
|
||||
/// Parse an Intermediate/Secure Intermediate length header.
|
||||
pub(crate) fn parse_intermediate_header(header: [u8; 4]) -> IntermediateHeader {
|
||||
let raw = u32::from_le_bytes(header);
|
||||
IntermediateHeader {
|
||||
wire_len: (raw & INTERMEDIATE_WIRE_LEN_MASK) as usize,
|
||||
quickack: (raw & INTERMEDIATE_QUICKACK_FLAG) != 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode an Intermediate/Secure Intermediate length header.
|
||||
pub(crate) fn encode_intermediate_header(wire_len: usize, quickack: bool) -> Option<u32> {
|
||||
if wire_len > INTERMEDIATE_WIRE_LEN_MASK as usize {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut raw = u32::try_from(wire_len).ok()?;
|
||||
if quickack {
|
||||
raw |= INTERMEDIATE_QUICKACK_FLAG;
|
||||
}
|
||||
Some(raw)
|
||||
}
|
||||
|
||||
/// Recover the VersionD body length visible to MTProto from the encrypted wire length.
|
||||
pub(crate) fn secure_version_d_body_len_from_wire_len(wire_len: usize) -> Option<usize> {
|
||||
if wire_len < 4 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(wire_len - (wire_len % 4))
|
||||
}
|
||||
|
||||
/// Generate outbound Secure tail length without ambiguous full-word padding.
|
||||
pub(crate) fn secure_version_d_padding_len(rng: &SecureRandom) -> usize {
|
||||
rng.range(SECURE_VERSION_D_PADDING_MAX) + 1
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn intermediate_header_roundtrip_preserves_quickack_zero_length() {
|
||||
let encoded = encode_intermediate_header(0, true).unwrap();
|
||||
assert_eq!(encoded, INTERMEDIATE_QUICKACK_FLAG);
|
||||
|
||||
let parsed = parse_intermediate_header(encoded.to_le_bytes());
|
||||
assert_eq!(parsed.wire_len, 0);
|
||||
assert!(parsed.quickack);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intermediate_header_rejects_lengths_above_31_bits() {
|
||||
assert_eq!(
|
||||
encode_intermediate_header(INTERMEDIATE_WIRE_LEN_MASK as usize, false),
|
||||
Some(INTERMEDIATE_WIRE_LEN_MASK)
|
||||
);
|
||||
assert_eq!(
|
||||
encode_intermediate_header(INTERMEDIATE_WIRE_LEN_MASK as usize + 1, false),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secure_version_d_body_len_strips_only_non_word_tail() {
|
||||
assert_eq!(secure_version_d_body_len_from_wire_len(3), None);
|
||||
assert_eq!(secure_version_d_body_len_from_wire_len(8), Some(8));
|
||||
assert_eq!(secure_version_d_body_len_from_wire_len(11), Some(8));
|
||||
assert_eq!(secure_version_d_body_len_from_wire_len(12), Some(12));
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
pub mod constants;
|
||||
pub mod frame;
|
||||
pub(crate) mod framing;
|
||||
pub mod obfuscation;
|
||||
pub mod tls;
|
||||
pub mod tls_fingerprint;
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub use constants::*;
|
||||
@@ -13,3 +15,5 @@ pub use frame::*;
|
||||
pub use obfuscation::*;
|
||||
#[allow(unused_imports)]
|
||||
pub use tls::*;
|
||||
#[allow(unused_imports)]
|
||||
pub use tls_fingerprint::*;
|
||||
|
||||
@@ -1239,6 +1239,18 @@ fn test_gen_fake_x25519_key() {
|
||||
assert_ne!(key1, key2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gen_fake_x25519mlkem768_server_key_share_shape() {
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
let key_share = gen_fake_x25519mlkem768_server_key_share(&rng);
|
||||
|
||||
assert_eq!(key_share.len(), X25519MLKEM768_SERVER_KEY_SHARE_LEN);
|
||||
assert!(
|
||||
key_share.iter().any(|byte| *byte != 0),
|
||||
"hybrid ServerHello key_share must not collapse to all-zero bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fake_x25519_key_is_nonzero_and_varies() {
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
@@ -1325,6 +1337,69 @@ fn server_hello_extension_types(record: &[u8]) -> Vec<u16> {
|
||||
out
|
||||
}
|
||||
|
||||
fn server_hello_key_share(record: &[u8]) -> Option<(u16, usize)> {
|
||||
if record.len() < 9 || record[0] != TLS_RECORD_HANDSHAKE || record[5] != 0x02 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let record_len = u16::from_be_bytes([record[3], record[4]]) as usize;
|
||||
if record.len() < 5 + record_len {
|
||||
return None;
|
||||
}
|
||||
|
||||
let hs_len = u32::from_be_bytes([0, record[6], record[7], record[8]]) as usize;
|
||||
let hs_start = 5;
|
||||
let hs_end = hs_start + 4 + hs_len;
|
||||
if hs_end > record.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut pos = hs_start + 4 + 2 + 32;
|
||||
if pos >= hs_end {
|
||||
return None;
|
||||
}
|
||||
let sid_len = record[pos] as usize;
|
||||
pos += 1 + sid_len;
|
||||
if pos + 2 + 1 + 2 > hs_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
pos += 2 + 1;
|
||||
let ext_len = u16::from_be_bytes([record[pos], record[pos + 1]]) as usize;
|
||||
pos += 2;
|
||||
let ext_end = pos + ext_len;
|
||||
if ext_end > hs_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
while pos + 4 <= ext_end {
|
||||
let etype = u16::from_be_bytes([record[pos], record[pos + 1]]);
|
||||
let elen = u16::from_be_bytes([record[pos + 2], record[pos + 3]]) as usize;
|
||||
pos += 4;
|
||||
if pos + elen > ext_end {
|
||||
return None;
|
||||
}
|
||||
if etype == extension_type::KEY_SHARE {
|
||||
if elen < 4 {
|
||||
return None;
|
||||
}
|
||||
let group = u16::from_be_bytes([record[pos], record[pos + 1]]);
|
||||
let key_exchange_len = u16::from_be_bytes([record[pos + 2], record[pos + 3]]) as usize;
|
||||
if 4 + key_exchange_len != elen {
|
||||
return None;
|
||||
}
|
||||
return Some((group, key_exchange_len));
|
||||
}
|
||||
pos += elen;
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn test_server_key_share(group: u16, len: usize) -> ServerHelloKeyShare {
|
||||
ServerHelloKeyShare::new(group, vec![0x42; len])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_server_hello_never_places_alpn_in_server_hello_extensions() {
|
||||
let secret = b"alpn_sh_forbidden";
|
||||
@@ -1372,6 +1447,7 @@ fn emulated_server_hello_never_places_alpn_in_server_hello_extensions() {
|
||||
app_data_record_sizes: vec![1024],
|
||||
ticket_record_sizes: Vec::new(),
|
||||
source: TlsProfileSource::Default,
|
||||
..TlsBehaviorProfile::default()
|
||||
},
|
||||
fetched_at: SystemTime::now(),
|
||||
domain: "example.com".to_string(),
|
||||
@@ -1386,6 +1462,10 @@ fn emulated_server_hello_never_places_alpn_in_server_hello_extensions() {
|
||||
true,
|
||||
ClientHelloTlsVersion::Tls13,
|
||||
[0x13, 0x01],
|
||||
&test_server_key_share(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_SERVER_KEY_SHARE_LEN,
|
||||
),
|
||||
&rng,
|
||||
Some(b"h2".to_vec()),
|
||||
0,
|
||||
@@ -1395,14 +1475,21 @@ fn emulated_server_hello_never_places_alpn_in_server_hello_extensions() {
|
||||
!exts.contains(&0x0010),
|
||||
"ALPN extension must not appear in emulated ServerHello"
|
||||
);
|
||||
assert_eq!(
|
||||
server_hello_key_share(&response),
|
||||
Some((
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_SERVER_KEY_SHARE_LEN
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tls_extension_builder() {
|
||||
let key = [0x42u8; 32];
|
||||
let key = vec![0x42u8; X25519MLKEM768_SERVER_KEY_SHARE_LEN];
|
||||
|
||||
let mut builder = TlsExtensionBuilder::new();
|
||||
builder.add_key_share(&key);
|
||||
builder.add_key_share(TLS_NAMED_GROUP_X25519MLKEM768, &key);
|
||||
builder.add_supported_versions(0x0304);
|
||||
|
||||
let result = builder.build();
|
||||
@@ -1415,10 +1502,10 @@ fn test_tls_extension_builder() {
|
||||
#[test]
|
||||
fn test_server_hello_builder() {
|
||||
let session_id = vec![0x01, 0x02, 0x03, 0x04];
|
||||
let key = [0x55u8; 32];
|
||||
let key = vec![0x55u8; X25519MLKEM768_SERVER_KEY_SHARE_LEN];
|
||||
|
||||
let builder = ServerHelloBuilder::new(session_id.clone())
|
||||
.with_x25519_key(&key)
|
||||
.with_key_share(TLS_NAMED_GROUP_X25519MLKEM768, &key)
|
||||
.with_tls13_version();
|
||||
|
||||
let record = builder.build_record();
|
||||
@@ -1452,6 +1539,41 @@ fn test_build_server_hello_structure() {
|
||||
let app_start = ccs_start + ccs_len;
|
||||
assert!(response.len() > app_start + 5);
|
||||
assert_eq!(response[app_start], TLS_RECORD_APPLICATION);
|
||||
|
||||
assert_eq!(
|
||||
server_hello_key_share(&response),
|
||||
Some((
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_SERVER_KEY_SHARE_LEN
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_server_hello_with_cipher_uses_selected_key_share_group() {
|
||||
let secret = b"test secret";
|
||||
let client_digest = [0x42u8; 32];
|
||||
let session_id = vec![0xAA; 32];
|
||||
let key_share =
|
||||
ServerHelloKeyShare::new(TLS_NAMED_GROUP_X25519, vec![0x55u8; X25519_KEY_SHARE_LEN]);
|
||||
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
let response = build_server_hello_with_cipher(
|
||||
secret,
|
||||
&client_digest,
|
||||
&session_id,
|
||||
2048,
|
||||
&rng,
|
||||
[0x13, 0x01],
|
||||
&key_share,
|
||||
None,
|
||||
0,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
server_hello_key_share(&response),
|
||||
Some((TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1474,10 +1596,10 @@ fn test_build_server_hello_digest() {
|
||||
#[test]
|
||||
fn test_server_hello_extensions_length() {
|
||||
let session_id = vec![0x01; 32];
|
||||
let key = [0x55u8; 32];
|
||||
let key = vec![0x55u8; X25519MLKEM768_SERVER_KEY_SHARE_LEN];
|
||||
|
||||
let builder = ServerHelloBuilder::new(session_id)
|
||||
.with_x25519_key(&key)
|
||||
.with_key_share(TLS_NAMED_GROUP_X25519MLKEM768, &key)
|
||||
.with_tls13_version();
|
||||
|
||||
let record = builder.build_record();
|
||||
@@ -1513,6 +1635,39 @@ fn build_client_hello_with_exts(exts: Vec<(u16, Vec<u8>)>, host: &str) -> Vec<u8
|
||||
build_client_hello_with_ciphers_and_exts(&[[0x13, 0x01]], exts, host)
|
||||
}
|
||||
|
||||
fn client_key_share_extension(entries: &[(u16, usize)]) -> Vec<u8> {
|
||||
let mut shares = Vec::new();
|
||||
for (group, key_exchange_len) in entries {
|
||||
assert!(*key_exchange_len <= u16::MAX as usize);
|
||||
shares.extend_from_slice(&group.to_be_bytes());
|
||||
shares.extend_from_slice(&(*key_exchange_len as u16).to_be_bytes());
|
||||
let start = shares.len();
|
||||
shares.resize(start + *key_exchange_len, 0x42);
|
||||
}
|
||||
|
||||
assert!(shares.len() <= u16::MAX as usize);
|
||||
let mut extension = Vec::new();
|
||||
extension.extend_from_slice(&(shares.len() as u16).to_be_bytes());
|
||||
extension.extend_from_slice(&shares);
|
||||
extension
|
||||
}
|
||||
|
||||
fn client_key_share_extension_with_payloads(entries: &[(u16, &[u8])]) -> Vec<u8> {
|
||||
let mut shares = Vec::new();
|
||||
for (group, key_exchange) in entries {
|
||||
assert!(key_exchange.len() <= u16::MAX as usize);
|
||||
shares.extend_from_slice(&group.to_be_bytes());
|
||||
shares.extend_from_slice(&(key_exchange.len() as u16).to_be_bytes());
|
||||
shares.extend_from_slice(key_exchange);
|
||||
}
|
||||
|
||||
assert!(shares.len() <= u16::MAX as usize);
|
||||
let mut extension = Vec::new();
|
||||
extension.extend_from_slice(&(shares.len() as u16).to_be_bytes());
|
||||
extension.extend_from_slice(&shares);
|
||||
extension
|
||||
}
|
||||
|
||||
fn build_client_hello_with_ciphers_and_exts(
|
||||
cipher_suites: &[[u8; 2]],
|
||||
exts: Vec<(u16, Vec<u8>)>,
|
||||
@@ -1674,7 +1829,7 @@ fn select_server_hello_cipher_suite_keeps_profile_cipher_when_offered() {
|
||||
);
|
||||
assert_eq!(
|
||||
select_server_hello_cipher_suite(&ch, [0x13, 0x03]),
|
||||
[0x13, 0x03]
|
||||
Some([0x13, 0x03])
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1687,30 +1842,222 @@ fn select_server_hello_cipher_suite_ignores_profile_tls12_cipher() {
|
||||
);
|
||||
assert_eq!(
|
||||
select_server_hello_cipher_suite(&ch, [0xc0, 0x2f]),
|
||||
[0x13, 0x03]
|
||||
Some([0x13, 0x03])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_cipher_suite_rejects_without_offered_tls13_suite() {
|
||||
let ch = build_client_hello_with_ciphers_and_exts(&[[0xc0, 0x2f]], Vec::new(), "example.com");
|
||||
assert_eq!(select_server_hello_cipher_suite(&ch, [0x13, 0x01]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_cipher_suite_falls_back_to_offered_tls13_suite() {
|
||||
let ch = build_client_hello_with_ciphers_and_exts(&[[0x13, 0x03]], Vec::new(), "example.com");
|
||||
assert_eq!(
|
||||
select_server_hello_cipher_suite(&ch, [0x13, 0x01]),
|
||||
[0x13, 0x03]
|
||||
Some([0x13, 0x03])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_cipher_suite_keeps_preferred_for_malformed_clienthello() {
|
||||
fn select_server_hello_cipher_suite_rejects_malformed_clienthello() {
|
||||
let mut ch =
|
||||
build_client_hello_with_ciphers_and_exts(&[[0x13, 0x03]], Vec::new(), "example.com");
|
||||
ch.truncate(12);
|
||||
assert_eq!(select_server_hello_cipher_suite(&ch, [0x13, 0x01]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_key_share_group_prefers_hybrid_when_valid_share_is_offered() {
|
||||
let key_share = client_key_share_extension(&[
|
||||
(0x0a0a, 1),
|
||||
(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
),
|
||||
(TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN),
|
||||
]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
|
||||
assert_eq!(
|
||||
select_server_hello_cipher_suite(&ch, [0x13, 0x01]),
|
||||
[0x13, 0x01]
|
||||
select_server_hello_key_share_group(&ch),
|
||||
Some(TLS_NAMED_GROUP_X25519MLKEM768)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_key_share_group_prefers_profiled_x25519_when_valid_share_is_offered() {
|
||||
let key_share = client_key_share_extension(&[
|
||||
(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
),
|
||||
(TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN),
|
||||
]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
|
||||
assert_eq!(
|
||||
select_server_hello_key_share_group_with_preference(&ch, Some(TLS_NAMED_GROUP_X25519)),
|
||||
Some(TLS_NAMED_GROUP_X25519)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_x25519mlkem768_server_key_share_accepts_tdesktop_canonical_share() {
|
||||
let key_share = client_key_share_extension(&[
|
||||
(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
),
|
||||
(TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN),
|
||||
]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
|
||||
let server_key_share = build_x25519mlkem768_server_key_share(&ch, &rng)
|
||||
.expect("tdesktop-like canonical share must build a ServerHello share");
|
||||
|
||||
assert_eq!(server_key_share.len(), X25519MLKEM768_SERVER_KEY_SHARE_LEN);
|
||||
assert!(
|
||||
server_key_share[..MLKEM768_SERVER_CIPHERTEXT_LEN]
|
||||
.iter()
|
||||
.any(|byte| *byte != 0),
|
||||
"ML-KEM ciphertext must not be all zero"
|
||||
);
|
||||
assert!(
|
||||
server_key_share[MLKEM768_SERVER_CIPHERTEXT_LEN..]
|
||||
.iter()
|
||||
.any(|byte| *byte != 0),
|
||||
"X25519 server share must not be all zero"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_x25519_server_key_share_accepts_tdesktop_fallback_share() {
|
||||
let key_share = client_key_share_extension(&[
|
||||
(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
),
|
||||
(TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN),
|
||||
]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
|
||||
let server_key_share = build_x25519_server_key_share(&ch, &rng)
|
||||
.expect("tdesktop-like X25519 share must build a ServerHello share");
|
||||
|
||||
assert_eq!(server_key_share.len(), X25519_KEY_SHARE_LEN);
|
||||
assert!(
|
||||
server_key_share.iter().any(|byte| *byte != 0),
|
||||
"X25519 server share must not be all zero"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_server_hello_key_share_prefers_profiled_x25519() {
|
||||
let key_share = client_key_share_extension(&[
|
||||
(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
),
|
||||
(TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN),
|
||||
]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
|
||||
let server_key_share = build_server_hello_key_share(&ch, Some(TLS_NAMED_GROUP_X25519), &rng)
|
||||
.expect("profiled X25519 share must be selected when client offers it");
|
||||
|
||||
assert_eq!(server_key_share.group(), TLS_NAMED_GROUP_X25519);
|
||||
assert_eq!(server_key_share.key_exchange().len(), X25519_KEY_SHARE_LEN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_server_hello_key_share_falls_back_from_bad_profiled_x25519_to_hybrid() {
|
||||
let key_share = client_key_share_extension(&[(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
)]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
|
||||
let server_key_share = build_server_hello_key_share(&ch, Some(TLS_NAMED_GROUP_X25519), &rng)
|
||||
.expect("hybrid share must be selected when profiled X25519 is unavailable");
|
||||
|
||||
assert_eq!(server_key_share.group(), TLS_NAMED_GROUP_X25519MLKEM768);
|
||||
assert_eq!(
|
||||
server_key_share.key_exchange().len(),
|
||||
X25519MLKEM768_SERVER_KEY_SHARE_LEN
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_x25519mlkem768_server_key_share_rejects_noncanonical_mlkem_key() {
|
||||
let mut key_exchange = vec![0x42; X25519MLKEM768_CLIENT_KEY_SHARE_LEN];
|
||||
key_exchange[..3].copy_from_slice(&[0xff, 0xff, 0xff]);
|
||||
let key_share = client_key_share_extension_with_payloads(&[(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
&key_exchange,
|
||||
)]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
|
||||
assert!(build_x25519mlkem768_server_key_share(&ch, &rng).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_x25519mlkem768_server_key_share_rejects_all_zero_x25519_share() {
|
||||
let mut key_exchange = vec![0x42; X25519MLKEM768_CLIENT_KEY_SHARE_LEN];
|
||||
key_exchange[MLKEM768_CLIENT_ENCAPSULATION_KEY_LEN..].fill(0);
|
||||
let key_share = client_key_share_extension_with_payloads(&[(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
&key_exchange,
|
||||
)]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
let rng = crate::crypto::SecureRandom::new();
|
||||
|
||||
assert!(build_x25519mlkem768_server_key_share(&ch, &rng).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_key_share_group_accepts_x25519_when_hybrid_is_absent() {
|
||||
let key_share = client_key_share_extension(&[(TLS_NAMED_GROUP_X25519, X25519_KEY_SHARE_LEN)]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
|
||||
assert_eq!(
|
||||
select_server_hello_key_share_group(&ch),
|
||||
Some(TLS_NAMED_GROUP_X25519)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_key_share_group_rejects_malformed_hybrid_len() {
|
||||
let key_share = client_key_share_extension(&[(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN - 1,
|
||||
)]);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
|
||||
assert_eq!(select_server_hello_key_share_group(&ch), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_server_hello_key_share_group_rejects_malformed_key_share_tail() {
|
||||
let mut key_share = client_key_share_extension(&[(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
)]);
|
||||
let shares_len = u16::from_be_bytes([key_share[0], key_share[1]]) + 1;
|
||||
key_share[0..2].copy_from_slice(&shares_len.to_be_bytes());
|
||||
key_share.push(0);
|
||||
let ch = build_client_hello_with_exts(vec![(0x0033, key_share)], "example.com");
|
||||
|
||||
assert_eq!(select_server_hello_key_share_group(&ch), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_sni_rejects_zero_length_host_name() {
|
||||
let mut sni_ext = Vec::new();
|
||||
|
||||
@@ -65,6 +65,7 @@ use super::constants::*;
|
||||
use crate::crypto::{SecureRandom, sha256_hmac};
|
||||
#[cfg(test)]
|
||||
use crate::error::ProxyError;
|
||||
use ml_kem::{B32, EncapsulationKey as MlKemEncapsulationKey, Key as MlKemKey, MlKem768};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use subtle::ConstantTimeEq;
|
||||
use x25519_dalek::{X25519_BASEPOINT_BYTES, x25519};
|
||||
@@ -109,9 +110,45 @@ mod cipher_suite {
|
||||
pub const TLS_CHACHA20_POLY1305_SHA256: [u8; 2] = [0x13, 0x03];
|
||||
}
|
||||
|
||||
/// TLS Named Curves
|
||||
/// TLS named groups used in KeyShare extensions.
|
||||
mod named_curve {
|
||||
pub const X25519: u16 = 0x001d;
|
||||
pub const X25519MLKEM768: u16 = 0x11ec;
|
||||
}
|
||||
|
||||
/// TLS X25519 named group.
|
||||
pub(crate) const TLS_NAMED_GROUP_X25519: u16 = named_curve::X25519;
|
||||
/// TLS X25519MLKEM768 named group.
|
||||
pub(crate) const TLS_NAMED_GROUP_X25519MLKEM768: u16 = named_curve::X25519MLKEM768;
|
||||
|
||||
const X25519_KEY_SHARE_LEN: usize = 32;
|
||||
const X25519MLKEM768_CLIENT_KEY_SHARE_LEN: usize = 1216;
|
||||
const X25519MLKEM768_SERVER_KEY_SHARE_LEN: usize = 1120;
|
||||
const MLKEM768_CLIENT_ENCAPSULATION_KEY_LEN: usize = 1184;
|
||||
const MLKEM768_SERVER_CIPHERTEXT_LEN: usize = 1088;
|
||||
|
||||
/// ServerHello key_share selected for the authenticated ClientHello.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ServerHelloKeyShare {
|
||||
group: u16,
|
||||
key_exchange: Vec<u8>,
|
||||
}
|
||||
|
||||
impl ServerHelloKeyShare {
|
||||
pub(crate) fn new(group: u16, key_exchange: Vec<u8>) -> Self {
|
||||
Self {
|
||||
group,
|
||||
key_exchange,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn group(&self) -> u16 {
|
||||
self.group
|
||||
}
|
||||
|
||||
pub(crate) fn key_exchange(&self) -> &[u8] {
|
||||
&self.key_exchange
|
||||
}
|
||||
}
|
||||
|
||||
// ============= TLS Validation Result =============
|
||||
@@ -144,26 +181,28 @@ impl TlsExtensionBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Add Key Share extension with X25519 key
|
||||
fn add_key_share(&mut self, public_key: &[u8; 32]) -> &mut Self {
|
||||
/// Add KeyShare extension with the selected named group.
|
||||
fn add_key_share(&mut self, group: u16, key_exchange: &[u8]) -> &mut Self {
|
||||
let Ok(key_exchange_len) = u16::try_from(key_exchange.len()) else {
|
||||
return self;
|
||||
};
|
||||
let Some(entry_len) = key_exchange.len().checked_add(4) else {
|
||||
return self;
|
||||
};
|
||||
let Ok(entry_len) = u16::try_from(entry_len) else {
|
||||
return self;
|
||||
};
|
||||
|
||||
// Extension type: key_share (0x0033)
|
||||
self.extensions
|
||||
.extend_from_slice(&extension_type::KEY_SHARE.to_be_bytes());
|
||||
|
||||
// Key share entry: curve (2) + key_len (2) + key (32) = 36 bytes
|
||||
// Extension data length
|
||||
let entry_len: u16 = 2 + 2 + 32; // curve + length + key
|
||||
// ServerHello key_share data is exactly one KeyShareEntry.
|
||||
self.extensions.extend_from_slice(&entry_len.to_be_bytes());
|
||||
|
||||
// Named curve: x25519
|
||||
self.extensions.extend_from_slice(&group.to_be_bytes());
|
||||
self.extensions
|
||||
.extend_from_slice(&named_curve::X25519.to_be_bytes());
|
||||
|
||||
// Key length
|
||||
self.extensions.extend_from_slice(&(32u16).to_be_bytes());
|
||||
|
||||
// Key data
|
||||
self.extensions.extend_from_slice(public_key);
|
||||
.extend_from_slice(&key_exchange_len.to_be_bytes());
|
||||
self.extensions.extend_from_slice(key_exchange);
|
||||
|
||||
self
|
||||
}
|
||||
@@ -232,8 +271,8 @@ impl ServerHelloBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
fn with_x25519_key(mut self, key: &[u8; 32]) -> Self {
|
||||
self.extensions.add_key_share(key);
|
||||
fn with_key_share(mut self, group: u16, key_exchange: &[u8]) -> Self {
|
||||
self.extensions.add_key_share(group, key_exchange);
|
||||
self
|
||||
}
|
||||
|
||||
@@ -508,9 +547,137 @@ fn validate_tls_handshake_at_time_with_boot_cap(
|
||||
/// Uses RFC 7748 X25519 scalar multiplication over the canonical basepoint,
|
||||
/// yielding distribution-consistent public keys for anti-fingerprinting.
|
||||
pub fn gen_fake_x25519_key(rng: &SecureRandom) -> [u8; 32] {
|
||||
let mut scalar = [0u8; 32];
|
||||
scalar.copy_from_slice(&rng.bytes(32));
|
||||
x25519(scalar, X25519_BASEPOINT_BYTES)
|
||||
let (_scalar, public_key) = gen_x25519_key_pair(rng);
|
||||
public_key
|
||||
}
|
||||
|
||||
fn gen_x25519_key_pair(rng: &SecureRandom) -> ([u8; 32], [u8; 32]) {
|
||||
let mut scalar = [0u8; X25519_KEY_SHARE_LEN];
|
||||
rng.fill(&mut scalar);
|
||||
let public_key = x25519(scalar, X25519_BASEPOINT_BYTES);
|
||||
(scalar, public_key)
|
||||
}
|
||||
|
||||
/// Generate a fake X25519MLKEM768 ServerHello key_share payload.
|
||||
pub(crate) fn gen_fake_x25519mlkem768_server_key_share(rng: &SecureRandom) -> Vec<u8> {
|
||||
let mut key_share = vec![0u8; X25519MLKEM768_SERVER_KEY_SHARE_LEN];
|
||||
// FakeTLS never derives TLS traffic secrets from this payload; only the
|
||||
// externally visible named group and vector lengths are protocol-facing.
|
||||
rng.fill(&mut key_share[..MLKEM768_SERVER_CIPHERTEXT_LEN]);
|
||||
let x25519_key = gen_fake_x25519_key(rng);
|
||||
key_share[MLKEM768_SERVER_CIPHERTEXT_LEN..].copy_from_slice(&x25519_key);
|
||||
key_share
|
||||
}
|
||||
|
||||
fn mlkem768_encapsulate_to_client(client_key: &[u8], rng: &SecureRandom) -> Option<Vec<u8>> {
|
||||
let key_bytes = MlKemKey::<MlKemEncapsulationKey<MlKem768>>::try_from(client_key).ok()?;
|
||||
let encapsulation_key = MlKemEncapsulationKey::<MlKem768>::new(&key_bytes).ok()?;
|
||||
let mut randomness = [0u8; 32];
|
||||
rng.fill(&mut randomness);
|
||||
let randomness = B32::try_from(randomness.as_slice()).ok()?;
|
||||
let (ciphertext, _shared_key) = encapsulation_key.encapsulate_deterministic(&randomness);
|
||||
let ciphertext = ciphertext.as_slice().to_vec();
|
||||
if ciphertext.len() == MLKEM768_SERVER_CIPHERTEXT_LEN {
|
||||
Some(ciphertext)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a valid X25519MLKEM768 ServerHello key_share for the authenticated ClientHello.
|
||||
pub(crate) fn build_x25519mlkem768_server_key_share(
|
||||
handshake: &[u8],
|
||||
rng: &SecureRandom,
|
||||
) -> Option<Vec<u8>> {
|
||||
let client_key_exchange = client_hello_key_share_group_entry(
|
||||
handshake,
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
X25519MLKEM768_CLIENT_KEY_SHARE_LEN,
|
||||
)?;
|
||||
let client_mlkem_key = client_key_exchange.get(..MLKEM768_CLIENT_ENCAPSULATION_KEY_LEN)?;
|
||||
let client_x25519_key = client_key_exchange.get(MLKEM768_CLIENT_ENCAPSULATION_KEY_LEN..)?;
|
||||
let mlkem_ciphertext = mlkem768_encapsulate_to_client(client_mlkem_key, rng)?;
|
||||
|
||||
let mut client_x25519 = [0u8; X25519_KEY_SHARE_LEN];
|
||||
client_x25519.copy_from_slice(client_x25519_key);
|
||||
let (server_x25519_scalar, server_x25519_key) = gen_x25519_key_pair(rng);
|
||||
let x25519_shared = x25519(server_x25519_scalar, client_x25519);
|
||||
if bool::from(x25519_shared.ct_eq(&[0u8; X25519_KEY_SHARE_LEN])) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut key_share = Vec::with_capacity(X25519MLKEM768_SERVER_KEY_SHARE_LEN);
|
||||
key_share.extend_from_slice(&mlkem_ciphertext);
|
||||
key_share.extend_from_slice(&server_x25519_key);
|
||||
Some(key_share)
|
||||
}
|
||||
|
||||
/// Build a valid X25519 ServerHello key_share for the authenticated ClientHello.
|
||||
pub(crate) fn build_x25519_server_key_share(
|
||||
handshake: &[u8],
|
||||
rng: &SecureRandom,
|
||||
) -> Option<Vec<u8>> {
|
||||
let client_key_exchange = client_hello_key_share_group_entry(
|
||||
handshake,
|
||||
TLS_NAMED_GROUP_X25519,
|
||||
X25519_KEY_SHARE_LEN,
|
||||
)?;
|
||||
let mut client_x25519 = [0u8; X25519_KEY_SHARE_LEN];
|
||||
client_x25519.copy_from_slice(client_key_exchange);
|
||||
let (server_x25519_scalar, server_x25519_key) = gen_x25519_key_pair(rng);
|
||||
let x25519_shared = x25519(server_x25519_scalar, client_x25519);
|
||||
if bool::from(x25519_shared.ct_eq(&[0u8; X25519_KEY_SHARE_LEN])) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(server_x25519_key.to_vec())
|
||||
}
|
||||
|
||||
fn build_server_hello_key_share_for_group(
|
||||
handshake: &[u8],
|
||||
group: u16,
|
||||
rng: &SecureRandom,
|
||||
) -> Option<ServerHelloKeyShare> {
|
||||
let expected_key_exchange_len = client_hello_key_share_group_len(group)?;
|
||||
client_hello_key_share_group_entry(handshake, group, expected_key_exchange_len)?;
|
||||
|
||||
// FakeTLS clients validate ServerHello shape and digest, not TLS traffic
|
||||
// secrets, so the response must mirror the offered group without binding to
|
||||
// the camouflage key bytes embedded in ClientHello.
|
||||
match group {
|
||||
TLS_NAMED_GROUP_X25519MLKEM768 => Some(ServerHelloKeyShare::new(
|
||||
group,
|
||||
gen_fake_x25519mlkem768_server_key_share(rng),
|
||||
)),
|
||||
TLS_NAMED_GROUP_X25519 => Some(ServerHelloKeyShare::new(
|
||||
group,
|
||||
gen_fake_x25519_key(rng).to_vec(),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn server_hello_key_share_candidate_order(preferred_group: Option<u16>) -> [u16; 2] {
|
||||
if preferred_group == Some(TLS_NAMED_GROUP_X25519) {
|
||||
[TLS_NAMED_GROUP_X25519, TLS_NAMED_GROUP_X25519MLKEM768]
|
||||
} else {
|
||||
[TLS_NAMED_GROUP_X25519MLKEM768, TLS_NAMED_GROUP_X25519]
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a ServerHello key_share using a profile-preferred group when possible.
|
||||
pub(crate) fn build_server_hello_key_share(
|
||||
handshake: &[u8],
|
||||
preferred_group: Option<u16>,
|
||||
rng: &SecureRandom,
|
||||
) -> Option<ServerHelloKeyShare> {
|
||||
for group in server_hello_key_share_candidate_order(preferred_group) {
|
||||
if let Some(key_share) = build_server_hello_key_share_for_group(handshake, group, rng) {
|
||||
return Some(key_share);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Build TLS ServerHello response
|
||||
@@ -530,6 +697,10 @@ pub fn build_server_hello(
|
||||
alpn: Option<Vec<u8>>,
|
||||
new_session_tickets: u8,
|
||||
) -> Vec<u8> {
|
||||
let server_key_share = ServerHelloKeyShare::new(
|
||||
TLS_NAMED_GROUP_X25519MLKEM768,
|
||||
gen_fake_x25519mlkem768_server_key_share(rng),
|
||||
);
|
||||
build_server_hello_with_cipher(
|
||||
secret,
|
||||
client_digest,
|
||||
@@ -537,6 +708,7 @@ pub fn build_server_hello(
|
||||
fake_cert_len,
|
||||
rng,
|
||||
cipher_suite::TLS_AES_128_GCM_SHA256,
|
||||
&server_key_share,
|
||||
alpn,
|
||||
new_session_tickets,
|
||||
)
|
||||
@@ -554,18 +726,18 @@ pub(crate) fn build_server_hello_with_cipher(
|
||||
fake_cert_len: usize,
|
||||
rng: &SecureRandom,
|
||||
selected_cipher_suite: [u8; 2],
|
||||
server_key_share: &ServerHelloKeyShare,
|
||||
alpn: Option<Vec<u8>>,
|
||||
new_session_tickets: u8,
|
||||
) -> Vec<u8> {
|
||||
const MIN_APP_DATA: usize = 64;
|
||||
const MAX_APP_DATA: usize = MAX_TLS_CIPHERTEXT_SIZE;
|
||||
let fake_cert_len = fake_cert_len.clamp(MIN_APP_DATA, MAX_APP_DATA);
|
||||
let x25519_key = gen_fake_x25519_key(rng);
|
||||
|
||||
// Build ServerHello
|
||||
let server_hello = ServerHelloBuilder::new(session_id.to_vec())
|
||||
.with_cipher_suite(selected_cipher_suite)
|
||||
.with_x25519_key(&x25519_key)
|
||||
.with_key_share(server_key_share.group(), server_key_share.key_exchange())
|
||||
.with_tls13_version()
|
||||
.build_record();
|
||||
|
||||
@@ -1003,6 +1175,148 @@ fn client_hello_cipher_suites_range(handshake: &[u8]) -> Option<(usize, usize)>
|
||||
Some((pos, cipher_end))
|
||||
}
|
||||
|
||||
fn client_hello_extensions_range(handshake: &[u8]) -> Option<(usize, usize)> {
|
||||
if handshake.len() < 5 || handshake[0] != TLS_RECORD_HANDSHAKE {
|
||||
return None;
|
||||
}
|
||||
|
||||
let record_len = u16::from_be_bytes([handshake[3], handshake[4]]) as usize;
|
||||
let record_end = 5usize.checked_add(record_len)?;
|
||||
if record_end > handshake.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut pos = 5;
|
||||
if handshake.get(pos) != Some(&0x01) {
|
||||
return None;
|
||||
}
|
||||
pos += 1;
|
||||
|
||||
if pos + 3 > record_end {
|
||||
return None;
|
||||
}
|
||||
let handshake_len = ((handshake[pos] as usize) << 16)
|
||||
| ((handshake[pos + 1] as usize) << 8)
|
||||
| handshake[pos + 2] as usize;
|
||||
pos += 3;
|
||||
let handshake_end = pos.checked_add(handshake_len)?;
|
||||
if handshake_end > record_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
if pos + 2 + 32 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
pos += 2 + 32;
|
||||
|
||||
let session_id_len = *handshake.get(pos)? as usize;
|
||||
pos = pos.checked_add(1)?.checked_add(session_id_len)?;
|
||||
if pos + 2 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cipher_len = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]) as usize;
|
||||
if cipher_len == 0 || cipher_len % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
pos += 2;
|
||||
pos = pos.checked_add(cipher_len)?;
|
||||
if pos + 1 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let compression_len = *handshake.get(pos)? as usize;
|
||||
pos = pos.checked_add(1)?.checked_add(compression_len)?;
|
||||
if pos == handshake_end {
|
||||
return Some((handshake_end, handshake_end));
|
||||
}
|
||||
if pos + 2 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let extensions_len = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]) as usize;
|
||||
pos += 2;
|
||||
let extensions_end = pos.checked_add(extensions_len)?;
|
||||
if extensions_end > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some((pos, extensions_end))
|
||||
}
|
||||
|
||||
fn key_share_extension_group_entry<'a>(
|
||||
data: &'a [u8],
|
||||
group: u16,
|
||||
expected_key_exchange_len: usize,
|
||||
) -> Option<&'a [u8]> {
|
||||
if data.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let shares_len = u16::from_be_bytes([data[0], data[1]]) as usize;
|
||||
if shares_len != data.len().saturating_sub(2) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut pos = 2usize;
|
||||
let shares_end = 2 + shares_len;
|
||||
let mut found_group = None;
|
||||
while pos + 4 <= shares_end {
|
||||
let entry_group = u16::from_be_bytes([data[pos], data[pos + 1]]);
|
||||
let key_exchange_len = u16::from_be_bytes([data[pos + 2], data[pos + 3]]) as usize;
|
||||
pos += 4;
|
||||
let Some(key_exchange_end) = pos.checked_add(key_exchange_len) else {
|
||||
return None;
|
||||
};
|
||||
if key_exchange_end > shares_end {
|
||||
return None;
|
||||
}
|
||||
if entry_group == group {
|
||||
if key_exchange_len != expected_key_exchange_len || found_group.is_some() {
|
||||
return None;
|
||||
}
|
||||
found_group = Some(&data[pos..key_exchange_end]);
|
||||
}
|
||||
pos = key_exchange_end;
|
||||
}
|
||||
|
||||
if pos == shares_end { found_group } else { None }
|
||||
}
|
||||
|
||||
fn client_hello_key_share_group_entry<'a>(
|
||||
handshake: &'a [u8],
|
||||
group: u16,
|
||||
expected_key_exchange_len: usize,
|
||||
) -> Option<&'a [u8]> {
|
||||
let Some((mut pos, extensions_end)) = client_hello_extensions_range(handshake) else {
|
||||
return None;
|
||||
};
|
||||
|
||||
while pos + 4 <= extensions_end {
|
||||
let ext_type = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]);
|
||||
let ext_len = u16::from_be_bytes([handshake[pos + 2], handshake[pos + 3]]) as usize;
|
||||
pos += 4;
|
||||
let Some(ext_end) = pos.checked_add(ext_len) else {
|
||||
return None;
|
||||
};
|
||||
if ext_end > extensions_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
if ext_type == extension_type::KEY_SHARE {
|
||||
return key_share_extension_group_entry(
|
||||
&handshake[pos..ext_end],
|
||||
group,
|
||||
expected_key_exchange_len,
|
||||
);
|
||||
}
|
||||
|
||||
pos = ext_end;
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn client_hello_offers_cipher_suite(
|
||||
handshake: &[u8],
|
||||
range: (usize, usize),
|
||||
@@ -1027,20 +1341,23 @@ fn is_tls13_cipher_suite(suite: [u8; 2]) -> bool {
|
||||
/// Select the ServerHello cipher suite from the already-received ClientHello.
|
||||
///
|
||||
/// This is intentionally a borrowed, zero-allocation scan. It runs only for an
|
||||
/// authenticated success response and keeps malformed or unexpected ClientHello
|
||||
/// shapes on the previous fallback behavior.
|
||||
pub(crate) fn select_server_hello_cipher_suite(handshake: &[u8], preferred: [u8; 2]) -> [u8; 2] {
|
||||
/// authenticated success response and fails closed for malformed or unsupported
|
||||
/// ClientHello shapes that cannot produce a DPI-consistent ServerHello.
|
||||
pub(crate) fn select_server_hello_cipher_suite(
|
||||
handshake: &[u8],
|
||||
preferred: [u8; 2],
|
||||
) -> Option<[u8; 2]> {
|
||||
let preferred = if is_tls13_cipher_suite(preferred) {
|
||||
preferred
|
||||
} else {
|
||||
cipher_suite::TLS_AES_128_GCM_SHA256
|
||||
};
|
||||
let Some(range) = client_hello_cipher_suites_range(handshake) else {
|
||||
return preferred;
|
||||
return None;
|
||||
};
|
||||
|
||||
if client_hello_offers_cipher_suite(handshake, range, preferred) {
|
||||
return preferred;
|
||||
return Some(preferred);
|
||||
}
|
||||
|
||||
for fallback in [
|
||||
@@ -1049,11 +1366,43 @@ pub(crate) fn select_server_hello_cipher_suite(handshake: &[u8], preferred: [u8;
|
||||
cipher_suite::TLS_AES_256_GCM_SHA384,
|
||||
] {
|
||||
if client_hello_offers_cipher_suite(handshake, range, fallback) {
|
||||
return fallback;
|
||||
return Some(fallback);
|
||||
}
|
||||
}
|
||||
|
||||
preferred
|
||||
None
|
||||
}
|
||||
|
||||
fn client_hello_key_share_group_len(group: u16) -> Option<usize> {
|
||||
match group {
|
||||
TLS_NAMED_GROUP_X25519MLKEM768 => Some(X25519MLKEM768_CLIENT_KEY_SHARE_LEN),
|
||||
TLS_NAMED_GROUP_X25519 => Some(X25519_KEY_SHARE_LEN),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Select the ServerHello key_share named group from the authenticated ClientHello.
|
||||
///
|
||||
/// Malformed key_share structures fail closed so authenticated but
|
||||
/// DPI-inconsistent ClientHellos take the ordinary masking fallback path.
|
||||
pub(crate) fn select_server_hello_key_share_group(handshake: &[u8]) -> Option<u16> {
|
||||
select_server_hello_key_share_group_with_preference(handshake, None)
|
||||
}
|
||||
|
||||
/// Select the ServerHello key_share named group with an origin-profile preference.
|
||||
pub(crate) fn select_server_hello_key_share_group_with_preference(
|
||||
handshake: &[u8],
|
||||
preferred_group: Option<u16>,
|
||||
) -> Option<u16> {
|
||||
for group in server_hello_key_share_candidate_order(preferred_group) {
|
||||
let expected_key_exchange_len = client_hello_key_share_group_len(group)?;
|
||||
if client_hello_key_share_group_entry(handshake, group, expected_key_exchange_len).is_some()
|
||||
{
|
||||
return Some(group);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if bytes look like a TLS ClientHello
|
||||
|
||||
450
src/protocol/tls_fingerprint.rs
Normal file
450
src/protocol/tls_fingerprint.rs
Normal file
@@ -0,0 +1,450 @@
|
||||
//! Passive JA3 / JA4 TLS ClientHello fingerprinting.
|
||||
|
||||
use crate::crypto::hash::md5;
|
||||
use crate::crypto::sha256;
|
||||
use crate::protocol::constants::TLS_RECORD_HANDSHAKE;
|
||||
|
||||
const EXT_SNI: u16 = 0x0000;
|
||||
const EXT_SUPPORTED_GROUPS: u16 = 0x000a;
|
||||
const EXT_EC_POINT_FORMATS: u16 = 0x000b;
|
||||
const EXT_SIGNATURE_ALGORITHMS: u16 = 0x000d;
|
||||
const EXT_ALPN: u16 = 0x0010;
|
||||
const EXT_SUPPORTED_VERSIONS: u16 = 0x002b;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TlsClientFingerprint {
|
||||
pub ja3: String,
|
||||
pub ja3_raw: String,
|
||||
pub ja4: String,
|
||||
pub ja4_raw: String,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ParsedClientHello {
|
||||
legacy_version: u16,
|
||||
ciphers: Vec<u16>,
|
||||
extensions: Vec<u16>,
|
||||
supported_groups: Vec<u16>,
|
||||
ec_point_formats: Vec<u8>,
|
||||
signature_algorithms: Vec<u16>,
|
||||
supported_versions: Vec<u16>,
|
||||
alpn_first: Option<Vec<u8>>,
|
||||
sni_present: bool,
|
||||
}
|
||||
|
||||
pub fn fingerprint_client_hello(handshake: &[u8]) -> Option<TlsClientFingerprint> {
|
||||
let parsed = parse_client_hello(handshake)?;
|
||||
let ja3_raw = ja3_raw(&parsed);
|
||||
let ja3 = hex::encode(md5(ja3_raw.as_bytes()));
|
||||
let (ja4, ja4_raw) = ja4(&parsed);
|
||||
|
||||
Some(TlsClientFingerprint {
|
||||
ja3,
|
||||
ja3_raw,
|
||||
ja4,
|
||||
ja4_raw,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_client_hello(handshake: &[u8]) -> Option<ParsedClientHello> {
|
||||
if handshake.len() < 5 || handshake[0] != TLS_RECORD_HANDSHAKE {
|
||||
return None;
|
||||
}
|
||||
|
||||
let record_len = read_u16_at(handshake, 3)? as usize;
|
||||
let record_end = 5usize.checked_add(record_len)?;
|
||||
if record_end > handshake.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut pos = 5usize;
|
||||
if *handshake.get(pos)? != 0x01 {
|
||||
return None;
|
||||
}
|
||||
pos = pos.checked_add(1)?;
|
||||
|
||||
if pos + 3 > record_end {
|
||||
return None;
|
||||
}
|
||||
let handshake_len = ((usize::from(handshake[pos])) << 16)
|
||||
| ((usize::from(handshake[pos + 1])) << 8)
|
||||
| usize::from(handshake[pos + 2]);
|
||||
pos = pos.checked_add(3)?;
|
||||
let handshake_end = pos.checked_add(handshake_len)?;
|
||||
if handshake_end > record_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
if pos + 2 + 32 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
let legacy_version = read_u16_at(handshake, pos)?;
|
||||
pos = pos.checked_add(2 + 32)?;
|
||||
|
||||
let session_id_len = usize::from(*handshake.get(pos)?);
|
||||
pos = pos.checked_add(1)?.checked_add(session_id_len)?;
|
||||
if pos + 2 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cipher_len = read_u16_at(handshake, pos)? as usize;
|
||||
pos = pos.checked_add(2)?;
|
||||
let cipher_end = pos.checked_add(cipher_len)?;
|
||||
if cipher_end > handshake_end || cipher_len % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
let mut ciphers = Vec::with_capacity(cipher_len / 2);
|
||||
while pos + 1 < cipher_end {
|
||||
let value = read_u16_at(handshake, pos)?;
|
||||
if !is_grease(value) {
|
||||
ciphers.push(value);
|
||||
}
|
||||
pos = pos.checked_add(2)?;
|
||||
}
|
||||
|
||||
let comp_len = usize::from(*handshake.get(pos)?);
|
||||
pos = pos.checked_add(1)?.checked_add(comp_len)?;
|
||||
if pos > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut parsed = ParsedClientHello {
|
||||
legacy_version,
|
||||
ciphers,
|
||||
..ParsedClientHello::default()
|
||||
};
|
||||
|
||||
if pos == handshake_end {
|
||||
return Some(parsed);
|
||||
}
|
||||
if pos + 2 > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let ext_len = read_u16_at(handshake, pos)? as usize;
|
||||
pos = pos.checked_add(2)?;
|
||||
let ext_end = pos.checked_add(ext_len)?;
|
||||
if ext_end > handshake_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
while pos + 4 <= ext_end {
|
||||
let etype = read_u16_at(handshake, pos)?;
|
||||
let elen = read_u16_at(handshake, pos + 2)? as usize;
|
||||
pos = pos.checked_add(4)?;
|
||||
let data_end = pos.checked_add(elen)?;
|
||||
if data_end > ext_end {
|
||||
return None;
|
||||
}
|
||||
let data = handshake.get(pos..data_end)?;
|
||||
|
||||
if !is_grease(etype) {
|
||||
parsed.extensions.push(etype);
|
||||
match etype {
|
||||
EXT_SNI => parsed.sni_present = true,
|
||||
EXT_SUPPORTED_GROUPS => {
|
||||
parsed.supported_groups = parse_u16_vector(data, 2)?;
|
||||
}
|
||||
EXT_EC_POINT_FORMATS => {
|
||||
parsed.ec_point_formats = parse_u8_vector(data)?;
|
||||
}
|
||||
EXT_SIGNATURE_ALGORITHMS => {
|
||||
parsed.signature_algorithms = parse_u16_vector(data, 2)?;
|
||||
}
|
||||
EXT_ALPN => {
|
||||
parsed.alpn_first = parse_alpn_first(data)?;
|
||||
}
|
||||
EXT_SUPPORTED_VERSIONS => {
|
||||
parsed.supported_versions = parse_u16_vector(data, 1)?;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
pos = data_end;
|
||||
}
|
||||
|
||||
if pos != ext_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(parsed)
|
||||
}
|
||||
|
||||
fn parse_u16_vector(data: &[u8], len_prefix_len: usize) -> Option<Vec<u16>> {
|
||||
let (list_len, mut pos) = match len_prefix_len {
|
||||
1 => (usize::from(*data.first()?), 1usize),
|
||||
2 => (read_u16_at(data, 0)? as usize, 2usize),
|
||||
_ => return None,
|
||||
};
|
||||
let list_end = pos.checked_add(list_len)?;
|
||||
if list_end > data.len() || list_len % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut out = Vec::with_capacity(list_len / 2);
|
||||
while pos + 1 < list_end {
|
||||
let value = read_u16_at(data, pos)?;
|
||||
if !is_grease(value) {
|
||||
out.push(value);
|
||||
}
|
||||
pos = pos.checked_add(2)?;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn parse_u8_vector(data: &[u8]) -> Option<Vec<u8>> {
|
||||
let list_len = usize::from(*data.first()?);
|
||||
let list_start = 1usize;
|
||||
let list_end = list_start.checked_add(list_len)?;
|
||||
if list_end > data.len() {
|
||||
return None;
|
||||
}
|
||||
Some(data.get(list_start..list_end)?.to_vec())
|
||||
}
|
||||
|
||||
fn parse_alpn_first(data: &[u8]) -> Option<Option<Vec<u8>>> {
|
||||
if data.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let list_len = read_u16_at(data, 0)? as usize;
|
||||
let mut pos = 2usize;
|
||||
let list_end = pos.checked_add(list_len)?;
|
||||
if list_end > data.len() {
|
||||
return None;
|
||||
}
|
||||
if pos == list_end {
|
||||
return Some(None);
|
||||
}
|
||||
|
||||
let protocol_len = usize::from(*data.get(pos)?);
|
||||
pos = pos.checked_add(1)?;
|
||||
let protocol_end = pos.checked_add(protocol_len)?;
|
||||
if protocol_end > list_end {
|
||||
return None;
|
||||
}
|
||||
if protocol_len == 0 {
|
||||
return Some(None);
|
||||
}
|
||||
Some(Some(data.get(pos..protocol_end)?.to_vec()))
|
||||
}
|
||||
|
||||
fn ja3_raw(parsed: &ParsedClientHello) -> String {
|
||||
format!(
|
||||
"{},{},{},{},{}",
|
||||
parsed.legacy_version,
|
||||
join_decimal_u16(&parsed.ciphers),
|
||||
join_decimal_u16(&parsed.extensions),
|
||||
join_decimal_u16(&parsed.supported_groups),
|
||||
join_decimal_u8(&parsed.ec_point_formats)
|
||||
)
|
||||
}
|
||||
|
||||
fn ja4(parsed: &ParsedClientHello) -> (String, String) {
|
||||
let a = format!(
|
||||
"t{}{}{:02}{:02}{}",
|
||||
ja4_version_code(parsed),
|
||||
if parsed.sni_present { "d" } else { "i" },
|
||||
count_ja4(parsed.ciphers.len()),
|
||||
count_ja4(parsed.extensions.len()),
|
||||
ja4_alpn_marker(parsed.alpn_first.as_deref())
|
||||
);
|
||||
|
||||
let mut ciphers = parsed.ciphers.clone();
|
||||
ciphers.sort_unstable();
|
||||
let cipher_raw = join_hex_u16(&ciphers);
|
||||
let cipher_hash = if ciphers.is_empty() {
|
||||
"000000000000".to_string()
|
||||
} else {
|
||||
sha256_truncated_12(&cipher_raw)
|
||||
};
|
||||
|
||||
let mut extensions_for_hash = parsed
|
||||
.extensions
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|value| *value != EXT_SNI && *value != EXT_ALPN)
|
||||
.collect::<Vec<_>>();
|
||||
extensions_for_hash.sort_unstable();
|
||||
let extension_raw = join_hex_u16(&extensions_for_hash);
|
||||
let signature_raw = join_hex_u16(&parsed.signature_algorithms);
|
||||
let extension_hash_input = if signature_raw.is_empty() {
|
||||
extension_raw.clone()
|
||||
} else {
|
||||
format!("{extension_raw}_{signature_raw}")
|
||||
};
|
||||
let extension_hash = if extensions_for_hash.is_empty() {
|
||||
"000000000000".to_string()
|
||||
} else {
|
||||
sha256_truncated_12(&extension_hash_input)
|
||||
};
|
||||
|
||||
(
|
||||
format!("{a}_{cipher_hash}_{extension_hash}"),
|
||||
format!("{a}_{cipher_raw}_{extension_hash_input}"),
|
||||
)
|
||||
}
|
||||
|
||||
fn ja4_version_code(parsed: &ParsedClientHello) -> &'static str {
|
||||
let version = parsed
|
||||
.supported_versions
|
||||
.iter()
|
||||
.copied()
|
||||
.max()
|
||||
.unwrap_or(parsed.legacy_version);
|
||||
match version {
|
||||
0x0304 => "13",
|
||||
0x0303 => "12",
|
||||
0x0302 => "11",
|
||||
0x0301 => "10",
|
||||
0x0300 => "s3",
|
||||
0x0002 => "s2",
|
||||
0xfeff => "d1",
|
||||
0xfefd => "d2",
|
||||
0xfefc => "d3",
|
||||
_ => "00",
|
||||
}
|
||||
}
|
||||
|
||||
fn ja4_alpn_marker(alpn_first: Option<&[u8]>) -> String {
|
||||
let Some(value) = alpn_first else {
|
||||
return "00".to_string();
|
||||
};
|
||||
let Some(first) = value.first().copied() else {
|
||||
return "00".to_string();
|
||||
};
|
||||
let last = value.last().copied().unwrap_or(first);
|
||||
if first.is_ascii_alphanumeric() && last.is_ascii_alphanumeric() {
|
||||
return format!("{}{}", first as char, last as char);
|
||||
}
|
||||
|
||||
let encoded = hex::encode(value);
|
||||
if encoded.is_empty() {
|
||||
return "00".to_string();
|
||||
}
|
||||
let first_hex = encoded.as_bytes()[0] as char;
|
||||
let last_hex = encoded.as_bytes()[encoded.len().saturating_sub(1)] as char;
|
||||
format!("{first_hex}{last_hex}")
|
||||
}
|
||||
|
||||
fn count_ja4(count: usize) -> usize {
|
||||
count.min(99)
|
||||
}
|
||||
|
||||
fn sha256_truncated_12(input: &str) -> String {
|
||||
let mut encoded = hex::encode(sha256(input.as_bytes()));
|
||||
encoded.truncate(12);
|
||||
encoded
|
||||
}
|
||||
|
||||
fn join_decimal_u16(values: &[u16]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(u16::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join("-")
|
||||
}
|
||||
|
||||
fn join_decimal_u8(values: &[u8]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(u8::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join("-")
|
||||
}
|
||||
|
||||
fn join_hex_u16(values: &[u16]) -> String {
|
||||
values
|
||||
.iter()
|
||||
.map(|value| format!("{value:04x}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(",")
|
||||
}
|
||||
|
||||
fn read_u16_at(buf: &[u8], pos: usize) -> Option<u16> {
|
||||
Some(u16::from_be_bytes([
|
||||
*buf.get(pos)?,
|
||||
*buf.get(pos.checked_add(1)?)?,
|
||||
]))
|
||||
}
|
||||
|
||||
fn is_grease(value: u16) -> bool {
|
||||
let high = (value >> 8) as u8;
|
||||
let low = value as u8;
|
||||
high == low && (high & 0x0f) == 0x0a
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_client_hello() -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
body.extend_from_slice(&[0x03, 0x03]);
|
||||
body.extend_from_slice(&[0x11; 32]);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&10u16.to_be_bytes());
|
||||
body.extend_from_slice(&[0x0a, 0x0a, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f, 0x00, 0xff]);
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
|
||||
let mut extensions = Vec::new();
|
||||
append_ext(&mut extensions, EXT_SNI, &[0, 0]);
|
||||
append_ext(&mut extensions, EXT_ALPN, &[0, 3, 2, b'h', b'2']);
|
||||
append_ext(
|
||||
&mut extensions,
|
||||
EXT_SUPPORTED_GROUPS,
|
||||
&[0, 6, 0x0a, 0x0a, 0x00, 0x17, 0x00, 0x1d],
|
||||
);
|
||||
append_ext(&mut extensions, EXT_EC_POINT_FORMATS, &[1, 0]);
|
||||
append_ext(
|
||||
&mut extensions,
|
||||
EXT_SIGNATURE_ALGORITHMS,
|
||||
&[0, 4, 0x04, 0x03, 0x08, 0x04],
|
||||
);
|
||||
append_ext(
|
||||
&mut extensions,
|
||||
EXT_SUPPORTED_VERSIONS,
|
||||
&[4, 0x03, 0x04, 0x03, 0x03],
|
||||
);
|
||||
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&extensions);
|
||||
|
||||
let mut record = Vec::new();
|
||||
record.push(TLS_RECORD_HANDSHAKE);
|
||||
record.extend_from_slice(&[0x03, 0x01]);
|
||||
record.extend_from_slice(&((body.len() + 4) as u16).to_be_bytes());
|
||||
record.push(0x01);
|
||||
record.extend_from_slice(&[
|
||||
((body.len() >> 16) & 0xff) as u8,
|
||||
((body.len() >> 8) & 0xff) as u8,
|
||||
(body.len() & 0xff) as u8,
|
||||
]);
|
||||
record.extend_from_slice(&body);
|
||||
record
|
||||
}
|
||||
|
||||
fn append_ext(out: &mut Vec<u8>, etype: u16, data: &[u8]) {
|
||||
out.extend_from_slice(&etype.to_be_bytes());
|
||||
out.extend_from_slice(&(data.len() as u16).to_be_bytes());
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ja3_and_ja4_ignore_grease_and_remain_stable() {
|
||||
let fp = fingerprint_client_hello(&sample_client_hello())
|
||||
.expect("sample ClientHello must fingerprint");
|
||||
assert_eq!(
|
||||
fp.ja3_raw,
|
||||
"771,4865-4866-49199-255,0-16-10-11-13-43,23-29,0"
|
||||
);
|
||||
assert!(fp.ja4.starts_with("t13d0406h2_"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_client_hello_returns_none() {
|
||||
let mut hello = sample_client_hello();
|
||||
hello.truncate(12);
|
||||
assert!(fingerprint_client_hello(&hello).is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,4 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
// Adaptive buffer policy is staged and retained for deterministic rollout.
|
||||
// Keep definitions compiled for compatibility and security test scaffolding.
|
||||
// Adaptive buffer policy shared by active Direct relay sessions.
|
||||
|
||||
use dashmap::DashMap;
|
||||
use std::cmp::max;
|
||||
@@ -13,29 +10,42 @@ const PROFILE_TTL: Duration = Duration::from_secs(300);
|
||||
const THROUGHPUT_UP_BPS: f64 = 8_000_000.0;
|
||||
const THROUGHPUT_DOWN_BPS: f64 = 2_000_000.0;
|
||||
const RATIO_CONFIRM_THRESHOLD: f64 = 1.12;
|
||||
const TIER1_HOLD_TICKS: u32 = 8;
|
||||
const TIER2_HOLD_TICKS: u32 = 4;
|
||||
const QUIET_DEMOTE_TICKS: u32 = 480;
|
||||
const HARD_COOLDOWN_TICKS: u32 = 20;
|
||||
const TIER1_HOLD: Duration = Duration::from_secs(2);
|
||||
const TIER2_HOLD: Duration = Duration::from_secs(1);
|
||||
const QUIET_DEMOTE: Duration = Duration::from_secs(120);
|
||||
const HARD_COOLDOWN: Duration = Duration::from_secs(5);
|
||||
const SUSTAINED_PRESSURE_DEMOTE: Duration = Duration::from_secs(30);
|
||||
const PRESSURE_DEMOTE_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
const HARD_PENDING_THRESHOLD: u32 = 3;
|
||||
const HARD_PARTIAL_RATIO_THRESHOLD: f64 = 0.25;
|
||||
#[cfg(test)]
|
||||
const DIRECT_C2S_CAP_BYTES: usize = 128 * 1024;
|
||||
#[cfg(test)]
|
||||
const DIRECT_S2C_CAP_BYTES: usize = 512 * 1024;
|
||||
#[cfg(test)]
|
||||
const ME_FRAMES_CAP: usize = 96;
|
||||
#[cfg(test)]
|
||||
const ME_BYTES_CAP: usize = 384 * 1024;
|
||||
#[cfg(test)]
|
||||
const ME_DELAY_MIN_US: u64 = 150;
|
||||
const MAX_USER_PROFILES_ENTRIES: usize = 50_000;
|
||||
const MAX_USER_KEY_BYTES: usize = 512;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
/// Per-session Direct copy-buffer capacity tier.
|
||||
pub enum AdaptiveTier {
|
||||
/// Conservative baseline capacity.
|
||||
Base = 0,
|
||||
/// First throughput promotion.
|
||||
Tier1 = 1,
|
||||
/// Sustained bidirectional pressure promotion.
|
||||
Tier2 = 2,
|
||||
/// Configured per-direction ceilings.
|
||||
Tier3 = 3,
|
||||
}
|
||||
|
||||
impl AdaptiveTier {
|
||||
/// Returns the next larger tier, saturating at `Tier3`.
|
||||
pub fn promote(self) -> Self {
|
||||
match self {
|
||||
Self::Base => Self::Tier1,
|
||||
@@ -45,6 +55,7 @@ impl AdaptiveTier {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the next smaller tier, saturating at `Base`.
|
||||
pub fn demote(self) -> Self {
|
||||
match self {
|
||||
Self::Base => Self::Base,
|
||||
@@ -54,6 +65,7 @@ impl AdaptiveTier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn ratio(self) -> (usize, usize) {
|
||||
match self {
|
||||
Self::Base => (1, 1),
|
||||
@@ -63,49 +75,70 @@ impl AdaptiveTier {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the stable numeric tier used by bounded metrics.
|
||||
pub fn as_u8(self) -> u8 {
|
||||
self as u8
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
/// Signal that caused an accepted adaptive tier transition.
|
||||
pub enum TierTransitionReason {
|
||||
/// Sustained throughput and directional ratio confirmation.
|
||||
SoftConfirmed,
|
||||
/// Short pending or partial-write pressure burst.
|
||||
HardPressure,
|
||||
/// Sustained low-throughput period.
|
||||
QuietDemotion,
|
||||
/// Sustained pending or partial-write pressure.
|
||||
SustainedWritePressure,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
/// Proposed transition emitted by the per-session controller.
|
||||
pub struct TierTransition {
|
||||
/// Tier active before the observation.
|
||||
pub from: AdaptiveTier,
|
||||
/// Tier requested after the observation.
|
||||
pub to: AdaptiveTier,
|
||||
/// Pressure or throughput condition that requested the transition.
|
||||
pub reason: TierTransitionReason,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
/// Directional byte and write-pressure deltas for one observation period.
|
||||
pub struct RelaySignalSample {
|
||||
/// Client-to-DC bytes copied during the period.
|
||||
pub c2s_bytes: u64,
|
||||
/// Bytes offered to DC-to-client writes during the period.
|
||||
pub s2c_requested_bytes: u64,
|
||||
/// Bytes accepted by DC-to-client writes during the period.
|
||||
pub s2c_written_bytes: u64,
|
||||
/// Successful DC-to-client write operations during the period.
|
||||
pub s2c_write_ops: u64,
|
||||
/// Partial DC-to-client write operations during the period.
|
||||
pub s2c_partial_writes: u64,
|
||||
/// Consecutive pending DC-to-client writes at sample time.
|
||||
pub s2c_consecutive_pending_writes: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
/// Stateful hysteresis controller for one active Direct session.
|
||||
pub struct SessionAdaptiveController {
|
||||
tier: AdaptiveTier,
|
||||
max_tier_seen: AdaptiveTier,
|
||||
throughput_ema_bps: f64,
|
||||
incoming_ema_bps: f64,
|
||||
outgoing_ema_bps: f64,
|
||||
tier1_hold_ticks: u32,
|
||||
tier2_hold_ticks: u32,
|
||||
quiet_ticks: u32,
|
||||
hard_cooldown_ticks: u32,
|
||||
tier1_hold: Duration,
|
||||
tier2_hold: Duration,
|
||||
quiet: Duration,
|
||||
hard_cooldown: Duration,
|
||||
sustained_pressure: Duration,
|
||||
}
|
||||
|
||||
impl SessionAdaptiveController {
|
||||
/// Creates a controller at the tier whose memory reservation was accepted.
|
||||
pub fn new(initial_tier: AdaptiveTier) -> Self {
|
||||
Self {
|
||||
tier: initial_tier,
|
||||
@@ -113,25 +146,33 @@ impl SessionAdaptiveController {
|
||||
throughput_ema_bps: 0.0,
|
||||
incoming_ema_bps: 0.0,
|
||||
outgoing_ema_bps: 0.0,
|
||||
tier1_hold_ticks: 0,
|
||||
tier2_hold_ticks: 0,
|
||||
quiet_ticks: 0,
|
||||
hard_cooldown_ticks: 0,
|
||||
tier1_hold: Duration::ZERO,
|
||||
tier2_hold: Duration::ZERO,
|
||||
quiet: Duration::ZERO,
|
||||
hard_cooldown: Duration::ZERO,
|
||||
sustained_pressure: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the highest tier proposed by controller observations.
|
||||
#[allow(dead_code)]
|
||||
pub fn max_tier_seen(&self) -> AdaptiveTier {
|
||||
self.max_tier_seen
|
||||
}
|
||||
|
||||
/// Returns the controller's current logical tier.
|
||||
pub fn tier(&self) -> AdaptiveTier {
|
||||
self.tier
|
||||
}
|
||||
|
||||
/// Observes one period and returns at most one hysteresis-controlled transition.
|
||||
pub fn observe(&mut self, sample: RelaySignalSample, tick_secs: f64) -> Option<TierTransition> {
|
||||
if tick_secs <= f64::EPSILON {
|
||||
return None;
|
||||
}
|
||||
|
||||
if self.hard_cooldown_ticks > 0 {
|
||||
self.hard_cooldown_ticks -= 1;
|
||||
}
|
||||
let tick = Duration::from_secs_f64(tick_secs);
|
||||
self.hard_cooldown = self.hard_cooldown.saturating_sub(tick);
|
||||
|
||||
let c2s_bps = (sample.c2s_bytes as f64 * 8.0) / tick_secs;
|
||||
let incoming_bps = (sample.s2c_requested_bytes as f64 * 8.0) / tick_secs;
|
||||
@@ -144,9 +185,9 @@ impl SessionAdaptiveController {
|
||||
|
||||
let tier1_now = self.throughput_ema_bps >= THROUGHPUT_UP_BPS;
|
||||
if tier1_now {
|
||||
self.tier1_hold_ticks = self.tier1_hold_ticks.saturating_add(1);
|
||||
self.tier1_hold = self.tier1_hold.saturating_add(tick);
|
||||
} else {
|
||||
self.tier1_hold_ticks = 0;
|
||||
self.tier1_hold = Duration::ZERO;
|
||||
}
|
||||
|
||||
let ratio = if self.outgoing_ema_bps <= f64::EPSILON {
|
||||
@@ -156,9 +197,9 @@ impl SessionAdaptiveController {
|
||||
};
|
||||
let tier2_now = ratio >= RATIO_CONFIRM_THRESHOLD;
|
||||
if tier2_now {
|
||||
self.tier2_hold_ticks = self.tier2_hold_ticks.saturating_add(1);
|
||||
self.tier2_hold = self.tier2_hold.saturating_add(tick);
|
||||
} else {
|
||||
self.tier2_hold_ticks = 0;
|
||||
self.tier2_hold = Duration::ZERO;
|
||||
}
|
||||
|
||||
let partial_ratio = if sample.s2c_write_ops == 0 {
|
||||
@@ -169,24 +210,37 @@ impl SessionAdaptiveController {
|
||||
let hard_now = sample.s2c_consecutive_pending_writes >= HARD_PENDING_THRESHOLD
|
||||
|| partial_ratio >= HARD_PARTIAL_RATIO_THRESHOLD;
|
||||
|
||||
if hard_now && self.hard_cooldown_ticks == 0 {
|
||||
return self.promote(TierTransitionReason::HardPressure, HARD_COOLDOWN_TICKS);
|
||||
if hard_now {
|
||||
self.sustained_pressure = self.sustained_pressure.saturating_add(tick);
|
||||
if self.sustained_pressure >= SUSTAINED_PRESSURE_DEMOTE {
|
||||
self.sustained_pressure = Duration::ZERO;
|
||||
return self.demote(
|
||||
TierTransitionReason::SustainedWritePressure,
|
||||
PRESSURE_DEMOTE_COOLDOWN,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
self.sustained_pressure = Duration::ZERO;
|
||||
}
|
||||
|
||||
if self.tier1_hold_ticks >= TIER1_HOLD_TICKS && self.tier2_hold_ticks >= TIER2_HOLD_TICKS {
|
||||
return self.promote(TierTransitionReason::SoftConfirmed, 0);
|
||||
if hard_now && self.hard_cooldown.is_zero() {
|
||||
return self.promote(TierTransitionReason::HardPressure, HARD_COOLDOWN);
|
||||
}
|
||||
|
||||
if self.tier1_hold >= TIER1_HOLD && self.tier2_hold >= TIER2_HOLD {
|
||||
return self.promote(TierTransitionReason::SoftConfirmed, Duration::ZERO);
|
||||
}
|
||||
|
||||
let demote_candidate =
|
||||
self.throughput_ema_bps < THROUGHPUT_DOWN_BPS && !tier2_now && !hard_now;
|
||||
if demote_candidate {
|
||||
self.quiet_ticks = self.quiet_ticks.saturating_add(1);
|
||||
if self.quiet_ticks >= QUIET_DEMOTE_TICKS {
|
||||
self.quiet_ticks = 0;
|
||||
return self.demote(TierTransitionReason::QuietDemotion);
|
||||
self.quiet = self.quiet.saturating_add(tick);
|
||||
if self.quiet >= QUIET_DEMOTE {
|
||||
self.quiet = Duration::ZERO;
|
||||
return self.demote(TierTransitionReason::QuietDemotion, Duration::ZERO);
|
||||
}
|
||||
} else {
|
||||
self.quiet_ticks = 0;
|
||||
self.quiet = Duration::ZERO;
|
||||
}
|
||||
|
||||
None
|
||||
@@ -195,7 +249,7 @@ impl SessionAdaptiveController {
|
||||
fn promote(
|
||||
&mut self,
|
||||
reason: TierTransitionReason,
|
||||
hard_cooldown_ticks: u32,
|
||||
hard_cooldown: Duration,
|
||||
) -> Option<TierTransition> {
|
||||
let from = self.tier;
|
||||
let to = from.promote();
|
||||
@@ -204,22 +258,27 @@ impl SessionAdaptiveController {
|
||||
}
|
||||
self.tier = to;
|
||||
self.max_tier_seen = max(self.max_tier_seen, to);
|
||||
self.hard_cooldown_ticks = hard_cooldown_ticks;
|
||||
self.tier1_hold_ticks = 0;
|
||||
self.tier2_hold_ticks = 0;
|
||||
self.quiet_ticks = 0;
|
||||
self.hard_cooldown = hard_cooldown;
|
||||
self.tier1_hold = Duration::ZERO;
|
||||
self.tier2_hold = Duration::ZERO;
|
||||
self.quiet = Duration::ZERO;
|
||||
Some(TierTransition { from, to, reason })
|
||||
}
|
||||
|
||||
fn demote(&mut self, reason: TierTransitionReason) -> Option<TierTransition> {
|
||||
fn demote(
|
||||
&mut self,
|
||||
reason: TierTransitionReason,
|
||||
hard_cooldown: Duration,
|
||||
) -> Option<TierTransition> {
|
||||
let from = self.tier;
|
||||
let to = from.demote();
|
||||
if from == to {
|
||||
return None;
|
||||
}
|
||||
self.tier = to;
|
||||
self.tier1_hold_ticks = 0;
|
||||
self.tier2_hold_ticks = 0;
|
||||
self.hard_cooldown = hard_cooldown;
|
||||
self.tier1_hold = Duration::ZERO;
|
||||
self.tier2_hold = Duration::ZERO;
|
||||
Some(TierTransition { from, to, reason })
|
||||
}
|
||||
}
|
||||
@@ -235,6 +294,8 @@ fn profiles() -> &'static DashMap<String, UserAdaptiveProfile> {
|
||||
USER_PROFILES.get_or_init(DashMap::new)
|
||||
}
|
||||
|
||||
/// Returns a fresh user's recent successful Direct tier, or `Base` when stale.
|
||||
#[allow(dead_code)]
|
||||
pub fn seed_tier_for_user(user: &str) -> AdaptiveTier {
|
||||
if user.len() > MAX_USER_KEY_BYTES {
|
||||
return AdaptiveTier::Base;
|
||||
@@ -253,6 +314,8 @@ pub fn seed_tier_for_user(user: &str) -> AdaptiveTier {
|
||||
AdaptiveTier::Base
|
||||
}
|
||||
|
||||
/// Records the highest successfully allocated tier for bounded session seeding.
|
||||
#[allow(dead_code)]
|
||||
pub fn record_user_tier(user: &str, tier: AdaptiveTier) {
|
||||
if user.len() > MAX_USER_KEY_BYTES {
|
||||
return;
|
||||
@@ -282,6 +345,8 @@ pub fn record_user_tier(user: &str, tier: AdaptiveTier) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Returns the legacy staged scaling policy retained by security fixtures.
|
||||
pub fn direct_copy_buffers_for_tier(
|
||||
tier: AdaptiveTier,
|
||||
base_c2s: usize,
|
||||
@@ -294,6 +359,32 @@ pub fn direct_copy_buffers_for_tier(
|
||||
)
|
||||
}
|
||||
|
||||
/// Maps an adaptive tier to independent capacities within configured ceilings.
|
||||
pub(crate) fn direct_copy_buffers_for_tier_with_ceilings(
|
||||
tier: AdaptiveTier,
|
||||
base_c2s: usize,
|
||||
base_s2c: usize,
|
||||
ceiling_c2s: usize,
|
||||
ceiling_s2c: usize,
|
||||
) -> (usize, usize) {
|
||||
(
|
||||
direct_direction_size(tier, base_c2s, ceiling_c2s),
|
||||
direct_direction_size(tier, base_s2c, ceiling_s2c),
|
||||
)
|
||||
}
|
||||
|
||||
fn direct_direction_size(tier: AdaptiveTier, base: usize, ceiling: usize) -> usize {
|
||||
let target = match tier {
|
||||
AdaptiveTier::Base => base,
|
||||
AdaptiveTier::Tier1 => ceiling / 4,
|
||||
AdaptiveTier::Tier2 => ceiling / 2,
|
||||
AdaptiveTier::Tier3 => ceiling,
|
||||
};
|
||||
target.max(base).min(ceiling.max(base)).max(1)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Returns the staged Middle-End flush policy retained by security fixtures.
|
||||
pub fn me_flush_policy_for_tier(
|
||||
tier: AdaptiveTier,
|
||||
base_frames: usize,
|
||||
@@ -323,6 +414,7 @@ fn ema(prev: f64, value: f64) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn scale(base: usize, numerator: usize, denominator: usize, cap: usize) -> usize {
|
||||
let scaled = base
|
||||
.saturating_mul(numerator)
|
||||
@@ -338,6 +430,10 @@ mod adaptive_buffers_security_tests;
|
||||
#[path = "tests/adaptive_buffers_record_race_security_tests.rs"]
|
||||
mod adaptive_buffers_record_race_security_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/adaptive_direct_budget_policy_tests.rs"]
|
||||
mod adaptive_direct_budget_policy_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -396,7 +492,7 @@ mod tests {
|
||||
fn test_quiet_demotion_is_slow_and_stepwise() {
|
||||
let mut ctrl = SessionAdaptiveController::new(AdaptiveTier::Tier2);
|
||||
let mut demotion = None;
|
||||
for _ in 0..QUIET_DEMOTE_TICKS {
|
||||
for _ in 0..480 {
|
||||
demotion = ctrl.observe(sample(1, 1, 1, 1, 0, 0), 0.25);
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ use crate::error::{HandshakeResult, ProxyError, Result, StreamError};
|
||||
use crate::ip_tracker::UserIpTracker;
|
||||
use crate::protocol::constants::*;
|
||||
use crate::protocol::tls;
|
||||
use crate::protocol::tls_fingerprint::{self, TlsClientFingerprint};
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::stats::{ReplayChecker, Stats};
|
||||
use crate::stream::{BufferPool, CryptoReader, CryptoWriter};
|
||||
@@ -112,7 +113,7 @@ use crate::proxy::handshake::{
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::proxy::handshake::{handle_mtproto_handshake, handle_tls_handshake};
|
||||
use crate::proxy::masking::handle_bad_client;
|
||||
use crate::proxy::masking::handle_bad_client_with_shared;
|
||||
use crate::proxy::middle_relay::handle_via_middle_proxy;
|
||||
use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController};
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
@@ -309,6 +310,7 @@ fn masking_outcome<R, W>(
|
||||
local_addr: SocketAddr,
|
||||
config: Arc<ProxyConfig>,
|
||||
beobachten: Arc<BeobachtenStore>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> HandshakeOutcome
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
@@ -324,7 +326,7 @@ where
|
||||
)
|
||||
.await;
|
||||
|
||||
handle_bad_client(
|
||||
handle_bad_client_with_shared(
|
||||
reader,
|
||||
writer,
|
||||
&initial_data,
|
||||
@@ -332,6 +334,7 @@ where
|
||||
local_addr,
|
||||
&config,
|
||||
&beobachten,
|
||||
shared.as_ref(),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
@@ -350,6 +353,60 @@ fn record_beobachten_class(
|
||||
beobachten.record(class, peer_ip, beobachten_ttl(config));
|
||||
}
|
||||
|
||||
fn tls_fingerprint_collection_enabled(config: &ProxyConfig) -> bool {
|
||||
config.general.beobachten || config.server.api.runtime_edge_enabled
|
||||
}
|
||||
|
||||
fn observe_tls_client_fingerprint(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
handshake: &[u8],
|
||||
) -> Option<TlsClientFingerprint> {
|
||||
if !tls_fingerprint_collection_enabled(config) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match tls_fingerprint::fingerprint_client_hello(handshake) {
|
||||
Some(fingerprint) => {
|
||||
stats.record_tls_fingerprint_observed(&fingerprint, peer_ip, beobachten_ttl(config));
|
||||
Some(fingerprint)
|
||||
}
|
||||
None => {
|
||||
stats.increment_tls_fingerprint_parse_error();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_tls_fingerprint_auth_success(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
fingerprint: Option<&TlsClientFingerprint>,
|
||||
user: &str,
|
||||
) {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
stats.record_tls_fingerprint_auth_success(
|
||||
fingerprint,
|
||||
peer_ip,
|
||||
user,
|
||||
beobachten_ttl(config),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_tls_fingerprint_bad_or_probe(
|
||||
stats: &Stats,
|
||||
config: &ProxyConfig,
|
||||
peer_ip: IpAddr,
|
||||
fingerprint: Option<&TlsClientFingerprint>,
|
||||
) {
|
||||
if let Some(fingerprint) = fingerprint {
|
||||
stats.record_tls_fingerprint_bad_or_probe(fingerprint, peer_ip, beobachten_ttl(config));
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_expected_64_got_0(kind: std::io::ErrorKind) -> Option<&'static str> {
|
||||
match kind {
|
||||
std::io::ErrorKind::UnexpectedEof => Some("expected_64_got_0_unexpected_eof"),
|
||||
@@ -663,6 +720,7 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -684,6 +742,7 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
};
|
||||
@@ -702,9 +761,13 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let tls_fingerprint =
|
||||
observe_tls_client_fingerprint(stats.as_ref(), &config, real_peer.ip(), &handshake);
|
||||
|
||||
let (read_half, write_half) = tokio::io::split(stream);
|
||||
|
||||
let (mut tls_reader, tls_writer, tls_user) = match handle_tls_handshake_with_shared(
|
||||
@@ -715,6 +778,12 @@ where
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
@@ -723,13 +792,27 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => {
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
increment_bad_on_unknown_tls_sni(stats.as_ref(), &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
record_tls_fingerprint_auth_success(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
real_peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
tls_user.as_str(),
|
||||
);
|
||||
|
||||
debug!(peer = %peer, "Reading MTProto handshake through TLS");
|
||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?;
|
||||
@@ -767,6 +850,7 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
@@ -796,6 +880,7 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -821,6 +906,7 @@ where
|
||||
local_addr,
|
||||
config.clone(),
|
||||
beobachten.clone(),
|
||||
shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
@@ -1019,6 +1105,12 @@ impl RunningClientHandler {
|
||||
#[cfg(unix)]
|
||||
let raw_fd = self.raw_fd;
|
||||
let rst_on_close = self.rst_on_close;
|
||||
// MSS for the bulk data phase: once the handshake (incl. ServerHello) is
|
||||
// sent, restore a normal MSS so only the handshake stays fragmented by the
|
||||
// low listener `client_mss`. Cuts pps ~10x (anti-DDoS abuse on pps-policing
|
||||
// hosts like FastVPS). None = keep handshake MSS for the whole connection.
|
||||
#[cfg(unix)]
|
||||
let bulk_mss: Option<u16> = self.config.server.client_mss_bulk_value().ok().flatten();
|
||||
|
||||
let outcome = match self.do_handshake().await? {
|
||||
Some(outcome) => outcome,
|
||||
@@ -1032,6 +1124,14 @@ impl RunningClientHandler {
|
||||
if matches!(rst_on_close, crate::config::RstOnCloseMode::Errors) {
|
||||
let _ = crate::transport::socket::clear_linger_fd(raw_fd);
|
||||
}
|
||||
// Handshake (ServerHello) done — raise MSS for bulk transfer.
|
||||
#[cfg(unix)]
|
||||
if let Some(mss) = bulk_mss {
|
||||
if let Err(e) = crate::transport::socket::set_tcp_mss_fd(raw_fd, u32::from(mss))
|
||||
{
|
||||
debug!(error = %e, "Failed to raise bulk MSS; keeping handshake MSS");
|
||||
}
|
||||
}
|
||||
fut.await
|
||||
}
|
||||
HandshakeOutcome::NeedsMasking(fut) => fut.await,
|
||||
@@ -1252,6 +1352,7 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -1273,6 +1374,7 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
};
|
||||
@@ -1292,9 +1394,17 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
let tls_fingerprint = observe_tls_client_fingerprint(
|
||||
self.stats.as_ref(),
|
||||
&self.config,
|
||||
peer.ip(),
|
||||
&handshake,
|
||||
);
|
||||
|
||||
let config = self.config.clone();
|
||||
let replay_checker = self.replay_checker.clone();
|
||||
let stats = self.stats.clone();
|
||||
@@ -1318,6 +1428,12 @@ impl RunningClientHandler {
|
||||
HandshakeResult::Success(result) => result,
|
||||
HandshakeResult::BadClient { reader, writer } => {
|
||||
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
return Ok(masking_outcome(
|
||||
reader,
|
||||
writer,
|
||||
@@ -1326,13 +1442,27 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => {
|
||||
record_tls_fingerprint_bad_or_probe(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
);
|
||||
increment_bad_on_unknown_tls_sni(stats.as_ref(), &e);
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
record_tls_fingerprint_auth_success(
|
||||
stats.as_ref(),
|
||||
&config,
|
||||
peer.ip(),
|
||||
tls_fingerprint.as_ref(),
|
||||
tls_user.as_str(),
|
||||
);
|
||||
|
||||
debug!(peer = %peer, "Reading MTProto handshake through TLS");
|
||||
let mtproto_data = tls_reader.read_exact(HANDSHAKE_LEN).await?;
|
||||
@@ -1380,6 +1510,7 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
@@ -1427,6 +1558,7 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
self.config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -1465,6 +1597,7 @@ impl RunningClientHandler {
|
||||
local_addr,
|
||||
config.clone(),
|
||||
self.beobachten.clone(),
|
||||
self.shared.clone(),
|
||||
));
|
||||
}
|
||||
HandshakeResult::Error(e) => return Err(e),
|
||||
@@ -1558,6 +1691,11 @@ impl RunningClientHandler {
|
||||
{
|
||||
let user = success.user.clone();
|
||||
|
||||
if !shared.is_user_enabled(&user) {
|
||||
warn!(user = %user, "Disabled user rejected");
|
||||
return Err(ProxyError::UserDisabled { user });
|
||||
}
|
||||
|
||||
let user_limit_reservation = match Self::acquire_user_connection_reservation_static(
|
||||
&user,
|
||||
&config,
|
||||
@@ -1576,6 +1714,8 @@ impl RunningClientHandler {
|
||||
|
||||
let route_snapshot = route_runtime.snapshot();
|
||||
let session_id = rng.u64();
|
||||
let _user_session = shared.register_user_session(&user, session_id);
|
||||
let session_cancel = _user_session.token();
|
||||
let selected_me_pool = if config.general.use_middle_proxy
|
||||
&& matches!(route_snapshot.mode, RelayRouteMode::Middle)
|
||||
{
|
||||
@@ -1607,6 +1747,7 @@ impl RunningClientHandler {
|
||||
route_runtime.subscribe(),
|
||||
route_snapshot,
|
||||
session_id,
|
||||
session_cancel.clone(),
|
||||
shared.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1625,6 +1766,7 @@ impl RunningClientHandler {
|
||||
route_snapshot,
|
||||
session_id,
|
||||
local_addr,
|
||||
session_cancel.clone(),
|
||||
shared.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1644,6 +1786,7 @@ impl RunningClientHandler {
|
||||
route_snapshot,
|
||||
session_id,
|
||||
local_addr,
|
||||
session_cancel,
|
||||
shared.clone(),
|
||||
)
|
||||
.await
|
||||
|
||||
568
src/proxy/direct_buffer_budget.rs
Normal file
568
src/proxy/direct_buffer_budget.rs
Normal file
@@ -0,0 +1,568 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::stats::Stats;
|
||||
use crate::stream::BufferPool;
|
||||
|
||||
use super::shared_state::ProxySharedState;
|
||||
|
||||
/// Accounting granularity for process-wide Direct copy-buffer reservations.
|
||||
pub(crate) const DIRECT_BUFFER_UNIT_BYTES: usize = 4 * 1024;
|
||||
/// Minimum client-to-DC copy-buffer capacity for one Direct session.
|
||||
pub(crate) const DIRECT_BASE_C2S_BYTES: usize = 4 * 1024;
|
||||
/// Minimum DC-to-client copy-buffer capacity for one Direct session.
|
||||
pub(crate) const DIRECT_BASE_S2C_BYTES: usize = 8 * 1024;
|
||||
|
||||
const AUTO_HARD_MIN_BYTES: usize = 64 * 1024 * 1024;
|
||||
const AUTO_HARD_MAX_BYTES: usize = 2 * 1024 * 1024 * 1024;
|
||||
const AUTO_HARD_FALLBACK_BYTES: usize = 512 * 1024 * 1024;
|
||||
const TARGET_FLOOR_MIN_BYTES: usize = 16 * 1024 * 1024;
|
||||
const CONTROL_INTERVAL: Duration = Duration::from_secs(1);
|
||||
const HEALTHY_RECOVERY_SAMPLES: u8 = 30;
|
||||
const BUFFER_POOL_TRIM_LOW_WATERMARK: usize = 64;
|
||||
const BUFFER_POOL_TRIM_HIGH_WATERMARK: usize = 128;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
/// Lock-free observability snapshot of the Direct copy-buffer envelope.
|
||||
pub(crate) struct DirectBufferBudgetSnapshot {
|
||||
/// Absolute process-wide copy-buffer ceiling.
|
||||
pub(crate) hard_limit_bytes: u64,
|
||||
/// Current pressure-adjusted promotion target.
|
||||
pub(crate) target_bytes: u64,
|
||||
/// Bytes currently covered by active session leases.
|
||||
pub(crate) reserved_bytes: u64,
|
||||
/// Effective host or cgroup memory limit.
|
||||
pub(crate) memory_total_bytes: u64,
|
||||
/// Effective host or cgroup memory headroom.
|
||||
pub(crate) memory_available_bytes: u64,
|
||||
/// Current process resident set size.
|
||||
pub(crate) process_rss_bytes: u64,
|
||||
/// Successful tier growth reservations.
|
||||
pub(crate) promotion_total: u64,
|
||||
/// Tier growth attempts rejected by the adaptive target.
|
||||
pub(crate) promotion_denied_total: u64,
|
||||
/// Sessions admitted at minimum size above the adaptive target.
|
||||
pub(crate) minimum_fallback_total: u64,
|
||||
/// Sessions rejected by the absolute ceiling.
|
||||
pub(crate) admission_rejected_total: u64,
|
||||
/// Quiet-period tier reductions.
|
||||
pub(crate) quiet_demotion_total: u64,
|
||||
/// Sustained write-pressure tier reductions.
|
||||
pub(crate) write_pressure_demotion_total: u64,
|
||||
/// Process-wide pressure tier reductions.
|
||||
pub(crate) global_pressure_demotion_total: u64,
|
||||
/// Current sessions for Base through Tier3.
|
||||
pub(crate) tier_sessions: [u64; 4],
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
struct SystemMemorySample {
|
||||
total_bytes: u64,
|
||||
available_bytes: u64,
|
||||
process_rss_bytes: u64,
|
||||
}
|
||||
|
||||
/// Process-wide hard envelope and adaptive target for Direct copy buffers.
|
||||
pub(crate) struct DirectBufferBudget {
|
||||
hard_limit_bytes: u64,
|
||||
target_bytes: AtomicU64,
|
||||
reserved_bytes: AtomicU64,
|
||||
pressure_generation: AtomicU64,
|
||||
pressure_tx: watch::Sender<u64>,
|
||||
memory_total_bytes: AtomicU64,
|
||||
memory_available_bytes: AtomicU64,
|
||||
process_rss_bytes: AtomicU64,
|
||||
promotion_total: AtomicU64,
|
||||
promotion_denied_total: AtomicU64,
|
||||
minimum_fallback_total: AtomicU64,
|
||||
admission_rejected_total: AtomicU64,
|
||||
quiet_demotion_total: AtomicU64,
|
||||
write_pressure_demotion_total: AtomicU64,
|
||||
global_pressure_demotion_total: AtomicU64,
|
||||
tier_sessions: [AtomicU64; 4],
|
||||
}
|
||||
|
||||
impl DirectBufferBudget {
|
||||
/// Creates an envelope with a fixed absolute ceiling.
|
||||
pub(crate) fn new(hard_limit_bytes: usize) -> Arc<Self> {
|
||||
let hard_limit_bytes = align_down(hard_limit_bytes.max(DIRECT_BUFFER_UNIT_BYTES)) as u64;
|
||||
let (pressure_tx, _) = watch::channel(0);
|
||||
Arc::new(Self {
|
||||
hard_limit_bytes,
|
||||
target_bytes: AtomicU64::new(hard_limit_bytes),
|
||||
reserved_bytes: AtomicU64::new(0),
|
||||
pressure_generation: AtomicU64::new(0),
|
||||
pressure_tx,
|
||||
memory_total_bytes: AtomicU64::new(0),
|
||||
memory_available_bytes: AtomicU64::new(0),
|
||||
process_rss_bytes: AtomicU64::new(0),
|
||||
promotion_total: AtomicU64::new(0),
|
||||
promotion_denied_total: AtomicU64::new(0),
|
||||
minimum_fallback_total: AtomicU64::new(0),
|
||||
admission_rejected_total: AtomicU64::new(0),
|
||||
quiet_demotion_total: AtomicU64::new(0),
|
||||
write_pressure_demotion_total: AtomicU64::new(0),
|
||||
global_pressure_demotion_total: AtomicU64::new(0),
|
||||
tier_sessions: std::array::from_fn(|_| AtomicU64::new(0)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the current pressure-adjusted reservation target.
|
||||
pub(crate) fn target_bytes(&self) -> usize {
|
||||
self.target_bytes.load(Ordering::Relaxed) as usize
|
||||
}
|
||||
|
||||
/// Subscribes to target reductions that require prompt session demotion.
|
||||
pub(crate) fn subscribe_pressure(&self) -> watch::Receiver<u64> {
|
||||
self.pressure_tx.subscribe()
|
||||
}
|
||||
|
||||
/// Reserves bytes against either the adaptive target or the absolute ceiling.
|
||||
pub(crate) fn try_reserve(
|
||||
self: &Arc<Self>,
|
||||
bytes: usize,
|
||||
allow_above_target: bool,
|
||||
) -> Option<DirectBufferLease> {
|
||||
let bytes = align_up(bytes) as u64;
|
||||
let limit = if allow_above_target {
|
||||
self.hard_limit_bytes
|
||||
} else {
|
||||
self.target_bytes
|
||||
.load(Ordering::Relaxed)
|
||||
.min(self.hard_limit_bytes)
|
||||
};
|
||||
if !self.try_add_reserved(bytes, limit) {
|
||||
return None;
|
||||
}
|
||||
self.tier_sessions[0].fetch_add(1, Ordering::Relaxed);
|
||||
Some(DirectBufferLease {
|
||||
budget: Arc::clone(self),
|
||||
reserved_bytes: bytes,
|
||||
tier: 0,
|
||||
})
|
||||
}
|
||||
|
||||
fn try_add_reserved(&self, bytes: u64, limit: u64) -> bool {
|
||||
let mut current = self.reserved_bytes.load(Ordering::Acquire);
|
||||
loop {
|
||||
if bytes > limit.saturating_sub(current) {
|
||||
return false;
|
||||
}
|
||||
match self.reserved_bytes.compare_exchange_weak(
|
||||
current,
|
||||
current + bytes,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
) {
|
||||
Ok(_) => return true,
|
||||
Err(observed) => current = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn target_floor_bytes(&self) -> u64 {
|
||||
(self.hard_limit_bytes / 8)
|
||||
.max(TARGET_FLOOR_MIN_BYTES as u64)
|
||||
.min(self.hard_limit_bytes)
|
||||
}
|
||||
|
||||
fn set_target_bytes(&self, target: u64) {
|
||||
let target =
|
||||
align_down(target.clamp(self.target_floor_bytes(), self.hard_limit_bytes) as usize)
|
||||
as u64;
|
||||
let previous = self.target_bytes.swap(target, Ordering::AcqRel);
|
||||
if target < previous {
|
||||
let generation = self
|
||||
.pressure_generation
|
||||
.fetch_add(1, Ordering::AcqRel)
|
||||
.wrapping_add(1);
|
||||
self.pressure_tx.send_replace(generation);
|
||||
}
|
||||
}
|
||||
|
||||
fn update_system_sample(&self, sample: SystemMemorySample) {
|
||||
self.memory_total_bytes
|
||||
.store(sample.total_bytes, Ordering::Relaxed);
|
||||
self.memory_available_bytes
|
||||
.store(sample.available_bytes, Ordering::Relaxed);
|
||||
self.process_rss_bytes
|
||||
.store(sample.process_rss_bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records a session that had to bypass the adaptive target at minimum size.
|
||||
pub(crate) fn increment_minimum_fallback(&self) {
|
||||
self.minimum_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records a session rejected because the absolute ceiling was exhausted.
|
||||
pub(crate) fn increment_admission_rejected(&self) {
|
||||
self.admission_rejected_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records a tier reduction after sustained low throughput.
|
||||
pub(crate) fn increment_quiet_demotion(&self) {
|
||||
self.quiet_demotion_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records a tier reduction after sustained partial or pending writes.
|
||||
pub(crate) fn increment_write_pressure_demotion(&self) {
|
||||
self.write_pressure_demotion_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records a tier reduction requested by the process-wide controller.
|
||||
pub(crate) fn increment_global_pressure_demotion(&self) {
|
||||
self.global_pressure_demotion_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Captures all bounded metrics without allocating or locking.
|
||||
pub(crate) fn snapshot(&self) -> DirectBufferBudgetSnapshot {
|
||||
DirectBufferBudgetSnapshot {
|
||||
hard_limit_bytes: self.hard_limit_bytes,
|
||||
target_bytes: self.target_bytes.load(Ordering::Relaxed),
|
||||
reserved_bytes: self.reserved_bytes.load(Ordering::Relaxed),
|
||||
memory_total_bytes: self.memory_total_bytes.load(Ordering::Relaxed),
|
||||
memory_available_bytes: self.memory_available_bytes.load(Ordering::Relaxed),
|
||||
process_rss_bytes: self.process_rss_bytes.load(Ordering::Relaxed),
|
||||
promotion_total: self.promotion_total.load(Ordering::Relaxed),
|
||||
promotion_denied_total: self.promotion_denied_total.load(Ordering::Relaxed),
|
||||
minimum_fallback_total: self.minimum_fallback_total.load(Ordering::Relaxed),
|
||||
admission_rejected_total: self.admission_rejected_total.load(Ordering::Relaxed),
|
||||
quiet_demotion_total: self.quiet_demotion_total.load(Ordering::Relaxed),
|
||||
write_pressure_demotion_total: self
|
||||
.write_pressure_demotion_total
|
||||
.load(Ordering::Relaxed),
|
||||
global_pressure_demotion_total: self
|
||||
.global_pressure_demotion_total
|
||||
.load(Ordering::Relaxed),
|
||||
tier_sessions: std::array::from_fn(|index| {
|
||||
self.tier_sessions[index].load(Ordering::Relaxed)
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the conservative ceiling used when memory discovery is unavailable.
|
||||
pub(crate) fn fallback_direct_buffer_hard_limit() -> usize {
|
||||
AUTO_HARD_FALLBACK_BYTES
|
||||
}
|
||||
|
||||
/// RAII ownership of all copy-buffer bytes retained by one Direct session.
|
||||
pub(crate) struct DirectBufferLease {
|
||||
budget: Arc<DirectBufferBudget>,
|
||||
reserved_bytes: u64,
|
||||
tier: usize,
|
||||
}
|
||||
|
||||
impl DirectBufferLease {
|
||||
/// Returns the currently covered allocation rounded to accounting units.
|
||||
pub(crate) fn reserved_bytes(&self) -> usize {
|
||||
self.reserved_bytes as usize
|
||||
}
|
||||
|
||||
/// Attempts to cover a larger tier before its buffers are resized.
|
||||
pub(crate) fn try_grow_to(&mut self, bytes: usize) -> bool {
|
||||
let bytes = align_up(bytes) as u64;
|
||||
if bytes <= self.reserved_bytes {
|
||||
return true;
|
||||
}
|
||||
let delta = bytes - self.reserved_bytes;
|
||||
let limit = self
|
||||
.budget
|
||||
.target_bytes
|
||||
.load(Ordering::Relaxed)
|
||||
.min(self.budget.hard_limit_bytes);
|
||||
if !self.budget.try_add_reserved(delta, limit) {
|
||||
self.budget
|
||||
.promotion_denied_total
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
}
|
||||
self.reserved_bytes = bytes;
|
||||
self.budget.promotion_total.fetch_add(1, Ordering::Relaxed);
|
||||
true
|
||||
}
|
||||
|
||||
/// Releases bytes only after both directional buffers report smaller coverage.
|
||||
pub(crate) fn shrink_to(&mut self, bytes: usize) {
|
||||
let bytes = align_up(bytes) as u64;
|
||||
if bytes >= self.reserved_bytes {
|
||||
return;
|
||||
}
|
||||
let released = self.reserved_bytes - bytes;
|
||||
self.reserved_bytes = bytes;
|
||||
self.budget
|
||||
.reserved_bytes
|
||||
.fetch_sub(released, Ordering::AcqRel);
|
||||
}
|
||||
|
||||
/// Updates bounded per-tier session gauges for an accepted transition.
|
||||
pub(crate) fn set_tier(&mut self, tier: usize) {
|
||||
let tier = tier.min(self.budget.tier_sessions.len() - 1);
|
||||
if tier == self.tier {
|
||||
return;
|
||||
}
|
||||
decrement_saturating(&self.budget.tier_sessions[self.tier]);
|
||||
self.budget.tier_sessions[tier].fetch_add(1, Ordering::Relaxed);
|
||||
self.tier = tier;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for DirectBufferLease {
|
||||
fn drop(&mut self) {
|
||||
self.budget
|
||||
.reserved_bytes
|
||||
.fetch_sub(self.reserved_bytes, Ordering::AcqRel);
|
||||
decrement_saturating(&self.budget.tier_sessions[self.tier]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the startup hard ceiling from config, cgroup, and host memory.
|
||||
pub(crate) async fn resolve_direct_buffer_hard_limit(configured: usize) -> usize {
|
||||
if configured != 0 {
|
||||
return align_down(configured);
|
||||
}
|
||||
let sample = read_system_memory_sample().await;
|
||||
if sample.total_bytes == 0 {
|
||||
return AUTO_HARD_FALLBACK_BYTES;
|
||||
}
|
||||
let derived = (sample.total_bytes / 4)
|
||||
.clamp(AUTO_HARD_MIN_BYTES as u64, AUTO_HARD_MAX_BYTES as u64)
|
||||
.min(sample.total_bytes);
|
||||
align_down(derived as usize).max(DIRECT_BUFFER_UNIT_BYTES)
|
||||
}
|
||||
|
||||
/// Runs the control-plane loop for Direct budget and shared pool pressure.
|
||||
pub(crate) async fn run_direct_buffer_budget_controller(
|
||||
budget: Arc<DirectBufferBudget>,
|
||||
buffer_pool: Arc<BufferPool>,
|
||||
stats: Arc<Stats>,
|
||||
shared: Arc<ProxySharedState>,
|
||||
max_connections: u32,
|
||||
) {
|
||||
let mut interval = tokio::time::interval(CONTROL_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut healthy_streak = 0u8;
|
||||
let mut previous_denied = 0u64;
|
||||
let mut previous_fallback = 0u64;
|
||||
let mut previous_rejected = 0u64;
|
||||
let pool_trim_low = buffer_pool
|
||||
.max_buffers()
|
||||
.min(BUFFER_POOL_TRIM_LOW_WATERMARK);
|
||||
let pool_trim_high = buffer_pool
|
||||
.max_buffers()
|
||||
.min(BUFFER_POOL_TRIM_HIGH_WATERMARK);
|
||||
let mut pool_trim_armed = true;
|
||||
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let sample = read_system_memory_sample().await;
|
||||
budget.update_system_sample(sample);
|
||||
|
||||
let snapshot = budget.snapshot();
|
||||
let denied_delta = snapshot
|
||||
.promotion_denied_total
|
||||
.saturating_sub(previous_denied);
|
||||
previous_denied = snapshot.promotion_denied_total;
|
||||
let fallback_delta = snapshot
|
||||
.minimum_fallback_total
|
||||
.saturating_sub(previous_fallback);
|
||||
previous_fallback = snapshot.minimum_fallback_total;
|
||||
let rejected_delta = snapshot
|
||||
.admission_rejected_total
|
||||
.saturating_sub(previous_rejected);
|
||||
previous_rejected = snapshot.admission_rejected_total;
|
||||
|
||||
let connection_pct = connection_fill_pct(stats.as_ref(), max_connections);
|
||||
let memory_available_pct = percentage(sample.available_bytes, sample.total_bytes);
|
||||
let target_utilization_pct = percentage(snapshot.reserved_bytes, snapshot.target_bytes);
|
||||
let pressure = shared.conntrack_pressure_active()
|
||||
|| connection_pct.is_some_and(|value| value >= 85)
|
||||
|| memory_available_pct.is_some_and(|value| value <= 15)
|
||||
|| target_utilization_pct.is_some_and(|value| value >= 90)
|
||||
|| denied_delta > 0
|
||||
|| fallback_delta > 0
|
||||
|| rejected_delta > 0;
|
||||
|
||||
if !pressure {
|
||||
pool_trim_armed = true;
|
||||
} else if pool_trim_armed && buffer_pool.pooled() > pool_trim_high {
|
||||
buffer_pool.trim_to(pool_trim_low);
|
||||
pool_trim_armed = false;
|
||||
}
|
||||
|
||||
let pool_snapshot = buffer_pool.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
pool_snapshot.allocated,
|
||||
pool_snapshot.allocated.saturating_sub(pool_snapshot.pooled),
|
||||
);
|
||||
stats.set_buffer_pool_replaced_nonstandard_total(pool_snapshot.replaced_nonstandard);
|
||||
|
||||
let headroom_target = if sample.total_bytes == 0 {
|
||||
snapshot.hard_limit_bytes
|
||||
} else {
|
||||
snapshot
|
||||
.reserved_bytes
|
||||
.saturating_add(sample.available_bytes / 4)
|
||||
.min(snapshot.hard_limit_bytes)
|
||||
};
|
||||
|
||||
if pressure {
|
||||
healthy_streak = 0;
|
||||
let reduced = snapshot.target_bytes.saturating_mul(3) / 4;
|
||||
budget.set_target_bytes(reduced.min(headroom_target));
|
||||
continue;
|
||||
}
|
||||
|
||||
let healthy = memory_available_pct.is_none_or(|value| value >= 30)
|
||||
&& connection_pct.is_none_or(|value| value <= 70);
|
||||
if !healthy {
|
||||
healthy_streak = 0;
|
||||
if headroom_target < snapshot.target_bytes {
|
||||
budget.set_target_bytes(headroom_target);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
healthy_streak = healthy_streak.saturating_add(1);
|
||||
if healthy_streak >= HEALTHY_RECOVERY_SAMPLES {
|
||||
healthy_streak = 0;
|
||||
let increment = (snapshot.target_bytes / 16).max(4 * 1024 * 1024);
|
||||
budget.set_target_bytes(
|
||||
snapshot
|
||||
.target_bytes
|
||||
.saturating_add(increment)
|
||||
.min(headroom_target),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn connection_fill_pct(stats: &Stats, max_connections: u32) -> Option<u8> {
|
||||
if max_connections == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(
|
||||
((stats.get_current_connections_total().saturating_mul(100)) / u64::from(max_connections))
|
||||
.min(100) as u8,
|
||||
)
|
||||
}
|
||||
|
||||
fn percentage(value: u64, total: u64) -> Option<u8> {
|
||||
if total == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(((value.saturating_mul(100)) / total).min(100) as u8)
|
||||
}
|
||||
|
||||
async fn read_system_memory_sample() -> SystemMemorySample {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let meminfo = tokio::fs::read_to_string("/proc/meminfo")
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let status = tokio::fs::read_to_string("/proc/self/status")
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let host_total = parse_kib_field(&meminfo, "MemTotal:");
|
||||
let host_available = parse_kib_field(&meminfo, "MemAvailable:");
|
||||
let process_rss = parse_kib_field(&status, "VmRSS:");
|
||||
|
||||
let cgroup_v2_max = read_cgroup_limit("/sys/fs/cgroup/memory.max").await;
|
||||
let cgroup_v2_current = read_u64_file("/sys/fs/cgroup/memory.current").await;
|
||||
let cgroup_v1_max = read_cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes").await;
|
||||
let cgroup_v1_current = read_u64_file("/sys/fs/cgroup/memory/memory.usage_in_bytes").await;
|
||||
let cgroup_max = cgroup_v2_max.or(cgroup_v1_max);
|
||||
let cgroup_current = cgroup_v2_current.or(cgroup_v1_current);
|
||||
|
||||
let total = match (host_total, cgroup_max) {
|
||||
(0, Some(limit)) => limit,
|
||||
(host, Some(limit)) => host.min(limit),
|
||||
(host, None) => host,
|
||||
};
|
||||
let cgroup_available = cgroup_max
|
||||
.zip(cgroup_current)
|
||||
.map(|(limit, current)| limit.saturating_sub(current));
|
||||
let available = match (host_available, cgroup_available) {
|
||||
(0, Some(value)) => value,
|
||||
(host, Some(value)) => host.min(value),
|
||||
(host, None) => host,
|
||||
};
|
||||
return SystemMemorySample {
|
||||
total_bytes: total,
|
||||
available_bytes: available,
|
||||
process_rss_bytes: process_rss,
|
||||
};
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
SystemMemorySample::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn read_cgroup_limit(path: &str) -> Option<u64> {
|
||||
let raw = tokio::fs::read_to_string(path).await.ok()?;
|
||||
let raw = raw.trim();
|
||||
if raw == "max" {
|
||||
return None;
|
||||
}
|
||||
let value = raw.parse::<u64>().ok()?;
|
||||
(value < (1u64 << 60)).then_some(value)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn read_u64_file(path: &str) -> Option<u64> {
|
||||
tokio::fs::read_to_string(path)
|
||||
.await
|
||||
.ok()?
|
||||
.trim()
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn parse_kib_field(raw: &str, key: &str) -> u64 {
|
||||
raw.lines()
|
||||
.find_map(|line| {
|
||||
let value = line.strip_prefix(key)?.split_whitespace().next()?;
|
||||
value.parse::<u64>().ok()
|
||||
})
|
||||
.unwrap_or(0)
|
||||
.saturating_mul(1024)
|
||||
}
|
||||
|
||||
fn align_up(bytes: usize) -> usize {
|
||||
bytes
|
||||
.div_ceil(DIRECT_BUFFER_UNIT_BYTES)
|
||||
.saturating_mul(DIRECT_BUFFER_UNIT_BYTES)
|
||||
}
|
||||
|
||||
fn align_down(bytes: usize) -> usize {
|
||||
bytes / DIRECT_BUFFER_UNIT_BYTES * DIRECT_BUFFER_UNIT_BYTES
|
||||
}
|
||||
|
||||
fn decrement_saturating(value: &AtomicU64) {
|
||||
let mut current = value.load(Ordering::Relaxed);
|
||||
while current != 0 {
|
||||
match value.compare_exchange_weak(
|
||||
current,
|
||||
current - 1,
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => return,
|
||||
Err(observed) => current = observed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/direct_buffer_budget_tests.rs"]
|
||||
mod tests;
|
||||
@@ -10,6 +10,7 @@ use std::time::Duration;
|
||||
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadHalf, WriteHalf, split};
|
||||
use tokio::sync::watch;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::config::ProxyConfig;
|
||||
@@ -258,6 +259,7 @@ where
|
||||
route_snapshot,
|
||||
session_id,
|
||||
SocketAddr::from(([0, 0, 0, 0], config.server.port)),
|
||||
CancellationToken::new(),
|
||||
ProxySharedState::new(),
|
||||
)
|
||||
.await
|
||||
@@ -276,6 +278,7 @@ pub(crate) async fn handle_via_direct_with_shared<R, W>(
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
local_addr: SocketAddr,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
@@ -302,14 +305,25 @@ where
|
||||
"Ignoring invalid scope hint and falling back to default upstream selection"
|
||||
);
|
||||
}
|
||||
let tg_stream = upstream_manager
|
||||
.connect(dc_addr, Some(success.dc_idx), scope_hint)
|
||||
.await?;
|
||||
let tg_stream = tokio::select! {
|
||||
result = upstream_manager.connect(dc_addr, Some(success.dc_idx), scope_hint) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, dc_addr = %dc_addr, "Connected, performing TG handshake");
|
||||
|
||||
let (tg_reader, tg_writer) =
|
||||
do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()).await?;
|
||||
let (tg_reader, tg_writer) = tokio::select! {
|
||||
result = do_tg_handshake_static(tg_stream, &success, &config, rng.as_ref()) => result?,
|
||||
_ = session_cancel.cancelled() => {
|
||||
return Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
debug!(peer = %success.peer, "TG handshake complete, starting relay");
|
||||
|
||||
@@ -331,19 +345,21 @@ where
|
||||
} else {
|
||||
Duration::from_secs(1800)
|
||||
};
|
||||
let relay_result = crate::proxy::relay::relay_bidirectional_with_activity_timeout_and_lease(
|
||||
let relay_result = crate::proxy::relay::relay_direct_adaptive(
|
||||
client_reader,
|
||||
client_writer,
|
||||
tg_reader,
|
||||
tg_writer,
|
||||
config.general.direct_relay_copy_buf_c2s_bytes,
|
||||
config.general.direct_relay_copy_buf_s2c_bytes,
|
||||
config.server.max_connections,
|
||||
user,
|
||||
Arc::clone(&stats),
|
||||
config.access.user_data_quota.get(user).copied(),
|
||||
buffer_pool,
|
||||
traffic_lease,
|
||||
relay_activity_timeout,
|
||||
session_cancel.clone(),
|
||||
Arc::clone(&shared.direct_buffer_budget),
|
||||
);
|
||||
tokio::pin!(relay_result);
|
||||
let relay_result = loop {
|
||||
@@ -371,6 +387,11 @@ where
|
||||
break relay_result.await;
|
||||
}
|
||||
}
|
||||
_ = session_cancel.cancelled() => {
|
||||
break Err(ProxyError::UserDisabled {
|
||||
user: user.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -379,7 +400,6 @@ where
|
||||
Err(e) => debug!(user = %user, error = %e, "Direct relay ended with error"),
|
||||
}
|
||||
|
||||
buffer_pool_trim.trim_to(buffer_pool_trim.max_buffers().min(64));
|
||||
let pool_snapshot = buffer_pool_trim.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
use dashmap::DashMap;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
use hmac::{Hmac, Mac};
|
||||
#[cfg(test)]
|
||||
use std::collections::HashSet;
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
@@ -33,8 +32,10 @@ use crate::stream::{CryptoReader, CryptoWriter, FakeTlsReader, FakeTlsWriter};
|
||||
use crate::tls_front::{TlsFrontCache, emulator};
|
||||
#[cfg(test)]
|
||||
use rand::RngExt;
|
||||
use sha2::Sha256;
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
mod tls_auth;
|
||||
|
||||
use self::tls_auth::{parse_tls_auth_material, validate_tls_secret_candidate};
|
||||
|
||||
const ACCESS_SECRET_BYTES: usize = 16;
|
||||
const UNKNOWN_SNI_WARN_COOLDOWN_SECS: u64 = 5;
|
||||
@@ -58,8 +59,6 @@ const OVERLOAD_CANDIDATE_BUDGET_UNHINTED: usize = 8;
|
||||
const EXPENSIVE_INVALID_SCAN_SATURATION_THRESHOLD: usize = 64;
|
||||
const RECENT_USER_RING_SCAN_LIMIT: usize = 32;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
#[cfg(test)]
|
||||
const AUTH_PROBE_BACKOFF_BASE_MS: u64 = 1;
|
||||
#[cfg(not(test))]
|
||||
@@ -104,23 +103,6 @@ fn should_emit_unknown_sni_warn_in(shared: &ProxySharedState, now: Instant) -> b
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ParsedTlsAuthMaterial {
|
||||
digest: [u8; tls::TLS_DIGEST_LEN],
|
||||
session_id: [u8; 32],
|
||||
session_id_len: usize,
|
||||
now: i64,
|
||||
ignore_time_skew: bool,
|
||||
boot_time_cap_secs: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TlsCandidateValidation {
|
||||
digest: [u8; tls::TLS_DIGEST_LEN],
|
||||
session_id: [u8; 32],
|
||||
session_id_len: usize,
|
||||
}
|
||||
|
||||
struct MtprotoCandidateValidation {
|
||||
proto_tag: ProtoTag,
|
||||
dc_idx: i16,
|
||||
@@ -251,104 +233,6 @@ fn budget_for_validation(total_users: usize, overload: bool, has_hint: bool) ->
|
||||
total_users.min(cap.max(1))
|
||||
}
|
||||
|
||||
fn parse_tls_auth_material(
|
||||
handshake: &[u8],
|
||||
ignore_time_skew: bool,
|
||||
replay_window_secs: u64,
|
||||
) -> Option<ParsedTlsAuthMaterial> {
|
||||
if handshake.len() < tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN + 1 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let digest: [u8; tls::TLS_DIGEST_LEN] = handshake
|
||||
[tls::TLS_DIGEST_POS..tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN]
|
||||
.try_into()
|
||||
.ok()?;
|
||||
|
||||
let session_id_len_pos = tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN;
|
||||
let session_id_len = usize::from(handshake.get(session_id_len_pos).copied()?);
|
||||
if session_id_len > 32 {
|
||||
return None;
|
||||
}
|
||||
let session_id_start = session_id_len_pos + 1;
|
||||
if handshake.len() < session_id_start + session_id_len {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut session_id = [0u8; 32];
|
||||
session_id[..session_id_len]
|
||||
.copy_from_slice(&handshake[session_id_start..session_id_start + session_id_len]);
|
||||
|
||||
let now = if !ignore_time_skew {
|
||||
let d = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?;
|
||||
i64::try_from(d.as_secs()).ok()?
|
||||
} else {
|
||||
0_i64
|
||||
};
|
||||
|
||||
let replay_window_u32 = u32::try_from(replay_window_secs).unwrap_or(u32::MAX);
|
||||
let boot_time_cap_secs = if ignore_time_skew {
|
||||
0
|
||||
} else {
|
||||
tls::BOOT_TIME_MAX_SECS
|
||||
.min(replay_window_u32)
|
||||
.min(tls::BOOT_TIME_COMPAT_MAX_SECS)
|
||||
};
|
||||
|
||||
Some(ParsedTlsAuthMaterial {
|
||||
digest,
|
||||
session_id,
|
||||
session_id_len,
|
||||
now,
|
||||
ignore_time_skew,
|
||||
boot_time_cap_secs,
|
||||
})
|
||||
}
|
||||
|
||||
fn compute_tls_hmac_zeroed_digest(secret: &[u8], handshake: &[u8]) -> [u8; 32] {
|
||||
let mut mac = HmacSha256::new_from_slice(secret).expect("HMAC accepts any key length");
|
||||
mac.update(&handshake[..tls::TLS_DIGEST_POS]);
|
||||
mac.update(&[0u8; tls::TLS_DIGEST_LEN]);
|
||||
mac.update(&handshake[tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN..]);
|
||||
mac.finalize().into_bytes().into()
|
||||
}
|
||||
|
||||
fn validate_tls_secret_candidate(
|
||||
parsed: &ParsedTlsAuthMaterial,
|
||||
handshake: &[u8],
|
||||
secret: &[u8],
|
||||
) -> Option<TlsCandidateValidation> {
|
||||
let computed = compute_tls_hmac_zeroed_digest(secret, handshake);
|
||||
if !bool::from(parsed.digest[..28].ct_eq(&computed[..28])) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let timestamp = u32::from_le_bytes([
|
||||
parsed.digest[28] ^ computed[28],
|
||||
parsed.digest[29] ^ computed[29],
|
||||
parsed.digest[30] ^ computed[30],
|
||||
parsed.digest[31] ^ computed[31],
|
||||
]);
|
||||
|
||||
if !parsed.ignore_time_skew {
|
||||
let is_boot_time = parsed.boot_time_cap_secs > 0 && timestamp < parsed.boot_time_cap_secs;
|
||||
if !is_boot_time {
|
||||
let time_diff = parsed.now - i64::from(timestamp);
|
||||
if !(tls::TIME_SKEW_MIN..=tls::TIME_SKEW_MAX).contains(&time_diff) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some(TlsCandidateValidation {
|
||||
digest: parsed.digest,
|
||||
session_id: parsed.session_id,
|
||||
session_id_len: parsed.session_id_len,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_mtproto_secret_candidate(
|
||||
handshake: &[u8; HANDSHAKE_LEN],
|
||||
dec_prekey: &[u8; PREKEY_LEN],
|
||||
@@ -1473,14 +1357,60 @@ where
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
}
|
||||
|
||||
let cached = if config.censorship.tls_emulation {
|
||||
let cached_entry = if config.censorship.tls_emulation {
|
||||
if let Some(cache) = tls_cache.as_ref() {
|
||||
let selected_domain =
|
||||
matched_tls_domain.unwrap_or(config.censorship.tls_domain.as_str());
|
||||
let cached_entry = cache.get(selected_domain).await;
|
||||
let use_full_cert_payload = if config.censorship.serverhello_compact
|
||||
&& matches!(client_tls_version, tls::ClientHelloTlsVersion::Tls12)
|
||||
{
|
||||
Some(cached_entry)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let preferred_key_share_group = cached_entry
|
||||
.as_ref()
|
||||
.and_then(|cached_entry| emulator::profiled_server_hello_key_share_group(cached_entry));
|
||||
let Some(server_key_share) =
|
||||
tls::build_server_hello_key_share(handshake, preferred_key_share_group, rng)
|
||||
else {
|
||||
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
|
||||
maybe_apply_server_hello_delay(config).await;
|
||||
debug!(
|
||||
peer = %peer,
|
||||
"TLS handshake rejected: ClientHello did not offer a usable TLS 1.3 key_share"
|
||||
);
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
};
|
||||
|
||||
let preferred_cipher_suite = if let Some(cached_entry) = cached_entry.as_ref() {
|
||||
if cached_entry.server_hello_template.cipher_suite == [0, 0] {
|
||||
[0x13, 0x01]
|
||||
} else {
|
||||
cached_entry.server_hello_template.cipher_suite
|
||||
}
|
||||
} else {
|
||||
[0x13, 0x01]
|
||||
};
|
||||
let Some(selected_cipher_suite) =
|
||||
tls::select_server_hello_cipher_suite(handshake, preferred_cipher_suite)
|
||||
else {
|
||||
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
|
||||
maybe_apply_server_hello_delay(config).await;
|
||||
debug!(
|
||||
peer = %peer,
|
||||
"TLS handshake rejected: ClientHello did not offer a supported TLS 1.3 cipher suite"
|
||||
);
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
};
|
||||
|
||||
let cached = if let Some(cached_entry) = cached_entry {
|
||||
let use_full_cert_payload = if config.censorship.serverhello_compact
|
||||
&& matches!(client_tls_version, tls::ClientHelloTlsVersion::Tls12)
|
||||
{
|
||||
if let Some(cache) = tls_cache.as_ref() {
|
||||
cache
|
||||
.take_full_cert_budget_for_ip(
|
||||
peer.ip(),
|
||||
@@ -1489,11 +1419,11 @@ where
|
||||
.await
|
||||
} else {
|
||||
true
|
||||
};
|
||||
Some((cached_entry, use_full_cert_payload))
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
true
|
||||
};
|
||||
Some((cached_entry, use_full_cert_payload))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -1504,13 +1434,6 @@ where
|
||||
let validation_session_id_slice = &validation_session_id[..validation_session_id_len];
|
||||
|
||||
let response = if let Some((cached_entry, use_full_cert_payload)) = cached {
|
||||
let preferred_cipher_suite = if cached_entry.server_hello_template.cipher_suite == [0, 0] {
|
||||
[0x13, 0x01]
|
||||
} else {
|
||||
cached_entry.server_hello_template.cipher_suite
|
||||
};
|
||||
let selected_cipher_suite =
|
||||
tls::select_server_hello_cipher_suite(handshake, preferred_cipher_suite);
|
||||
emulator::build_emulated_server_hello(
|
||||
&validated_secret,
|
||||
&validation_digest,
|
||||
@@ -1520,12 +1443,12 @@ where
|
||||
config.censorship.serverhello_compact,
|
||||
client_tls_version,
|
||||
selected_cipher_suite,
|
||||
&server_key_share,
|
||||
rng,
|
||||
selected_alpn.clone(),
|
||||
config.censorship.tls_new_session_tickets,
|
||||
)
|
||||
} else {
|
||||
let selected_cipher_suite = tls::select_server_hello_cipher_suite(handshake, [0x13, 0x01]);
|
||||
tls::build_server_hello_with_cipher(
|
||||
&validated_secret,
|
||||
&validation_digest,
|
||||
@@ -1533,6 +1456,7 @@ where
|
||||
config.censorship.fake_cert_len,
|
||||
rng,
|
||||
selected_cipher_suite,
|
||||
&server_key_share,
|
||||
selected_alpn.clone(),
|
||||
config.censorship.tls_new_session_tickets,
|
||||
)
|
||||
@@ -1817,7 +1741,16 @@ where
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
}
|
||||
|
||||
let validation = matched_validation.expect("validation must exist when matched");
|
||||
let Some(validation) = matched_validation else {
|
||||
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
|
||||
maybe_apply_server_hello_delay(config).await;
|
||||
warn!(
|
||||
peer = %peer,
|
||||
user = %matched_user,
|
||||
"MTProto handshake matched user without validation material"
|
||||
);
|
||||
return HandshakeResult::BadClient { reader, writer };
|
||||
};
|
||||
|
||||
if config
|
||||
.access
|
||||
|
||||
126
src/proxy/handshake/tls_auth.rs
Normal file
126
src/proxy/handshake/tls_auth.rs
Normal file
@@ -0,0 +1,126 @@
|
||||
use hmac::{Hmac, Mac};
|
||||
use sha2::Sha256;
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
use crate::protocol::tls;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
/// Parsed TLS authentication material extracted from a ClientHello candidate.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct ParsedTlsAuthMaterial {
|
||||
digest: [u8; tls::TLS_DIGEST_LEN],
|
||||
session_id: [u8; 32],
|
||||
session_id_len: usize,
|
||||
now: i64,
|
||||
ignore_time_skew: bool,
|
||||
boot_time_cap_secs: u32,
|
||||
}
|
||||
|
||||
/// Successful TLS secret validation output used by the handshake state machine.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) struct TlsCandidateValidation {
|
||||
pub(super) digest: [u8; tls::TLS_DIGEST_LEN],
|
||||
pub(super) session_id: [u8; 32],
|
||||
pub(super) session_id_len: usize,
|
||||
}
|
||||
|
||||
/// Parse TLS auth digest and session-id material from a candidate handshake.
|
||||
pub(super) fn parse_tls_auth_material(
|
||||
handshake: &[u8],
|
||||
ignore_time_skew: bool,
|
||||
replay_window_secs: u64,
|
||||
) -> Option<ParsedTlsAuthMaterial> {
|
||||
if handshake.len() < tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN + 1 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let digest: [u8; tls::TLS_DIGEST_LEN] = handshake
|
||||
[tls::TLS_DIGEST_POS..tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN]
|
||||
.try_into()
|
||||
.ok()?;
|
||||
|
||||
let session_id_len_pos = tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN;
|
||||
let session_id_len = usize::from(handshake.get(session_id_len_pos).copied()?);
|
||||
if session_id_len > 32 {
|
||||
return None;
|
||||
}
|
||||
let session_id_start = session_id_len_pos + 1;
|
||||
if handshake.len() < session_id_start + session_id_len {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut session_id = [0u8; 32];
|
||||
session_id[..session_id_len]
|
||||
.copy_from_slice(&handshake[session_id_start..session_id_start + session_id_len]);
|
||||
|
||||
let now = if !ignore_time_skew {
|
||||
let d = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?;
|
||||
i64::try_from(d.as_secs()).ok()?
|
||||
} else {
|
||||
0_i64
|
||||
};
|
||||
|
||||
let replay_window_u32 = u32::try_from(replay_window_secs).unwrap_or(u32::MAX);
|
||||
let boot_time_cap_secs = if ignore_time_skew {
|
||||
0
|
||||
} else {
|
||||
tls::BOOT_TIME_MAX_SECS
|
||||
.min(replay_window_u32)
|
||||
.min(tls::BOOT_TIME_COMPAT_MAX_SECS)
|
||||
};
|
||||
|
||||
Some(ParsedTlsAuthMaterial {
|
||||
digest,
|
||||
session_id,
|
||||
session_id_len,
|
||||
now,
|
||||
ignore_time_skew,
|
||||
boot_time_cap_secs,
|
||||
})
|
||||
}
|
||||
|
||||
fn compute_tls_hmac_zeroed_digest(secret: &[u8], handshake: &[u8]) -> Option<[u8; 32]> {
|
||||
let mut mac = HmacSha256::new_from_slice(secret).ok()?;
|
||||
mac.update(&handshake[..tls::TLS_DIGEST_POS]);
|
||||
mac.update(&[0u8; tls::TLS_DIGEST_LEN]);
|
||||
mac.update(&handshake[tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN..]);
|
||||
Some(mac.finalize().into_bytes().into())
|
||||
}
|
||||
|
||||
/// Validate a candidate secret against parsed TLS authentication material.
|
||||
pub(super) fn validate_tls_secret_candidate(
|
||||
parsed: &ParsedTlsAuthMaterial,
|
||||
handshake: &[u8],
|
||||
secret: &[u8],
|
||||
) -> Option<TlsCandidateValidation> {
|
||||
let computed = compute_tls_hmac_zeroed_digest(secret, handshake)?;
|
||||
if !bool::from(parsed.digest[..28].ct_eq(&computed[..28])) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let timestamp = u32::from_le_bytes([
|
||||
parsed.digest[28] ^ computed[28],
|
||||
parsed.digest[29] ^ computed[29],
|
||||
parsed.digest[30] ^ computed[30],
|
||||
parsed.digest[31] ^ computed[31],
|
||||
]);
|
||||
|
||||
if !parsed.ignore_time_skew {
|
||||
let is_boot_time = parsed.boot_time_cap_secs > 0 && timestamp < parsed.boot_time_cap_secs;
|
||||
if !is_boot_time {
|
||||
let time_diff = parsed.now - i64::from(timestamp);
|
||||
if !(tls::TIME_SKEW_MIN..=tls::TIME_SKEW_MAX).contains(&time_diff) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some(TlsCandidateValidation {
|
||||
digest: parsed.digest,
|
||||
session_id: parsed.session_id,
|
||||
session_id_len: parsed.session_id_len,
|
||||
})
|
||||
}
|
||||
@@ -3,12 +3,15 @@
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::network::dns_overrides::resolve_socket_addr;
|
||||
use crate::protocol::tls;
|
||||
use crate::proxy::shared_state::ProxySharedState;
|
||||
use crate::stats::beobachten::BeobachtenStore;
|
||||
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
|
||||
use crate::transport::socket::configure_tcp_socket;
|
||||
#[cfg(unix)]
|
||||
use nix::ifaddrs::getifaddrs;
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{Rng, RngExt, SeedableRng};
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::str;
|
||||
#[cfg(test)]
|
||||
@@ -17,9 +20,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant as StdInstant};
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
#[cfg(unix)]
|
||||
use tokio::net::UnixStream;
|
||||
use tokio::net::{TcpStream, lookup_host};
|
||||
#[cfg(unix)]
|
||||
use tokio::sync::Mutex as AsyncMutex;
|
||||
use tokio::time::{Instant, timeout};
|
||||
@@ -36,6 +39,8 @@ const MASK_RELAY_TIMEOUT: Duration = Duration::from_millis(200);
|
||||
#[cfg(test)]
|
||||
const MASK_RELAY_IDLE_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
const MASK_BUFFER_SIZE: usize = 8192;
|
||||
const MASK_BUFFER_GROW_AFTER_BYTES: usize = 256 * 1024;
|
||||
const MASK_BUFFER_MAX_SIZE: usize = 64 * 1024;
|
||||
#[cfg(unix)]
|
||||
#[cfg(not(test))]
|
||||
const LOCAL_INTERFACE_CACHE_TTL: Duration = Duration::from_secs(300);
|
||||
@@ -53,6 +58,27 @@ struct MaskTcpTarget<'a> {
|
||||
port: u16,
|
||||
}
|
||||
|
||||
fn mask_copy_read_len(total: usize, byte_cap: usize) -> usize {
|
||||
// Keep short scanner probes on the small baseline buffer and grow only
|
||||
// after the session has proven to be sustained masking relay traffic.
|
||||
let active_buffer_size = if total >= MASK_BUFFER_GROW_AFTER_BYTES {
|
||||
MASK_BUFFER_MAX_SIZE
|
||||
} else {
|
||||
MASK_BUFFER_SIZE
|
||||
};
|
||||
|
||||
if byte_cap == 0 {
|
||||
return active_buffer_size;
|
||||
}
|
||||
|
||||
let remaining_budget = byte_cap.saturating_sub(total);
|
||||
if remaining_budget == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
remaining_budget.min(active_buffer_size)
|
||||
}
|
||||
|
||||
async fn copy_with_idle_timeout<R, W>(
|
||||
reader: &mut R,
|
||||
writer: &mut W,
|
||||
@@ -64,21 +90,18 @@ where
|
||||
R: AsyncRead + Unpin,
|
||||
W: AsyncWrite + Unpin,
|
||||
{
|
||||
let mut buf = Box::new([0u8; MASK_BUFFER_SIZE]);
|
||||
let mut buf = vec![0u8; MASK_BUFFER_SIZE];
|
||||
let mut total = 0usize;
|
||||
let mut ended_by_eof = false;
|
||||
let unlimited = byte_cap == 0;
|
||||
|
||||
loop {
|
||||
let read_len = if unlimited {
|
||||
MASK_BUFFER_SIZE
|
||||
} else {
|
||||
let remaining_budget = byte_cap.saturating_sub(total);
|
||||
if remaining_budget == 0 {
|
||||
break;
|
||||
}
|
||||
remaining_budget.min(MASK_BUFFER_SIZE)
|
||||
};
|
||||
let read_len = mask_copy_read_len(total, byte_cap);
|
||||
if read_len == 0 {
|
||||
break;
|
||||
}
|
||||
if buf.len() < read_len {
|
||||
buf.resize(read_len, 0);
|
||||
}
|
||||
let read_res = timeout(idle_timeout, reader.read(&mut buf[..read_len])).await;
|
||||
let n = match read_res {
|
||||
Ok(Ok(n)) => n,
|
||||
@@ -250,6 +273,32 @@ async fn consume_client_data_with_timeout_and_cap<R>(
|
||||
}
|
||||
}
|
||||
|
||||
fn mask_failure_drain_cap(config: &ProxyConfig) -> usize {
|
||||
let configured_cap = config.censorship.mask_relay_max_bytes;
|
||||
if configured_cap == 0 {
|
||||
return MASK_BUFFER_SIZE;
|
||||
}
|
||||
|
||||
configured_cap.min(MASK_BUFFER_SIZE)
|
||||
}
|
||||
|
||||
async fn consume_mask_failure_path<R>(
|
||||
reader: R,
|
||||
config: &ProxyConfig,
|
||||
relay_timeout: Duration,
|
||||
idle_timeout: Duration,
|
||||
) where
|
||||
R: AsyncRead + Unpin,
|
||||
{
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
mask_failure_drain_cap(config),
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn wait_mask_connect_budget(started: Instant) {
|
||||
let elapsed = started.elapsed();
|
||||
if elapsed < MASK_TIMEOUT {
|
||||
@@ -385,7 +434,7 @@ mod tls_domain_mask_host_tests {
|
||||
let mut config = ProxyConfig::default();
|
||||
config.censorship.tls_domain = "a.com".to_string();
|
||||
config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()];
|
||||
config.censorship.mask_host = Some("a.com".to_string());
|
||||
config.censorship.mask_host = None;
|
||||
config
|
||||
}
|
||||
|
||||
@@ -419,6 +468,15 @@ mod tls_domain_mask_host_tests {
|
||||
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mask_host_uses_primary_domain_when_dynamic_masking_is_disabled() {
|
||||
let mut config = config_with_tls_domains();
|
||||
config.censorship.mask_dynamic = false;
|
||||
let initial_data = client_hello_with_sni("b.com");
|
||||
|
||||
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "a.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exclusive_mask_target_overrides_only_matching_sni() {
|
||||
let mut config = config_with_tls_domains();
|
||||
@@ -471,6 +529,32 @@ fn parse_mask_host_ip_literal(host: &str) -> Option<IpAddr> {
|
||||
host.parse::<IpAddr>().ok()
|
||||
}
|
||||
|
||||
async fn resolve_mask_target_addrs(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
) -> std::io::Result<Vec<SocketAddr>> {
|
||||
if let Some(addr) = resolve_socket_addr(mask_host, mask_port) {
|
||||
return Ok(vec![addr]);
|
||||
}
|
||||
|
||||
if let Some(ip) = parse_mask_host_ip_literal(mask_host) {
|
||||
return Ok(vec![SocketAddr::new(ip, mask_port)]);
|
||||
}
|
||||
|
||||
let addrs = timeout(MASK_TIMEOUT, lookup_host((mask_host, mask_port)))
|
||||
.await
|
||||
.map_err(|_| IoError::new(ErrorKind::TimedOut, "mask target DNS lookup timed out"))??;
|
||||
let addrs = addrs.collect::<Vec<_>>();
|
||||
if addrs.is_empty() {
|
||||
return Err(IoError::new(
|
||||
ErrorKind::NotFound,
|
||||
"mask target DNS lookup returned no addresses",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(addrs)
|
||||
}
|
||||
|
||||
fn matching_tls_domain_for_sni<'a>(config: &'a ProxyConfig, sni: &str) -> Option<&'a str> {
|
||||
if config.censorship.tls_domain.eq_ignore_ascii_case(sni) {
|
||||
return Some(config.censorship.tls_domain.as_str());
|
||||
@@ -577,24 +661,32 @@ fn default_mask_tcp_target_for_initial_data<'a>(
|
||||
.as_deref()
|
||||
.unwrap_or(&config.censorship.tls_domain);
|
||||
|
||||
if !configured_mask_host.eq_ignore_ascii_case(&config.censorship.tls_domain) {
|
||||
if config.censorship.mask_host.is_none() && config.censorship.mask_dynamic {
|
||||
let extracted_sni = if sni.is_none() {
|
||||
tls::extract_sni_from_client_hello(initial_data)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(host) = sni
|
||||
.or(extracted_sni.as_deref())
|
||||
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
|
||||
{
|
||||
return MaskTcpTarget {
|
||||
host,
|
||||
port: config.censorship.mask_port,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(mask_host) = config.censorship.mask_host.as_deref() {
|
||||
return MaskTcpTarget {
|
||||
host: configured_mask_host,
|
||||
host: mask_host,
|
||||
port: config.censorship.mask_port,
|
||||
};
|
||||
}
|
||||
|
||||
let extracted_sni = if sni.is_none() {
|
||||
tls::extract_sni_from_client_hello(initial_data)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let host = sni
|
||||
.or(extracted_sni.as_deref())
|
||||
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
|
||||
.unwrap_or(configured_mask_host);
|
||||
MaskTcpTarget {
|
||||
host,
|
||||
host: configured_mask_host,
|
||||
port: config.censorship.mask_port,
|
||||
}
|
||||
}
|
||||
@@ -744,7 +836,7 @@ fn is_mask_target_local_listener_with_interfaces(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_override: Option<SocketAddr>,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
interface_ips: &[IpAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
@@ -754,7 +846,7 @@ fn is_mask_target_local_listener_with_interfaces(
|
||||
let local_ip = canonical_ip(local_addr.ip());
|
||||
let literal_mask_ip = parse_mask_host_ip_literal(mask_host).map(canonical_ip);
|
||||
|
||||
if let Some(addr) = resolved_override {
|
||||
for addr in resolved_addrs {
|
||||
let resolved_ip = canonical_ip(addr.ip());
|
||||
if resolved_ip == local_ip {
|
||||
return true;
|
||||
@@ -791,7 +883,7 @@ fn is_mask_target_local_listener(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_override: Option<SocketAddr>,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
@@ -802,7 +894,7 @@ fn is_mask_target_local_listener(
|
||||
mask_host,
|
||||
mask_port,
|
||||
local_addr,
|
||||
resolved_override,
|
||||
resolved_addrs,
|
||||
&interfaces,
|
||||
)
|
||||
}
|
||||
@@ -811,7 +903,7 @@ async fn is_mask_target_local_listener_async(
|
||||
mask_host: &str,
|
||||
mask_port: u16,
|
||||
local_addr: SocketAddr,
|
||||
resolved_override: Option<SocketAddr>,
|
||||
resolved_addrs: &[SocketAddr],
|
||||
) -> bool {
|
||||
if mask_port != local_addr.port() {
|
||||
return false;
|
||||
@@ -822,7 +914,7 @@ async fn is_mask_target_local_listener_async(
|
||||
mask_host,
|
||||
mask_port,
|
||||
local_addr,
|
||||
resolved_override,
|
||||
resolved_addrs,
|
||||
&interfaces,
|
||||
)
|
||||
}
|
||||
@@ -860,7 +952,14 @@ fn build_mask_proxy_header(
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle a bad client by forwarding to mask host
|
||||
fn configure_mask_backend_socket(stream: &TcpStream) {
|
||||
if let Err(e) = configure_tcp_socket(stream, false, Duration::from_secs(0)) {
|
||||
debug!(error = %e, "Failed to configure mask backend socket");
|
||||
}
|
||||
}
|
||||
|
||||
/// Handles a bad client by forwarding it to the configured mask target.
|
||||
#[cfg(test)]
|
||||
pub async fn handle_bad_client<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
@@ -872,6 +971,34 @@ pub async fn handle_bad_client<R, W>(
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let shared = ProxySharedState::new();
|
||||
handle_bad_client_with_shared(
|
||||
reader,
|
||||
writer,
|
||||
initial_data,
|
||||
peer,
|
||||
local_addr,
|
||||
config,
|
||||
beobachten,
|
||||
shared.as_ref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Handles a bad client with shared pre-auth fallback admission state.
|
||||
pub(crate) async fn handle_bad_client_with_shared<R, W>(
|
||||
reader: R,
|
||||
writer: W,
|
||||
initial_data: &[u8],
|
||||
peer: SocketAddr,
|
||||
local_addr: SocketAddr,
|
||||
config: &ProxyConfig,
|
||||
beobachten: &BeobachtenStore,
|
||||
shared: &ProxySharedState,
|
||||
) where
|
||||
R: AsyncRead + Unpin + Send + 'static,
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let client_type = detect_client_type(initial_data);
|
||||
if config.general.beobachten {
|
||||
@@ -894,6 +1021,17 @@ pub async fn handle_bad_client<R, W>(
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(_masking_permit) = shared.try_acquire_masking_fallback_permit() else {
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
"Masking fallback concurrency limit reached"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
};
|
||||
|
||||
let client_sni = tls::extract_sni_from_client_hello(initial_data);
|
||||
let exclusive_tcp_target = client_sni
|
||||
.as_deref()
|
||||
@@ -956,24 +1094,12 @@ pub async fn handle_bad_client<R, W>(
|
||||
Ok(Err(e)) => {
|
||||
wait_mask_connect_budget_if_needed(connect_started, config).await;
|
||||
debug!(error = %e, "Failed to connect to mask unix socket");
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("Timeout connecting to mask unix socket");
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
}
|
||||
@@ -986,11 +1112,27 @@ pub async fn handle_bad_client<R, W>(
|
||||
let mask_host = mask_target.host;
|
||||
let mask_port = mask_target.port;
|
||||
|
||||
let resolved_mask_addrs = match resolve_mask_target_addrs(mask_host, mask_port).await {
|
||||
Ok(addrs) => addrs,
|
||||
Err(e) => {
|
||||
let outcome_started = Instant::now();
|
||||
debug!(
|
||||
client_type = client_type,
|
||||
host = %mask_host,
|
||||
port = mask_port,
|
||||
error = %e,
|
||||
"Failed to resolve mask target"
|
||||
);
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Fail closed when fallback points at our own listener endpoint.
|
||||
// Self-referential masking can create recursive proxy loops under
|
||||
// misconfiguration and leak distinguishable load spikes to adversaries.
|
||||
let resolved_mask_addr = resolve_socket_addr(mask_host, mask_port);
|
||||
if is_mask_target_local_listener_async(mask_host, mask_port, local_addr, resolved_mask_addr)
|
||||
if is_mask_target_local_listener_async(mask_host, mask_port, local_addr, &resolved_mask_addrs)
|
||||
.await
|
||||
{
|
||||
let outcome_started = Instant::now();
|
||||
@@ -1001,13 +1143,7 @@ pub async fn handle_bad_client<R, W>(
|
||||
local = %local_addr,
|
||||
"Mask target resolves to local listener; refusing self-referential masking fallback"
|
||||
);
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
return;
|
||||
}
|
||||
@@ -1022,14 +1158,15 @@ pub async fn handle_bad_client<R, W>(
|
||||
"Forwarding bad client to mask host"
|
||||
);
|
||||
|
||||
// Apply runtime DNS override for mask target when configured.
|
||||
let mask_addr = resolved_mask_addr
|
||||
.map(|addr| addr.to_string())
|
||||
.unwrap_or_else(|| format!("{}:{}", mask_host, mask_port));
|
||||
let connect_started = Instant::now();
|
||||
let connect_result = timeout(MASK_TIMEOUT, TcpStream::connect(&mask_addr)).await;
|
||||
let connect_result = timeout(
|
||||
MASK_TIMEOUT,
|
||||
TcpStream::connect(resolved_mask_addrs.as_slice()),
|
||||
)
|
||||
.await;
|
||||
match connect_result {
|
||||
Ok(Ok(stream)) => {
|
||||
configure_mask_backend_socket(&stream);
|
||||
let proxy_header =
|
||||
build_mask_proxy_header(config.censorship.mask_proxy_protocol, peer, local_addr);
|
||||
|
||||
@@ -1068,24 +1205,12 @@ pub async fn handle_bad_client<R, W>(
|
||||
Ok(Err(e)) => {
|
||||
wait_mask_connect_budget_if_needed(connect_started, config).await;
|
||||
debug!(error = %e, "Failed to connect to mask host");
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!("Timeout connecting to mask host");
|
||||
consume_client_data_with_timeout_and_cap(
|
||||
reader,
|
||||
config.censorship.mask_relay_max_bytes,
|
||||
relay_timeout,
|
||||
idle_timeout,
|
||||
)
|
||||
.await;
|
||||
consume_mask_failure_path(reader, config, relay_timeout, idle_timeout).await;
|
||||
wait_mask_outcome_budget(outcome_started, config).await;
|
||||
}
|
||||
}
|
||||
@@ -1173,20 +1298,17 @@ async fn consume_client_data<R: AsyncRead + Unpin>(
|
||||
idle_timeout: Duration,
|
||||
) {
|
||||
// Keep drain path fail-closed under slow-loris stalls.
|
||||
let mut buf = Box::new([0u8; MASK_BUFFER_SIZE]);
|
||||
let mut buf = vec![0u8; MASK_BUFFER_SIZE];
|
||||
let mut total = 0usize;
|
||||
let unlimited = byte_cap == 0;
|
||||
|
||||
loop {
|
||||
let read_len = if unlimited {
|
||||
MASK_BUFFER_SIZE
|
||||
} else {
|
||||
let remaining_budget = byte_cap.saturating_sub(total);
|
||||
if remaining_budget == 0 {
|
||||
break;
|
||||
}
|
||||
remaining_budget.min(MASK_BUFFER_SIZE)
|
||||
};
|
||||
let read_len = mask_copy_read_len(total, byte_cap);
|
||||
if read_len == 0 {
|
||||
break;
|
||||
}
|
||||
if buf.len() < read_len {
|
||||
buf.resize(read_len, 0);
|
||||
}
|
||||
let n = match timeout(idle_timeout, reader.read(&mut buf[..read_len])).await {
|
||||
Ok(Ok(n)) => n,
|
||||
Ok(Err(_)) | Err(_) => break,
|
||||
@@ -1197,7 +1319,7 @@ async fn consume_client_data<R: AsyncRead + Unpin>(
|
||||
}
|
||||
|
||||
total = total.saturating_add(n);
|
||||
if !unlimited && total >= byte_cap {
|
||||
if byte_cap != 0 && total >= byte_cap {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1315,6 +1437,10 @@ mod masking_interface_cache_concurrency_security_tests;
|
||||
#[path = "tests/masking_production_cap_regression_security_tests.rs"]
|
||||
mod masking_production_cap_regression_security_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/masking_relay_manual_perf_tests.rs"]
|
||||
mod masking_relay_manual_perf_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/masking_extended_attack_surface_security_tests.rs"]
|
||||
mod masking_extended_attack_surface_security_tests;
|
||||
|
||||
@@ -52,7 +52,7 @@ use self::c2me::{
|
||||
};
|
||||
use self::d2c::{
|
||||
MeD2cFlushPolicy, MeWriterResponseOutcome, classify_me_d2c_flush_reason,
|
||||
flush_client_or_cancel, observe_me_d2c_flush_event,
|
||||
flush_client_or_cancel, me_d2c_flush_reason_requires_client_flush, observe_me_d2c_flush_event,
|
||||
process_me_writer_response_with_traffic_lease,
|
||||
};
|
||||
use self::desync::{RelayForensicsState, hash_ip_in, report_desync_frame_too_large_in};
|
||||
@@ -69,7 +69,9 @@ use self::quota::{
|
||||
#[cfg(test)]
|
||||
use self::c2me::enqueue_c2me_command;
|
||||
#[cfg(test)]
|
||||
use self::d2c::{compute_intermediate_secure_wire_len, process_me_writer_response};
|
||||
use self::d2c::{
|
||||
compute_intermediate_secure_wire_len, process_me_writer_response, write_client_payload,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use self::desync::{
|
||||
clear_desync_dedup_for_testing_in_shared, desync_dedup_get_for_testing,
|
||||
@@ -166,3 +168,7 @@ mod middle_relay_atomic_quota_invariant_tests;
|
||||
#[cfg(test)]
|
||||
#[path = "tests/middle_relay_baseline_invariant_tests.rs"]
|
||||
mod middle_relay_baseline_invariant_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tests/middle_relay_d2c_flush_padding_security_tests.rs"]
|
||||
mod middle_relay_d2c_flush_padding_security_tests;
|
||||
|
||||
@@ -55,6 +55,37 @@ pub(super) fn classify_me_d2c_flush_reason(
|
||||
MeD2cFlushReason::QueueDrain
|
||||
}
|
||||
|
||||
pub(super) fn me_d2c_flush_reason_requires_client_flush(_reason: MeD2cFlushReason) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn all_flush_reasons_trigger_physical_flush() {
|
||||
assert!(me_d2c_flush_reason_requires_client_flush(
|
||||
MeD2cFlushReason::QueueDrain
|
||||
));
|
||||
assert!(me_d2c_flush_reason_requires_client_flush(
|
||||
MeD2cFlushReason::AckImmediate
|
||||
));
|
||||
assert!(me_d2c_flush_reason_requires_client_flush(
|
||||
MeD2cFlushReason::BatchFrames
|
||||
));
|
||||
assert!(me_d2c_flush_reason_requires_client_flush(
|
||||
MeD2cFlushReason::BatchBytes
|
||||
));
|
||||
assert!(me_d2c_flush_reason_requires_client_flush(
|
||||
MeD2cFlushReason::MaxDelay
|
||||
));
|
||||
assert!(me_d2c_flush_reason_requires_client_flush(
|
||||
MeD2cFlushReason::Close
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn observe_me_d2c_flush_event(
|
||||
stats: &Stats,
|
||||
reason: MeD2cFlushReason,
|
||||
@@ -276,20 +307,13 @@ pub(in crate::proxy::middle_relay) fn compute_intermediate_secure_wire_len(
|
||||
let wire_len = data_len
|
||||
.checked_add(padding_len)
|
||||
.ok_or_else(|| ProxyError::Proxy("Frame length overflow".into()))?;
|
||||
if wire_len > 0x7fff_ffffusize {
|
||||
return Err(ProxyError::Proxy(format!(
|
||||
"Intermediate/Secure frame too large: {wire_len}"
|
||||
)));
|
||||
}
|
||||
|
||||
let len_val = crate::protocol::framing::encode_intermediate_header(wire_len, quickack)
|
||||
.ok_or_else(|| {
|
||||
ProxyError::Proxy(format!("Intermediate/Secure frame too large: {wire_len}"))
|
||||
})?;
|
||||
let total = 4usize
|
||||
.checked_add(wire_len)
|
||||
.ok_or_else(|| ProxyError::Proxy("Frame buffer size overflow".into()))?;
|
||||
let mut len_val = u32::try_from(wire_len)
|
||||
.map_err(|_| ProxyError::Proxy("Frame length conversion overflow".into()))?;
|
||||
if quickack {
|
||||
len_val |= 0x8000_0000;
|
||||
}
|
||||
Ok((len_val, total))
|
||||
}
|
||||
|
||||
|
||||
@@ -236,12 +236,8 @@ where
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
let quickack = (len_buf[3] & 0x80) != 0;
|
||||
(
|
||||
(u32::from_le_bytes(len_buf) & 0x7fff_ffff) as usize,
|
||||
quickack,
|
||||
Some(len_buf),
|
||||
)
|
||||
let header = crate::protocol::framing::parse_intermediate_header(len_buf);
|
||||
(header.wire_len, header.quickack, Some(len_buf))
|
||||
}
|
||||
};
|
||||
|
||||
@@ -331,7 +327,8 @@ where
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Secure Intermediate: strip validated trailing padding bytes.
|
||||
// Secure Intermediate strips only non-aligned tail padding; full-word
|
||||
// padding is indistinguishable from payload in VersionD framing.
|
||||
if proto_tag == ProtoTag::Secure {
|
||||
payload.truncate(secure_payload_len);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ pub(crate) async fn handle_via_middle_proxy<R, W>(
|
||||
mut route_rx: watch::Receiver<RouteCutoverState>,
|
||||
route_snapshot: RouteCutoverState,
|
||||
session_id: u64,
|
||||
session_cancel: CancellationToken,
|
||||
shared: Arc<ProxySharedState>,
|
||||
) -> Result<()>
|
||||
where
|
||||
@@ -20,6 +21,10 @@ where
|
||||
W: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let user = success.user.clone();
|
||||
if session_cancel.is_cancelled() {
|
||||
return Err(ProxyError::UserDisabled { user });
|
||||
}
|
||||
|
||||
let quota_limit = config.access.user_data_quota.get(&user).copied();
|
||||
let quota_user_stats = quota_limit.map(|_| stats.get_or_create_user_stats_handle(&user));
|
||||
let peer = success.peer;
|
||||
@@ -144,6 +149,7 @@ where
|
||||
peer,
|
||||
translated_local_addr,
|
||||
payload,
|
||||
_permit,
|
||||
flags,
|
||||
effective_tag_array,
|
||||
)
|
||||
@@ -486,12 +492,18 @@ where
|
||||
d2c_flush_policy.max_bytes,
|
||||
max_delay_fired,
|
||||
);
|
||||
let flush_started_at = if stats_clone.telemetry_policy().me_level.allows_debug() {
|
||||
let physical_flush =
|
||||
me_d2c_flush_reason_requires_client_flush(flush_reason);
|
||||
let flush_started_at = if physical_flush
|
||||
&& stats_clone.telemetry_policy().me_level.allows_debug()
|
||||
{
|
||||
Some(Instant::now())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||
if physical_flush {
|
||||
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||
}
|
||||
let flush_duration_us = flush_started_at.map(|started| {
|
||||
started
|
||||
.elapsed()
|
||||
@@ -590,6 +602,25 @@ where
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
_ = session_cancel.cancelled() => {
|
||||
warn!(
|
||||
user = %user,
|
||||
conn_id,
|
||||
"Disabled user middle session cancelled"
|
||||
);
|
||||
let _ = enqueue_c2me_command_in(
|
||||
shared.as_ref(),
|
||||
&c2me_tx,
|
||||
C2MeCommand::Close,
|
||||
c2me_send_timeout,
|
||||
stats.as_ref(),
|
||||
)
|
||||
.await;
|
||||
main_result = Err(ProxyError::UserDisabled {
|
||||
user: user.clone(),
|
||||
});
|
||||
break;
|
||||
}
|
||||
changed = route_rx.changed(), if route_watch_open => {
|
||||
if changed.is_err() {
|
||||
route_watch_open = false;
|
||||
@@ -792,7 +823,6 @@ where
|
||||
|
||||
clear_relay_idle_candidate_in(shared.as_ref(), conn_id);
|
||||
me_pool.registry().unregister(conn_id).await;
|
||||
buffer_pool.trim_to(buffer_pool.max_buffers().min(64));
|
||||
let pool_snapshot = buffer_pool.stats();
|
||||
stats.set_buffer_pool_gauges(
|
||||
pool_snapshot.pooled,
|
||||
|
||||
@@ -60,6 +60,8 @@
|
||||
|
||||
pub mod adaptive_buffers;
|
||||
pub mod client;
|
||||
// Process-wide Direct relay copy-buffer ownership and pressure policy.
|
||||
pub(crate) mod direct_buffer_budget;
|
||||
pub mod direct_relay;
|
||||
pub mod handshake;
|
||||
pub mod masking;
|
||||
|
||||
@@ -55,11 +55,13 @@ use crate::error::{ProxyError, Result};
|
||||
use crate::proxy::traffic_limiter::TrafficLease;
|
||||
use crate::stats::Stats;
|
||||
use crate::stream::BufferPool;
|
||||
use std::future::pending;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, copy_bidirectional_with_sizes};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
// ============= Constants =============
|
||||
@@ -82,8 +84,11 @@ fn watchdog_delta(current: u64, previous: u64) -> u64 {
|
||||
current.saturating_sub(previous)
|
||||
}
|
||||
|
||||
mod adaptive_copy;
|
||||
mod io;
|
||||
|
||||
pub(crate) use self::adaptive_copy::relay_direct_adaptive;
|
||||
|
||||
use self::io::{CombinedStream, SharedCounters, StatsIo, is_quota_io_error};
|
||||
#[cfg(test)]
|
||||
use self::io::{quota_adaptive_interval_bytes, should_immediate_quota_check};
|
||||
@@ -191,6 +196,85 @@ pub async fn relay_bidirectional_with_activity_timeout_and_lease<CR, CW, SR, SW>
|
||||
traffic_lease: Option<Arc<TrafficLease>>,
|
||||
activity_timeout: Duration,
|
||||
) -> Result<()>
|
||||
where
|
||||
CR: AsyncRead + Unpin + Send + 'static,
|
||||
CW: AsyncWrite + Unpin + Send + 'static,
|
||||
SR: AsyncRead + Unpin + Send + 'static,
|
||||
SW: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
relay_bidirectional_with_activity_timeout_lease_cancel_inner(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
c2s_buf_size,
|
||||
s2c_buf_size,
|
||||
user,
|
||||
stats,
|
||||
quota_limit,
|
||||
_buffer_pool,
|
||||
traffic_lease,
|
||||
activity_timeout,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub async fn relay_bidirectional_with_activity_timeout_lease_and_cancel<CR, CW, SR, SW>(
|
||||
client_reader: CR,
|
||||
client_writer: CW,
|
||||
server_reader: SR,
|
||||
server_writer: SW,
|
||||
c2s_buf_size: usize,
|
||||
s2c_buf_size: usize,
|
||||
user: &str,
|
||||
stats: Arc<Stats>,
|
||||
quota_limit: Option<u64>,
|
||||
_buffer_pool: Arc<BufferPool>,
|
||||
traffic_lease: Option<Arc<TrafficLease>>,
|
||||
activity_timeout: Duration,
|
||||
session_cancel: CancellationToken,
|
||||
) -> Result<()>
|
||||
where
|
||||
CR: AsyncRead + Unpin + Send + 'static,
|
||||
CW: AsyncWrite + Unpin + Send + 'static,
|
||||
SR: AsyncRead + Unpin + Send + 'static,
|
||||
SW: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
relay_bidirectional_with_activity_timeout_lease_cancel_inner(
|
||||
client_reader,
|
||||
client_writer,
|
||||
server_reader,
|
||||
server_writer,
|
||||
c2s_buf_size,
|
||||
s2c_buf_size,
|
||||
user,
|
||||
stats,
|
||||
quota_limit,
|
||||
_buffer_pool,
|
||||
traffic_lease,
|
||||
activity_timeout,
|
||||
Some(session_cancel),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn relay_bidirectional_with_activity_timeout_lease_cancel_inner<CR, CW, SR, SW>(
|
||||
client_reader: CR,
|
||||
client_writer: CW,
|
||||
server_reader: SR,
|
||||
server_writer: SW,
|
||||
c2s_buf_size: usize,
|
||||
s2c_buf_size: usize,
|
||||
user: &str,
|
||||
stats: Arc<Stats>,
|
||||
quota_limit: Option<u64>,
|
||||
_buffer_pool: Arc<BufferPool>,
|
||||
traffic_lease: Option<Arc<TrafficLease>>,
|
||||
activity_timeout: Duration,
|
||||
session_cancel: Option<CancellationToken>,
|
||||
) -> Result<()>
|
||||
where
|
||||
CR: AsyncRead + Unpin + Send + 'static,
|
||||
CW: AsyncWrite + Unpin + Send + 'static,
|
||||
@@ -287,14 +371,29 @@ where
|
||||
//
|
||||
// When the watchdog fires, select! drops the copy future,
|
||||
// releasing the &mut borrows on client and server.
|
||||
let copy_result = tokio::select! {
|
||||
enum RelayOutcome {
|
||||
Copy(std::io::Result<(u64, u64)>),
|
||||
ActivityTimeout,
|
||||
UserDisabled,
|
||||
}
|
||||
|
||||
let cancel_wait = async move {
|
||||
match session_cancel {
|
||||
Some(token) => token.cancelled().await,
|
||||
None => pending::<()>().await,
|
||||
}
|
||||
};
|
||||
tokio::pin!(cancel_wait);
|
||||
|
||||
let relay_outcome = tokio::select! {
|
||||
result = copy_bidirectional_with_sizes(
|
||||
&mut client,
|
||||
&mut server,
|
||||
c2s_buf_size.max(1),
|
||||
s2c_buf_size.max(1),
|
||||
) => Some(result),
|
||||
_ = watchdog => None, // Activity timeout — cancel relay
|
||||
) => RelayOutcome::Copy(result),
|
||||
_ = watchdog => RelayOutcome::ActivityTimeout,
|
||||
_ = &mut cancel_wait => RelayOutcome::UserDisabled,
|
||||
};
|
||||
|
||||
// ── Clean shutdown ──────────────────────────────────────────────
|
||||
@@ -308,8 +407,8 @@ where
|
||||
let s2c_ops = counters.s2c_ops.load(Ordering::Relaxed);
|
||||
let duration = epoch.elapsed();
|
||||
|
||||
match copy_result {
|
||||
Some(Ok((c2s, s2c))) => {
|
||||
match relay_outcome {
|
||||
RelayOutcome::Copy(Ok((c2s, s2c))) => {
|
||||
// Normal completion — one side closed the connection
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
@@ -322,7 +421,7 @@ where
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Some(Err(e)) if is_quota_io_error(&e) => {
|
||||
RelayOutcome::Copy(Err(e)) if is_quota_io_error(&e) => {
|
||||
let c2s = counters.c2s_bytes.load(Ordering::Relaxed);
|
||||
let s2c = counters.s2c_bytes.load(Ordering::Relaxed);
|
||||
warn!(
|
||||
@@ -338,7 +437,7 @@ where
|
||||
user: user_owned.clone(),
|
||||
})
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
RelayOutcome::Copy(Err(e)) => {
|
||||
// I/O error in one of the directions
|
||||
let c2s = counters.c2s_bytes.load(Ordering::Relaxed);
|
||||
let s2c = counters.s2c_bytes.load(Ordering::Relaxed);
|
||||
@@ -354,7 +453,7 @@ where
|
||||
);
|
||||
Err(e.into())
|
||||
}
|
||||
None => {
|
||||
RelayOutcome::ActivityTimeout => {
|
||||
// Activity timeout (watchdog fired)
|
||||
let c2s = counters.c2s_bytes.load(Ordering::Relaxed);
|
||||
let s2c = counters.s2c_bytes.load(Ordering::Relaxed);
|
||||
@@ -369,6 +468,22 @@ where
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
RelayOutcome::UserDisabled => {
|
||||
let c2s = counters.c2s_bytes.load(Ordering::Relaxed);
|
||||
let s2c = counters.s2c_bytes.load(Ordering::Relaxed);
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = c2s,
|
||||
s2c_bytes = s2c,
|
||||
c2s_msgs = c2s_ops,
|
||||
s2c_msgs = s2c_ops,
|
||||
duration_secs = duration.as_secs(),
|
||||
"Relay finished (user disabled)"
|
||||
);
|
||||
Err(ProxyError::UserDisabled {
|
||||
user: user_owned.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
527
src/proxy/relay/adaptive_copy.rs
Normal file
527
src/proxy/relay/adaptive_copy.rs
Normal file
@@ -0,0 +1,527 @@
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf, copy_buf};
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::error::{ProxyError, Result};
|
||||
use crate::proxy::adaptive_buffers::{
|
||||
AdaptiveTier, RelaySignalSample, SessionAdaptiveController, TierTransitionReason,
|
||||
direct_copy_buffers_for_tier_with_ceilings,
|
||||
};
|
||||
use crate::proxy::direct_buffer_budget::{
|
||||
DIRECT_BASE_C2S_BYTES, DIRECT_BASE_S2C_BYTES, DirectBufferBudget, DirectBufferLease,
|
||||
};
|
||||
use crate::proxy::traffic_limiter::TrafficLease;
|
||||
use crate::stats::Stats;
|
||||
|
||||
use super::WATCHDOG_INTERVAL;
|
||||
use super::io::{SharedCounters, StatsIo, is_quota_io_error};
|
||||
use super::watchdog_delta;
|
||||
|
||||
mod write_pressure;
|
||||
|
||||
use self::write_pressure::WritePressureIo;
|
||||
|
||||
struct AdaptiveBufferState {
|
||||
desired_bytes: AtomicUsize,
|
||||
actual_bytes: AtomicUsize,
|
||||
}
|
||||
|
||||
impl AdaptiveBufferState {
|
||||
fn new(bytes: usize) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
desired_bytes: AtomicUsize::new(bytes.max(1)),
|
||||
actual_bytes: AtomicUsize::new(bytes.max(1)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct AdaptiveBufReader<R> {
|
||||
inner: R,
|
||||
buffer: Box<[u8]>,
|
||||
pos: usize,
|
||||
cap: usize,
|
||||
state: Arc<AdaptiveBufferState>,
|
||||
}
|
||||
|
||||
impl<R> AdaptiveBufReader<R> {
|
||||
fn new(inner: R, state: Arc<AdaptiveBufferState>) -> Self {
|
||||
let bytes = state.actual_bytes.load(Ordering::Relaxed).max(1);
|
||||
Self {
|
||||
inner,
|
||||
buffer: vec![0; bytes].into_boxed_slice(),
|
||||
pos: 0,
|
||||
cap: 0,
|
||||
state,
|
||||
}
|
||||
}
|
||||
|
||||
fn resize_if_drained(&mut self) {
|
||||
if self.pos != self.cap {
|
||||
return;
|
||||
}
|
||||
let desired = self.state.desired_bytes.load(Ordering::Acquire).max(1);
|
||||
if desired == self.buffer.len() {
|
||||
return;
|
||||
}
|
||||
self.buffer = vec![0; desired].into_boxed_slice();
|
||||
self.pos = 0;
|
||||
self.cap = 0;
|
||||
self.state.actual_bytes.store(desired, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncRead for AdaptiveBufReader<R> {
|
||||
fn poll_read(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
output: &mut ReadBuf<'_>,
|
||||
) -> Poll<io::Result<()>> {
|
||||
let this = self.get_mut();
|
||||
if this.pos < this.cap {
|
||||
let available = &this.buffer[this.pos..this.cap];
|
||||
let copied = available.len().min(output.remaining());
|
||||
output.put_slice(&available[..copied]);
|
||||
this.pos += copied;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
this.resize_if_drained();
|
||||
Pin::new(&mut this.inner).poll_read(cx, output)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> AsyncBufRead for AdaptiveBufReader<R> {
|
||||
fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
|
||||
let this = self.get_mut();
|
||||
if this.pos < this.cap {
|
||||
return Poll::Ready(Ok(&this.buffer[this.pos..this.cap]));
|
||||
}
|
||||
|
||||
this.resize_if_drained();
|
||||
let mut read_buf = ReadBuf::new(&mut this.buffer);
|
||||
match Pin::new(&mut this.inner).poll_read(cx, &mut read_buf) {
|
||||
Poll::Ready(Ok(())) => {
|
||||
this.pos = 0;
|
||||
this.cap = read_buf.filled().len();
|
||||
Poll::Ready(Ok(&this.buffer[..this.cap]))
|
||||
}
|
||||
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
|
||||
fn consume(self: Pin<&mut Self>, amount: usize) {
|
||||
let this = self.get_mut();
|
||||
this.pos = this.pos.saturating_add(amount).min(this.cap);
|
||||
}
|
||||
}
|
||||
|
||||
enum AdaptiveRelayOutcome {
|
||||
Copy(io::Result<(u64, u64)>),
|
||||
ActivityTimeout,
|
||||
UserDisabled,
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Relays one Direct session with independently resizable directional buffers.
|
||||
pub(crate) async fn relay_direct_adaptive<CR, CW, SR, SW>(
|
||||
client_reader: CR,
|
||||
client_writer: CW,
|
||||
server_reader: SR,
|
||||
server_writer: SW,
|
||||
ceiling_c2s_bytes: usize,
|
||||
ceiling_s2c_bytes: usize,
|
||||
max_connections: u32,
|
||||
user: &str,
|
||||
stats: Arc<Stats>,
|
||||
quota_limit: Option<u64>,
|
||||
traffic_lease: Option<Arc<TrafficLease>>,
|
||||
activity_timeout: Duration,
|
||||
session_cancel: CancellationToken,
|
||||
budget: Arc<DirectBufferBudget>,
|
||||
) -> Result<()>
|
||||
where
|
||||
CR: AsyncRead + Unpin + Send + 'static,
|
||||
CW: AsyncWrite + Unpin + Send + 'static,
|
||||
SR: AsyncRead + Unpin + Send + 'static,
|
||||
SW: AsyncWrite + Unpin + Send + 'static,
|
||||
{
|
||||
let activity_timeout = activity_timeout.max(Duration::from_secs(1));
|
||||
let epoch = Instant::now();
|
||||
let counters = Arc::new(SharedCounters::new());
|
||||
let quota_exceeded = Arc::new(AtomicBool::new(false));
|
||||
let user_owned = user.to_string();
|
||||
|
||||
let (base_c2s, base_s2c) = initial_base_sizes(
|
||||
ceiling_c2s_bytes,
|
||||
ceiling_s2c_bytes,
|
||||
max_connections,
|
||||
budget.target_bytes(),
|
||||
);
|
||||
let base_total = base_c2s.saturating_add(base_s2c);
|
||||
let mut lease = match budget.try_reserve(base_total, false) {
|
||||
Some(lease) => lease,
|
||||
None => {
|
||||
let minimum_total = DIRECT_BASE_C2S_BYTES + DIRECT_BASE_S2C_BYTES;
|
||||
match budget.try_reserve(minimum_total, true) {
|
||||
Some(lease) => {
|
||||
budget.increment_minimum_fallback();
|
||||
lease
|
||||
}
|
||||
None => {
|
||||
budget.increment_admission_rejected();
|
||||
return Err(ProxyError::Proxy(
|
||||
"Direct relay buffer pressure: budget exhausted".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let effective_base = if lease.reserved_bytes() < base_total {
|
||||
(DIRECT_BASE_C2S_BYTES, DIRECT_BASE_S2C_BYTES)
|
||||
} else {
|
||||
(base_c2s, base_s2c)
|
||||
};
|
||||
let c2s_state = AdaptiveBufferState::new(effective_base.0);
|
||||
let s2c_state = AdaptiveBufferState::new(effective_base.1);
|
||||
|
||||
let mut controller = SessionAdaptiveController::new(AdaptiveTier::Base);
|
||||
|
||||
let c2s_client = StatsIo::new_with_traffic_lease(
|
||||
client_reader,
|
||||
Arc::clone(&counters),
|
||||
Arc::clone(&stats),
|
||||
user_owned.clone(),
|
||||
traffic_lease.clone(),
|
||||
quota_limit,
|
||||
Arc::clone("a_exceeded),
|
||||
epoch,
|
||||
);
|
||||
let client_writer = StatsIo::new_with_traffic_lease(
|
||||
client_writer,
|
||||
Arc::clone(&counters),
|
||||
Arc::clone(&stats),
|
||||
user_owned.clone(),
|
||||
traffic_lease,
|
||||
quota_limit,
|
||||
Arc::clone("a_exceeded),
|
||||
epoch,
|
||||
);
|
||||
let mut client_writer = WritePressureIo::new(client_writer, Arc::clone(&counters));
|
||||
let mut c2s_reader = AdaptiveBufReader::new(c2s_client, Arc::clone(&c2s_state));
|
||||
let mut s2c_reader = AdaptiveBufReader::new(server_reader, Arc::clone(&s2c_state));
|
||||
let mut server_writer = server_writer;
|
||||
let mut pressure_rx = budget.subscribe_pressure();
|
||||
|
||||
let relay_outcome = {
|
||||
let copy = async {
|
||||
let c2s = async {
|
||||
let copied = copy_buf(&mut c2s_reader, &mut server_writer).await?;
|
||||
server_writer.shutdown().await?;
|
||||
Ok::<u64, io::Error>(copied)
|
||||
};
|
||||
let s2c = async {
|
||||
let copied = copy_buf(&mut s2c_reader, &mut client_writer).await?;
|
||||
client_writer.shutdown().await?;
|
||||
Ok::<u64, io::Error>(copied)
|
||||
};
|
||||
tokio::try_join!(c2s, s2c)
|
||||
};
|
||||
tokio::pin!(copy);
|
||||
|
||||
let mut interval = tokio::time::interval(WATCHDOG_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
interval.tick().await;
|
||||
let mut previous = RelaySignalSample::default();
|
||||
let mut previous_log_c2s = 0u64;
|
||||
let mut previous_log_s2c = 0u64;
|
||||
let mut previous_sample_at = epoch;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut copy => break AdaptiveRelayOutcome::Copy(result),
|
||||
_ = session_cancel.cancelled() => break AdaptiveRelayOutcome::UserDisabled,
|
||||
changed = pressure_rx.changed() => {
|
||||
if changed.is_ok() {
|
||||
apply_global_pressure_demotion(
|
||||
&mut controller,
|
||||
&mut lease,
|
||||
&c2s_state,
|
||||
&s2c_state,
|
||||
effective_base,
|
||||
(ceiling_c2s_bytes, ceiling_s2c_bytes),
|
||||
budget.as_ref(),
|
||||
);
|
||||
reconcile_reservation(&mut lease, &c2s_state, &s2c_state);
|
||||
}
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
let now = Instant::now();
|
||||
let idle = counters.idle_duration(now, epoch);
|
||||
if quota_exceeded.load(Ordering::Acquire) {
|
||||
warn!(user = %user_owned, "User data quota reached, closing relay");
|
||||
break AdaptiveRelayOutcome::ActivityTimeout;
|
||||
}
|
||||
if idle >= activity_timeout {
|
||||
warn!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = counters.c2s_bytes.load(Ordering::Relaxed),
|
||||
s2c_bytes = counters.s2c_bytes.load(Ordering::Relaxed),
|
||||
idle_secs = idle.as_secs(),
|
||||
"Activity timeout"
|
||||
);
|
||||
break AdaptiveRelayOutcome::ActivityTimeout;
|
||||
}
|
||||
|
||||
let sample = current_sample(counters.as_ref());
|
||||
let c2s_delta = watchdog_delta(sample.c2s_bytes, previous_log_c2s);
|
||||
let s2c_delta = watchdog_delta(sample.s2c_written_bytes, previous_log_s2c);
|
||||
if c2s_delta > 0 || s2c_delta > 0 {
|
||||
let secs = now.saturating_duration_since(previous_sample_at).as_secs_f64();
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
c2s_kbps = (c2s_delta as f64 / secs / 1024.0) as u64,
|
||||
s2c_kbps = (s2c_delta as f64 / secs / 1024.0) as u64,
|
||||
c2s_total = sample.c2s_bytes,
|
||||
s2c_total = sample.s2c_written_bytes,
|
||||
"Relay active"
|
||||
);
|
||||
}
|
||||
|
||||
let delta = sample_delta(sample, previous);
|
||||
let tick_secs = now.saturating_duration_since(previous_sample_at).as_secs_f64();
|
||||
if let Some(transition) = controller.observe(delta, tick_secs) {
|
||||
apply_controller_transition(
|
||||
transition,
|
||||
&mut controller,
|
||||
&mut lease,
|
||||
&c2s_state,
|
||||
&s2c_state,
|
||||
effective_base,
|
||||
(ceiling_c2s_bytes, ceiling_s2c_bytes),
|
||||
budget.as_ref(),
|
||||
);
|
||||
}
|
||||
reconcile_reservation(&mut lease, &c2s_state, &s2c_state);
|
||||
previous = sample;
|
||||
previous_log_c2s = sample.c2s_bytes;
|
||||
previous_log_s2c = sample.s2c_written_bytes;
|
||||
previous_sample_at = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let _ = client_writer.shutdown().await;
|
||||
let _ = server_writer.shutdown().await;
|
||||
let c2s_ops = counters.c2s_ops.load(Ordering::Relaxed);
|
||||
let s2c_ops = counters.s2c_ops.load(Ordering::Relaxed);
|
||||
let duration = epoch.elapsed();
|
||||
match relay_outcome {
|
||||
AdaptiveRelayOutcome::Copy(Ok((c2s, s2c))) => {
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = c2s,
|
||||
s2c_bytes = s2c,
|
||||
c2s_msgs = c2s_ops,
|
||||
s2c_msgs = s2c_ops,
|
||||
duration_secs = duration.as_secs(),
|
||||
"Relay finished"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
AdaptiveRelayOutcome::Copy(Err(error)) if is_quota_io_error(&error) => {
|
||||
warn!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = counters.c2s_bytes.load(Ordering::Relaxed),
|
||||
s2c_bytes = counters.s2c_bytes.load(Ordering::Relaxed),
|
||||
c2s_msgs = c2s_ops,
|
||||
s2c_msgs = s2c_ops,
|
||||
duration_secs = duration.as_secs(),
|
||||
"Data quota reached, closing relay"
|
||||
);
|
||||
Err(ProxyError::DataQuotaExceeded { user: user_owned })
|
||||
}
|
||||
AdaptiveRelayOutcome::Copy(Err(error)) => {
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = counters.c2s_bytes.load(Ordering::Relaxed),
|
||||
s2c_bytes = counters.s2c_bytes.load(Ordering::Relaxed),
|
||||
c2s_msgs = c2s_ops,
|
||||
s2c_msgs = s2c_ops,
|
||||
duration_secs = duration.as_secs(),
|
||||
error = %error,
|
||||
"Relay error"
|
||||
);
|
||||
Err(error.into())
|
||||
}
|
||||
AdaptiveRelayOutcome::ActivityTimeout => {
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = counters.c2s_bytes.load(Ordering::Relaxed),
|
||||
s2c_bytes = counters.s2c_bytes.load(Ordering::Relaxed),
|
||||
c2s_msgs = c2s_ops,
|
||||
s2c_msgs = s2c_ops,
|
||||
duration_secs = duration.as_secs(),
|
||||
"Relay finished (activity timeout)"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
AdaptiveRelayOutcome::UserDisabled => {
|
||||
debug!(
|
||||
user = %user_owned,
|
||||
c2s_bytes = counters.c2s_bytes.load(Ordering::Relaxed),
|
||||
s2c_bytes = counters.s2c_bytes.load(Ordering::Relaxed),
|
||||
c2s_msgs = c2s_ops,
|
||||
s2c_msgs = s2c_ops,
|
||||
duration_secs = duration.as_secs(),
|
||||
"Relay finished (user disabled)"
|
||||
);
|
||||
Err(ProxyError::UserDisabled { user: user_owned })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn initial_base_sizes(
|
||||
ceiling_c2s: usize,
|
||||
ceiling_s2c: usize,
|
||||
max_connections: u32,
|
||||
target_bytes: usize,
|
||||
) -> (usize, usize) {
|
||||
let configured_total = ceiling_c2s.saturating_add(ceiling_s2c);
|
||||
let configured_worst_case = configured_total.saturating_mul(max_connections as usize);
|
||||
if max_connections != 0 && configured_worst_case <= target_bytes {
|
||||
return (ceiling_c2s, ceiling_s2c);
|
||||
}
|
||||
(
|
||||
DIRECT_BASE_C2S_BYTES.min(ceiling_c2s),
|
||||
DIRECT_BASE_S2C_BYTES.min(ceiling_s2c),
|
||||
)
|
||||
}
|
||||
|
||||
fn current_sample(counters: &SharedCounters) -> RelaySignalSample {
|
||||
RelaySignalSample {
|
||||
c2s_bytes: counters.c2s_bytes.load(Ordering::Relaxed),
|
||||
s2c_requested_bytes: counters.s2c_requested_bytes.load(Ordering::Relaxed),
|
||||
s2c_written_bytes: counters.s2c_bytes.load(Ordering::Relaxed),
|
||||
s2c_write_ops: counters.s2c_ops.load(Ordering::Relaxed),
|
||||
s2c_partial_writes: counters.s2c_partial_writes.load(Ordering::Relaxed),
|
||||
s2c_consecutive_pending_writes: counters
|
||||
.s2c_consecutive_pending_writes
|
||||
.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_delta(current: RelaySignalSample, previous: RelaySignalSample) -> RelaySignalSample {
|
||||
RelaySignalSample {
|
||||
c2s_bytes: current.c2s_bytes.saturating_sub(previous.c2s_bytes),
|
||||
s2c_requested_bytes: current
|
||||
.s2c_requested_bytes
|
||||
.saturating_sub(previous.s2c_requested_bytes),
|
||||
s2c_written_bytes: current
|
||||
.s2c_written_bytes
|
||||
.saturating_sub(previous.s2c_written_bytes),
|
||||
s2c_write_ops: current.s2c_write_ops.saturating_sub(previous.s2c_write_ops),
|
||||
s2c_partial_writes: current
|
||||
.s2c_partial_writes
|
||||
.saturating_sub(previous.s2c_partial_writes),
|
||||
s2c_consecutive_pending_writes: current.s2c_consecutive_pending_writes,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn apply_controller_transition(
|
||||
transition: crate::proxy::adaptive_buffers::TierTransition,
|
||||
controller: &mut SessionAdaptiveController,
|
||||
lease: &mut DirectBufferLease,
|
||||
c2s_state: &AdaptiveBufferState,
|
||||
s2c_state: &AdaptiveBufferState,
|
||||
base: (usize, usize),
|
||||
ceilings: (usize, usize),
|
||||
budget: &DirectBufferBudget,
|
||||
) {
|
||||
let sizes = direct_copy_buffers_for_tier_with_ceilings(
|
||||
transition.to,
|
||||
base.0,
|
||||
base.1,
|
||||
ceilings.0,
|
||||
ceilings.1,
|
||||
);
|
||||
if transition.to > transition.from {
|
||||
if !lease.try_grow_to(sizes.0.saturating_add(sizes.1)) {
|
||||
*controller = SessionAdaptiveController::new(transition.from);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
match transition.reason {
|
||||
TierTransitionReason::QuietDemotion => budget.increment_quiet_demotion(),
|
||||
TierTransitionReason::SustainedWritePressure => {
|
||||
budget.increment_write_pressure_demotion();
|
||||
}
|
||||
TierTransitionReason::SoftConfirmed | TierTransitionReason::HardPressure => {}
|
||||
}
|
||||
}
|
||||
set_desired_sizes(c2s_state, s2c_state, sizes);
|
||||
lease.set_tier(transition.to.as_u8() as usize);
|
||||
}
|
||||
|
||||
fn apply_global_pressure_demotion(
|
||||
controller: &mut SessionAdaptiveController,
|
||||
lease: &mut DirectBufferLease,
|
||||
c2s_state: &AdaptiveBufferState,
|
||||
s2c_state: &AdaptiveBufferState,
|
||||
base: (usize, usize),
|
||||
ceilings: (usize, usize),
|
||||
budget: &DirectBufferBudget,
|
||||
) {
|
||||
let current = controller.tier();
|
||||
let target = current.demote();
|
||||
if target == current {
|
||||
return;
|
||||
}
|
||||
*controller = SessionAdaptiveController::new(target);
|
||||
let sizes =
|
||||
direct_copy_buffers_for_tier_with_ceilings(target, base.0, base.1, ceilings.0, ceilings.1);
|
||||
set_desired_sizes(c2s_state, s2c_state, sizes);
|
||||
lease.set_tier(target.as_u8() as usize);
|
||||
budget.increment_global_pressure_demotion();
|
||||
}
|
||||
|
||||
fn set_desired_sizes(
|
||||
c2s_state: &AdaptiveBufferState,
|
||||
s2c_state: &AdaptiveBufferState,
|
||||
sizes: (usize, usize),
|
||||
) {
|
||||
c2s_state
|
||||
.desired_bytes
|
||||
.store(sizes.0.max(1), Ordering::Release);
|
||||
s2c_state
|
||||
.desired_bytes
|
||||
.store(sizes.1.max(1), Ordering::Release);
|
||||
}
|
||||
|
||||
fn reconcile_reservation(
|
||||
lease: &mut DirectBufferLease,
|
||||
c2s_state: &AdaptiveBufferState,
|
||||
s2c_state: &AdaptiveBufferState,
|
||||
) {
|
||||
// Promotion reserves the desired allocation before either reader grows.
|
||||
// Demotion keeps the actual allocation covered until its buffered bytes drain.
|
||||
let covered_c2s = c2s_state
|
||||
.actual_bytes
|
||||
.load(Ordering::Acquire)
|
||||
.max(c2s_state.desired_bytes.load(Ordering::Acquire));
|
||||
let covered_s2c = s2c_state
|
||||
.actual_bytes
|
||||
.load(Ordering::Acquire)
|
||||
.max(s2c_state.desired_bytes.load(Ordering::Acquire));
|
||||
lease.shrink_to(covered_c2s.saturating_add(covered_s2c));
|
||||
}
|
||||
72
src/proxy/relay/adaptive_copy/write_pressure.rs
Normal file
72
src/proxy/relay/adaptive_copy/write_pressure.rs
Normal file
@@ -0,0 +1,72 @@
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
use super::super::io::SharedCounters;
|
||||
|
||||
/// Direct-only writer wrapper that exposes bounded backpressure signals.
|
||||
pub(super) struct WritePressureIo<W> {
|
||||
inner: W,
|
||||
counters: Arc<SharedCounters>,
|
||||
}
|
||||
|
||||
impl<W> WritePressureIo<W> {
|
||||
/// Wraps the client writer without changing its I/O or error contract.
|
||||
pub(super) fn new(inner: W, counters: Arc<SharedCounters>) -> Self {
|
||||
Self { inner, counters }
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: AsyncWrite + Unpin> AsyncWrite for WritePressureIo<W> {
|
||||
fn poll_write(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
buffer: &[u8],
|
||||
) -> Poll<io::Result<usize>> {
|
||||
let this = self.get_mut();
|
||||
if !buffer.is_empty() {
|
||||
this.counters
|
||||
.s2c_requested_bytes
|
||||
.fetch_add(buffer.len() as u64, Ordering::Relaxed);
|
||||
}
|
||||
match Pin::new(&mut this.inner).poll_write(cx, buffer) {
|
||||
Poll::Ready(Ok(written)) => {
|
||||
this.counters
|
||||
.s2c_consecutive_pending_writes
|
||||
.store(0, Ordering::Relaxed);
|
||||
if written < buffer.len() {
|
||||
this.counters
|
||||
.s2c_partial_writes
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
Poll::Ready(Err(error)) => {
|
||||
this.counters
|
||||
.s2c_consecutive_pending_writes
|
||||
.store(0, Ordering::Relaxed);
|
||||
Poll::Ready(Err(error))
|
||||
}
|
||||
Poll::Pending => {
|
||||
let _ = this.counters.s2c_consecutive_pending_writes.fetch_update(
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
|current| Some(current.saturating_add(1)),
|
||||
);
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.get_mut().inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Pin::new(&mut self.get_mut().inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::time::Instant;
|
||||
@@ -20,6 +20,12 @@ pub(in crate::proxy::relay) struct SharedCounters {
|
||||
pub(in crate::proxy::relay) c2s_ops: AtomicU64,
|
||||
/// Number of poll_write completions (≈ S→C chunks)
|
||||
pub(in crate::proxy::relay) s2c_ops: AtomicU64,
|
||||
/// Bytes presented to client writes, including retried pending writes.
|
||||
pub(in crate::proxy::relay) s2c_requested_bytes: AtomicU64,
|
||||
/// Successful client writes that consumed only part of the offered slice.
|
||||
pub(in crate::proxy::relay) s2c_partial_writes: AtomicU64,
|
||||
/// Consecutive pending client writes observed by the active copy loop.
|
||||
pub(in crate::proxy::relay) s2c_consecutive_pending_writes: AtomicU32,
|
||||
/// Milliseconds since relay epoch of last I/O activity
|
||||
last_activity_ms: AtomicU64,
|
||||
}
|
||||
@@ -31,6 +37,9 @@ impl SharedCounters {
|
||||
s2c_bytes: AtomicU64::new(0),
|
||||
c2s_ops: AtomicU64::new(0),
|
||||
s2c_ops: AtomicU64::new(0),
|
||||
s2c_requested_bytes: AtomicU64::new(0),
|
||||
s2c_partial_writes: AtomicU64::new(0),
|
||||
s2c_consecutive_pending_writes: AtomicU32::new(0),
|
||||
last_activity_ms: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use crate::stats::UserStats;
|
||||
use std::io;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct QuotaIoSentinel;
|
||||
@@ -52,17 +51,5 @@ pub(super) fn refund_reserved_quota_bytes(user_stats: &UserStats, reserved_bytes
|
||||
if reserved_bytes == 0 {
|
||||
return;
|
||||
}
|
||||
let mut current = user_stats.quota_used.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = current.saturating_sub(reserved_bytes);
|
||||
match user_stats.quota_used.compare_exchange_weak(
|
||||
current,
|
||||
next,
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => return,
|
||||
Err(observed) => current = observed,
|
||||
}
|
||||
}
|
||||
user_stats.refund_quota(reserved_bytes);
|
||||
}
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
use std::collections::HashSet;
|
||||
use std::collections::hash_map::RandomState;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::proxy::direct_buffer_budget::{DirectBufferBudget, fallback_direct_buffer_hard_limit};
|
||||
use crate::proxy::handshake::{AuthProbeSaturationState, AuthProbeState};
|
||||
use crate::proxy::middle_relay::{DesyncDedupRotationState, RelayIdleCandidateRegistry};
|
||||
use crate::proxy::traffic_limiter::TrafficLimiter;
|
||||
|
||||
const HANDSHAKE_RECENT_USER_RING_LEN: usize = 64;
|
||||
const MASKING_FALLBACK_MAX_CONCURRENT: usize = 512;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ConntrackCloseReason {
|
||||
@@ -67,12 +70,48 @@ pub(crate) struct ProxySharedState {
|
||||
pub(crate) handshake: HandshakeSharedState,
|
||||
pub(crate) middle_relay: MiddleRelaySharedState,
|
||||
pub(crate) traffic_limiter: Arc<TrafficLimiter>,
|
||||
pub(crate) direct_buffer_budget: Arc<DirectBufferBudget>,
|
||||
disabled_users: DashMap<String, ()>,
|
||||
active_user_sessions: DashMap<(String, u64), CancellationToken>,
|
||||
pub(crate) conntrack_pressure_active: AtomicBool,
|
||||
pub(crate) conntrack_close_tx: Mutex<Option<mpsc::Sender<ConntrackCloseEvent>>>,
|
||||
masking_fallback_permits: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[must_use = "registered user sessions must be kept alive until relay completion"]
|
||||
pub(crate) struct UserSessionRegistration {
|
||||
token: CancellationToken,
|
||||
_guard: UserSessionGuard,
|
||||
}
|
||||
|
||||
impl UserSessionRegistration {
|
||||
pub(crate) fn token(&self) -> CancellationToken {
|
||||
self.token.clone()
|
||||
}
|
||||
}
|
||||
|
||||
struct UserSessionGuard {
|
||||
shared: Arc<ProxySharedState>,
|
||||
key: (String, u64),
|
||||
}
|
||||
|
||||
impl Drop for UserSessionGuard {
|
||||
fn drop(&mut self) {
|
||||
self.shared.active_user_sessions.remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
impl ProxySharedState {
|
||||
pub(crate) fn new() -> Arc<Self> {
|
||||
Self::new_with_direct_buffer_budget(DirectBufferBudget::new(
|
||||
fallback_direct_buffer_hard_limit(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Creates process state with the startup-resolved Direct buffer envelope.
|
||||
pub(crate) fn new_with_direct_buffer_budget(
|
||||
direct_buffer_budget: Arc<DirectBufferBudget>,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
handshake: HandshakeSharedState {
|
||||
auth_probe: DashMap::new(),
|
||||
@@ -101,11 +140,92 @@ impl ProxySharedState {
|
||||
relay_idle_mark_seq: AtomicU64::new(0),
|
||||
},
|
||||
traffic_limiter: TrafficLimiter::new(),
|
||||
direct_buffer_budget,
|
||||
disabled_users: DashMap::new(),
|
||||
active_user_sessions: DashMap::new(),
|
||||
conntrack_pressure_active: AtomicBool::new(false),
|
||||
conntrack_close_tx: Mutex::new(None),
|
||||
masking_fallback_permits: Arc::new(Semaphore::new(MASKING_FALLBACK_MAX_CONCURRENT)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Attempts to reserve one masking fallback slot for a pre-auth connection.
|
||||
pub(crate) fn try_acquire_masking_fallback_permit(&self) -> Option<OwnedSemaphorePermit> {
|
||||
self.masking_fallback_permits
|
||||
.clone()
|
||||
.try_acquire_owned()
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub(crate) fn is_user_enabled(&self, user: &str) -> bool {
|
||||
!self.disabled_users.contains_key(user)
|
||||
}
|
||||
|
||||
pub(crate) fn set_user_enabled(&self, user: &str, enabled: bool) -> bool {
|
||||
if enabled {
|
||||
self.disabled_users.remove(user);
|
||||
false
|
||||
} else {
|
||||
self.disabled_users.insert(user.to_string(), ()).is_none()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn apply_user_enabled_config(
|
||||
&self,
|
||||
user_enabled: &HashMap<String, bool>,
|
||||
) -> Vec<String> {
|
||||
let desired_disabled = user_enabled
|
||||
.iter()
|
||||
.filter_map(|(user, enabled)| (!*enabled).then_some(user.clone()))
|
||||
.collect::<HashSet<_>>();
|
||||
let current_disabled = self
|
||||
.disabled_users
|
||||
.iter()
|
||||
.map(|entry| entry.key().clone())
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
for user in current_disabled.difference(&desired_disabled) {
|
||||
self.disabled_users.remove(user);
|
||||
}
|
||||
let newly_disabled = desired_disabled
|
||||
.difference(¤t_disabled)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for user in desired_disabled {
|
||||
self.disabled_users.insert(user, ());
|
||||
}
|
||||
newly_disabled
|
||||
}
|
||||
|
||||
pub(crate) fn register_user_session(
|
||||
self: &Arc<Self>,
|
||||
user: &str,
|
||||
session_id: u64,
|
||||
) -> UserSessionRegistration {
|
||||
let token = CancellationToken::new();
|
||||
let key = (user.to_string(), session_id);
|
||||
self.active_user_sessions.insert(key.clone(), token.clone());
|
||||
UserSessionRegistration {
|
||||
token,
|
||||
_guard: UserSessionGuard {
|
||||
shared: Arc::clone(self),
|
||||
key,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_user_sessions(&self, user: &str) -> usize {
|
||||
let tokens = self
|
||||
.active_user_sessions
|
||||
.iter()
|
||||
.filter_map(|entry| (entry.key().0 == user).then(|| entry.value().clone()))
|
||||
.collect::<Vec<_>>();
|
||||
for token in &tokens {
|
||||
token.cancel();
|
||||
}
|
||||
tokens.len()
|
||||
}
|
||||
|
||||
pub(crate) fn set_conntrack_close_sender(&self, tx: mpsc::Sender<ConntrackCloseEvent>) {
|
||||
match self.conntrack_close_tx.lock() {
|
||||
Ok(mut guard) => {
|
||||
@@ -166,3 +286,48 @@ impl ProxySharedState {
|
||||
self.conntrack_pressure_active.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn user_enabled_config_sync_tracks_disabled_overrides() {
|
||||
let shared = ProxySharedState::new();
|
||||
assert!(shared.is_user_enabled("alice"));
|
||||
|
||||
let mut user_enabled = HashMap::new();
|
||||
user_enabled.insert("alice".to_string(), false);
|
||||
user_enabled.insert("bob".to_string(), true);
|
||||
|
||||
let mut newly_disabled = shared.apply_user_enabled_config(&user_enabled);
|
||||
newly_disabled.sort();
|
||||
assert_eq!(newly_disabled, vec!["alice".to_string()]);
|
||||
assert!(!shared.is_user_enabled("alice"));
|
||||
assert!(shared.is_user_enabled("bob"));
|
||||
|
||||
assert!(shared.apply_user_enabled_config(&user_enabled).is_empty());
|
||||
|
||||
user_enabled.clear();
|
||||
assert!(shared.apply_user_enabled_config(&user_enabled).is_empty());
|
||||
assert!(shared.is_user_enabled("alice"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_user_sessions_cancels_only_registered_matching_user() {
|
||||
let shared = ProxySharedState::new();
|
||||
let alice_1 = shared.register_user_session("alice", 1);
|
||||
let alice_2 = shared.register_user_session("alice", 2);
|
||||
let bob = shared.register_user_session("bob", 1);
|
||||
let alice_1_token = alice_1.token();
|
||||
let alice_2_token = alice_2.token();
|
||||
let bob_token = bob.token();
|
||||
|
||||
drop(alice_1);
|
||||
|
||||
assert_eq!(shared.cancel_user_sessions("alice"), 1);
|
||||
assert!(!alice_1_token.is_cancelled());
|
||||
assert!(alice_2_token.is_cancelled());
|
||||
assert!(!bob_token.is_cancelled());
|
||||
}
|
||||
}
|
||||
|
||||
58
src/proxy/tests/adaptive_direct_budget_policy_tests.rs
Normal file
58
src/proxy/tests/adaptive_direct_budget_policy_tests.rs
Normal file
@@ -0,0 +1,58 @@
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn configured_direct_sizes_are_strict_tier_ceilings() {
|
||||
let base = (4 * 1024, 8 * 1024);
|
||||
let ceilings = (64 * 1024, 256 * 1024);
|
||||
assert_eq!(
|
||||
direct_copy_buffers_for_tier_with_ceilings(
|
||||
AdaptiveTier::Base,
|
||||
base.0,
|
||||
base.1,
|
||||
ceilings.0,
|
||||
ceilings.1,
|
||||
),
|
||||
base
|
||||
);
|
||||
assert_eq!(
|
||||
direct_copy_buffers_for_tier_with_ceilings(
|
||||
AdaptiveTier::Tier3,
|
||||
base.0,
|
||||
base.1,
|
||||
ceilings.0,
|
||||
ceilings.1,
|
||||
),
|
||||
ceilings
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sustained_pending_pressure_demotes_after_transient_promotion() {
|
||||
let mut controller = SessionAdaptiveController::new(AdaptiveTier::Tier1);
|
||||
let pressure = RelaySignalSample {
|
||||
c2s_bytes: 0,
|
||||
s2c_requested_bytes: 1024,
|
||||
s2c_written_bytes: 0,
|
||||
s2c_write_ops: 0,
|
||||
s2c_partial_writes: 0,
|
||||
s2c_consecutive_pending_writes: 3,
|
||||
};
|
||||
|
||||
let first = controller
|
||||
.observe(pressure, 10.0)
|
||||
.expect("transient pressure must retain the staged promotion");
|
||||
assert_eq!(first.reason, TierTransitionReason::HardPressure);
|
||||
|
||||
let second = controller
|
||||
.observe(pressure, 10.0)
|
||||
.expect("bounded transient pressure may promote one additional tier");
|
||||
assert_eq!(second.reason, TierTransitionReason::HardPressure);
|
||||
let sustained = controller
|
||||
.observe(pressure, 10.0)
|
||||
.expect("sustained pressure must release one tier");
|
||||
assert_eq!(
|
||||
sustained.reason,
|
||||
TierTransitionReason::SustainedWritePressure
|
||||
);
|
||||
assert_eq!(sustained.to, AdaptiveTier::Tier2);
|
||||
}
|
||||
@@ -39,6 +39,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
@@ -85,17 +86,72 @@ fn make_valid_tls_client_hello(secret: &[u8], timestamp: u32, tls_len: usize, fi
|
||||
"TLS length must fit into record header"
|
||||
);
|
||||
|
||||
let total_len = 5 + tls_len;
|
||||
let mut handshake = vec![fill; total_len];
|
||||
|
||||
handshake[0] = 0x16;
|
||||
handshake[1] = 0x03;
|
||||
handshake[2] = 0x01;
|
||||
handshake[3..5].copy_from_slice(&(tls_len as u16).to_be_bytes());
|
||||
|
||||
const TLS_AES_128_GCM_SHA256: [u8; 2] = [0x13, 0x01];
|
||||
const TLS_EXTENSION_KEY_SHARE: u16 = 0x0033;
|
||||
const TLS_EXTENSION_PADDING: u16 = 0x0015;
|
||||
const X25519_KEY_SHARE_LEN: usize = 32;
|
||||
let session_id_len: usize = 32;
|
||||
handshake[tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN] = session_id_len as u8;
|
||||
|
||||
let mut extensions = Vec::new();
|
||||
let mut key_share = Vec::new();
|
||||
key_share.extend_from_slice(&tls::TLS_NAMED_GROUP_X25519.to_be_bytes());
|
||||
key_share.extend_from_slice(&(X25519_KEY_SHARE_LEN as u16).to_be_bytes());
|
||||
key_share.push(9);
|
||||
key_share.resize(key_share.len() + X25519_KEY_SHARE_LEN - 1, 0);
|
||||
|
||||
let mut key_share_extension = Vec::new();
|
||||
key_share_extension.extend_from_slice(&(key_share.len() as u16).to_be_bytes());
|
||||
key_share_extension.extend_from_slice(&key_share);
|
||||
extensions.extend_from_slice(&TLS_EXTENSION_KEY_SHARE.to_be_bytes());
|
||||
extensions.extend_from_slice(&(key_share_extension.len() as u16).to_be_bytes());
|
||||
extensions.extend_from_slice(&key_share_extension);
|
||||
|
||||
let base_tls_len = 4
|
||||
+ 2
|
||||
+ 32
|
||||
+ 1
|
||||
+ session_id_len
|
||||
+ 2
|
||||
+ TLS_AES_128_GCM_SHA256.len()
|
||||
+ 1
|
||||
+ 1
|
||||
+ 2
|
||||
+ extensions.len();
|
||||
assert!(
|
||||
tls_len == base_tls_len || tls_len >= base_tls_len + 4,
|
||||
"TLS length must leave room for a complete padding extension"
|
||||
);
|
||||
if tls_len > base_tls_len {
|
||||
let padding_len = tls_len - base_tls_len - 4;
|
||||
extensions.extend_from_slice(&TLS_EXTENSION_PADDING.to_be_bytes());
|
||||
extensions.extend_from_slice(&(padding_len as u16).to_be_bytes());
|
||||
extensions.resize(extensions.len() + padding_len, fill);
|
||||
}
|
||||
|
||||
let body_len = tls_len - 4;
|
||||
let mut body = Vec::with_capacity(body_len);
|
||||
body.extend_from_slice(&TLS_VERSION);
|
||||
body.extend_from_slice(&[fill; 32]);
|
||||
body.push(session_id_len as u8);
|
||||
body.extend_from_slice(&[fill; 32]);
|
||||
body.extend_from_slice(&(TLS_AES_128_GCM_SHA256.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&TLS_AES_128_GCM_SHA256);
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&extensions);
|
||||
assert_eq!(body.len(), body_len);
|
||||
|
||||
let mut handshake = Vec::with_capacity(5 + tls_len);
|
||||
handshake.push(0x16);
|
||||
handshake.extend_from_slice(&[0x03, 0x01]);
|
||||
handshake.extend_from_slice(&(tls_len as u16).to_be_bytes());
|
||||
handshake.push(0x01);
|
||||
let body_len_bytes = (body_len as u32).to_be_bytes();
|
||||
handshake.extend_from_slice(&body_len_bytes[1..4]);
|
||||
handshake.extend_from_slice(&body);
|
||||
|
||||
// The proxy authenticates TLS-fronted clients through the random field.
|
||||
handshake[tls::TLS_DIGEST_POS..tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN].fill(0);
|
||||
let computed = sha256_hmac(secret, &handshake);
|
||||
let mut digest = computed;
|
||||
@@ -182,10 +238,11 @@ async fn run_tls_success_mtproto_fail_capture(
|
||||
assert_eq!(tls_response_head[0], 0x16);
|
||||
read_and_discard_tls_record_body(&mut client_side, tls_response_head).await;
|
||||
|
||||
client_side.write_all(&bad_mtproto_record).await.unwrap();
|
||||
let mut client_payload = bad_mtproto_record;
|
||||
for record in trailing_records {
|
||||
client_side.write_all(&record).await.unwrap();
|
||||
client_payload.extend_from_slice(&record);
|
||||
}
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
|
||||
let got = tokio::time::timeout(Duration::from_secs(4), accept_task)
|
||||
.await
|
||||
@@ -434,11 +491,9 @@ async fn blackhat_campaign_06_replayed_tls_hello_is_masked_without_serverhello()
|
||||
client_side.read_exact(&mut head).await.unwrap();
|
||||
assert_eq!(head[0], 0x16);
|
||||
read_and_discard_tls_record_body(&mut client_side, head).await;
|
||||
client_side
|
||||
.write_all(&invalid_mtproto_record)
|
||||
.await
|
||||
.unwrap();
|
||||
client_side.write_all(&first_tail).await.unwrap();
|
||||
let mut client_payload = invalid_mtproto_record;
|
||||
client_payload.extend_from_slice(&first_tail);
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
} else {
|
||||
let mut one = [0u8; 1];
|
||||
let no_server_hello = tokio::time::timeout(
|
||||
@@ -740,8 +795,9 @@ async fn blackhat_campaign_12_parallel_tls_success_mtproto_fail_sessions_keep_is
|
||||
let mut head = [0u8; 5];
|
||||
client_side.read_exact(&mut head).await.unwrap();
|
||||
read_and_discard_tls_record_body(&mut client_side, head).await;
|
||||
client_side.write_all(&bad).await.unwrap();
|
||||
client_side.write_all(&tail).await.unwrap();
|
||||
let mut client_payload = bad;
|
||||
client_payload.extend_from_slice(&tail);
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
client_side.shutdown().await.unwrap();
|
||||
|
||||
let result = tokio::time::timeout(Duration::from_secs(5), handler)
|
||||
|
||||
@@ -35,6 +35,7 @@ fn build_harness(config: ProxyConfig) -> PipelineHarness {
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
@@ -64,17 +65,72 @@ fn make_valid_tls_client_hello(secret: &[u8], timestamp: u32, tls_len: usize, fi
|
||||
"TLS length must fit into record header"
|
||||
);
|
||||
|
||||
let total_len = 5 + tls_len;
|
||||
let mut handshake = vec![fill; total_len];
|
||||
|
||||
handshake[0] = 0x16;
|
||||
handshake[1] = 0x03;
|
||||
handshake[2] = 0x01;
|
||||
handshake[3..5].copy_from_slice(&(tls_len as u16).to_be_bytes());
|
||||
|
||||
const TLS_AES_128_GCM_SHA256: [u8; 2] = [0x13, 0x01];
|
||||
const TLS_EXTENSION_KEY_SHARE: u16 = 0x0033;
|
||||
const TLS_EXTENSION_PADDING: u16 = 0x0015;
|
||||
const X25519_KEY_SHARE_LEN: usize = 32;
|
||||
let session_id_len: usize = 32;
|
||||
handshake[tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN] = session_id_len as u8;
|
||||
|
||||
let mut extensions = Vec::new();
|
||||
let mut key_share = Vec::new();
|
||||
key_share.extend_from_slice(&tls::TLS_NAMED_GROUP_X25519.to_be_bytes());
|
||||
key_share.extend_from_slice(&(X25519_KEY_SHARE_LEN as u16).to_be_bytes());
|
||||
key_share.push(9);
|
||||
key_share.resize(key_share.len() + X25519_KEY_SHARE_LEN - 1, 0);
|
||||
|
||||
let mut key_share_extension = Vec::new();
|
||||
key_share_extension.extend_from_slice(&(key_share.len() as u16).to_be_bytes());
|
||||
key_share_extension.extend_from_slice(&key_share);
|
||||
extensions.extend_from_slice(&TLS_EXTENSION_KEY_SHARE.to_be_bytes());
|
||||
extensions.extend_from_slice(&(key_share_extension.len() as u16).to_be_bytes());
|
||||
extensions.extend_from_slice(&key_share_extension);
|
||||
|
||||
let base_tls_len = 4
|
||||
+ 2
|
||||
+ 32
|
||||
+ 1
|
||||
+ session_id_len
|
||||
+ 2
|
||||
+ TLS_AES_128_GCM_SHA256.len()
|
||||
+ 1
|
||||
+ 1
|
||||
+ 2
|
||||
+ extensions.len();
|
||||
assert!(
|
||||
tls_len == base_tls_len || tls_len >= base_tls_len + 4,
|
||||
"TLS length must leave room for a complete padding extension"
|
||||
);
|
||||
if tls_len > base_tls_len {
|
||||
let padding_len = tls_len - base_tls_len - 4;
|
||||
extensions.extend_from_slice(&TLS_EXTENSION_PADDING.to_be_bytes());
|
||||
extensions.extend_from_slice(&(padding_len as u16).to_be_bytes());
|
||||
extensions.resize(extensions.len() + padding_len, fill);
|
||||
}
|
||||
|
||||
let body_len = tls_len - 4;
|
||||
let mut body = Vec::with_capacity(body_len);
|
||||
body.extend_from_slice(&TLS_VERSION);
|
||||
body.extend_from_slice(&[fill; 32]);
|
||||
body.push(session_id_len as u8);
|
||||
body.extend_from_slice(&[fill; 32]);
|
||||
body.extend_from_slice(&(TLS_AES_128_GCM_SHA256.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&TLS_AES_128_GCM_SHA256);
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&extensions);
|
||||
assert_eq!(body.len(), body_len);
|
||||
|
||||
let mut handshake = Vec::with_capacity(5 + tls_len);
|
||||
handshake.push(0x16);
|
||||
handshake.extend_from_slice(&[0x03, 0x01]);
|
||||
handshake.extend_from_slice(&(tls_len as u16).to_be_bytes());
|
||||
handshake.push(0x01);
|
||||
let body_len_bytes = (body_len as u32).to_be_bytes();
|
||||
handshake.extend_from_slice(&body_len_bytes[1..4]);
|
||||
handshake.extend_from_slice(&body);
|
||||
|
||||
// The proxy authenticates TLS-fronted clients through the random field.
|
||||
handshake[tls::TLS_DIGEST_POS..tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN].fill(0);
|
||||
let computed = sha256_hmac(secret, &handshake);
|
||||
let mut digest = computed;
|
||||
@@ -239,11 +295,9 @@ async fn tls_mtproto_bad_client_does_not_reinject_clienthello_into_mask_backend(
|
||||
assert_eq!(tls_response_head[0], 0x16);
|
||||
read_and_discard_tls_record_body(&mut client_side, tls_response_head).await;
|
||||
|
||||
client_side
|
||||
.write_all(&invalid_mtproto_record)
|
||||
.await
|
||||
.unwrap();
|
||||
client_side.write_all(&trailing_record).await.unwrap();
|
||||
let mut client_payload = invalid_mtproto_record;
|
||||
client_payload.extend_from_slice(&trailing_record);
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(3), accept_task)
|
||||
.await
|
||||
|
||||
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
|
||||
@@ -19,6 +19,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
|
||||
@@ -33,6 +33,7 @@ fn new_upstream_manager(stats: Arc<Stats>) -> Arc<UpstreamManager> {
|
||||
selected_scope: String::new(),
|
||||
ipv4: None,
|
||||
ipv6: None,
|
||||
prefer: None,
|
||||
}],
|
||||
1,
|
||||
1,
|
||||
@@ -79,17 +80,72 @@ fn make_valid_tls_client_hello(secret: &[u8], timestamp: u32, tls_len: usize, fi
|
||||
"TLS length must fit into record header"
|
||||
);
|
||||
|
||||
let total_len = 5 + tls_len;
|
||||
let mut handshake = vec![fill; total_len];
|
||||
|
||||
handshake[0] = 0x16;
|
||||
handshake[1] = 0x03;
|
||||
handshake[2] = 0x01;
|
||||
handshake[3..5].copy_from_slice(&(tls_len as u16).to_be_bytes());
|
||||
|
||||
const TLS_AES_128_GCM_SHA256: [u8; 2] = [0x13, 0x01];
|
||||
const TLS_EXTENSION_KEY_SHARE: u16 = 0x0033;
|
||||
const TLS_EXTENSION_PADDING: u16 = 0x0015;
|
||||
const X25519_KEY_SHARE_LEN: usize = 32;
|
||||
let session_id_len: usize = 32;
|
||||
handshake[tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN] = session_id_len as u8;
|
||||
|
||||
let mut extensions = Vec::new();
|
||||
let mut key_share = Vec::new();
|
||||
key_share.extend_from_slice(&tls::TLS_NAMED_GROUP_X25519.to_be_bytes());
|
||||
key_share.extend_from_slice(&(X25519_KEY_SHARE_LEN as u16).to_be_bytes());
|
||||
key_share.push(9);
|
||||
key_share.resize(key_share.len() + X25519_KEY_SHARE_LEN - 1, 0);
|
||||
|
||||
let mut key_share_extension = Vec::new();
|
||||
key_share_extension.extend_from_slice(&(key_share.len() as u16).to_be_bytes());
|
||||
key_share_extension.extend_from_slice(&key_share);
|
||||
extensions.extend_from_slice(&TLS_EXTENSION_KEY_SHARE.to_be_bytes());
|
||||
extensions.extend_from_slice(&(key_share_extension.len() as u16).to_be_bytes());
|
||||
extensions.extend_from_slice(&key_share_extension);
|
||||
|
||||
let base_tls_len = 4
|
||||
+ 2
|
||||
+ 32
|
||||
+ 1
|
||||
+ session_id_len
|
||||
+ 2
|
||||
+ TLS_AES_128_GCM_SHA256.len()
|
||||
+ 1
|
||||
+ 1
|
||||
+ 2
|
||||
+ extensions.len();
|
||||
assert!(
|
||||
tls_len == base_tls_len || tls_len >= base_tls_len + 4,
|
||||
"TLS length must leave room for a complete padding extension"
|
||||
);
|
||||
if tls_len > base_tls_len {
|
||||
let padding_len = tls_len - base_tls_len - 4;
|
||||
extensions.extend_from_slice(&TLS_EXTENSION_PADDING.to_be_bytes());
|
||||
extensions.extend_from_slice(&(padding_len as u16).to_be_bytes());
|
||||
extensions.resize(extensions.len() + padding_len, fill);
|
||||
}
|
||||
|
||||
let body_len = tls_len - 4;
|
||||
let mut body = Vec::with_capacity(body_len);
|
||||
body.extend_from_slice(&TLS_VERSION);
|
||||
body.extend_from_slice(&[fill; 32]);
|
||||
body.push(session_id_len as u8);
|
||||
body.extend_from_slice(&[fill; 32]);
|
||||
body.extend_from_slice(&(TLS_AES_128_GCM_SHA256.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&TLS_AES_128_GCM_SHA256);
|
||||
body.push(1);
|
||||
body.push(0);
|
||||
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||
body.extend_from_slice(&extensions);
|
||||
assert_eq!(body.len(), body_len);
|
||||
|
||||
let mut handshake = Vec::with_capacity(5 + tls_len);
|
||||
handshake.push(0x16);
|
||||
handshake.extend_from_slice(&[0x03, 0x01]);
|
||||
handshake.extend_from_slice(&(tls_len as u16).to_be_bytes());
|
||||
handshake.push(0x01);
|
||||
let body_len_bytes = (body_len as u32).to_be_bytes();
|
||||
handshake.extend_from_slice(&body_len_bytes[1..4]);
|
||||
handshake.extend_from_slice(&body);
|
||||
|
||||
// The proxy authenticates TLS-fronted clients through the random field.
|
||||
handshake[tls::TLS_DIGEST_POS..tls::TLS_DIGEST_POS + tls::TLS_DIGEST_LEN].fill(0);
|
||||
let computed = sha256_hmac(secret, &handshake);
|
||||
let mut digest = computed;
|
||||
@@ -172,13 +228,11 @@ async fn run_tls_success_mtproto_fail_capture(
|
||||
assert_eq!(tls_response_head[0], 0x16);
|
||||
read_tls_record_body(&mut client_side, tls_response_head).await;
|
||||
|
||||
client_side
|
||||
.write_all(&invalid_mtproto_record)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut client_payload = invalid_mtproto_record;
|
||||
for record in trailing_records {
|
||||
client_side.write_all(&record).await.unwrap();
|
||||
client_payload.extend_from_slice(&record);
|
||||
}
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
|
||||
let got = tokio::time::timeout(Duration::from_secs(3), accept_task)
|
||||
.await
|
||||
@@ -343,11 +397,9 @@ async fn replayed_tls_hello_gets_no_serverhello_and_is_masked() {
|
||||
client_side.read_exact(&mut head).await.unwrap();
|
||||
assert_eq!(head[0], 0x16);
|
||||
read_tls_record_body(&mut client_side, head).await;
|
||||
client_side
|
||||
.write_all(&invalid_mtproto_record)
|
||||
.await
|
||||
.unwrap();
|
||||
client_side.write_all(&first_tail).await.unwrap();
|
||||
let mut client_payload = invalid_mtproto_record;
|
||||
client_payload.extend_from_slice(&first_tail);
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
} else {
|
||||
let mut one = [0u8; 1];
|
||||
let no_server_hello = tokio::time::timeout(
|
||||
@@ -418,11 +470,9 @@ async fn connects_bad_increments_once_per_invalid_mtproto() {
|
||||
let mut head = [0u8; 5];
|
||||
client_side.read_exact(&mut head).await.unwrap();
|
||||
read_tls_record_body(&mut client_side, head).await;
|
||||
client_side
|
||||
.write_all(&invalid_mtproto_record)
|
||||
.await
|
||||
.unwrap();
|
||||
client_side.write_all(&tail).await.unwrap();
|
||||
let mut client_payload = invalid_mtproto_record;
|
||||
client_payload.extend_from_slice(&tail);
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(3), accept_task)
|
||||
.await
|
||||
@@ -675,8 +725,9 @@ async fn concurrent_tls_mtproto_fail_sessions_are_isolated() {
|
||||
let mut head = [0u8; 5];
|
||||
client_side.read_exact(&mut head).await.unwrap();
|
||||
read_tls_record_body(&mut client_side, head).await;
|
||||
client_side.write_all(&invalid_mtproto).await.unwrap();
|
||||
client_side.write_all(&trailing).await.unwrap();
|
||||
let mut client_payload = invalid_mtproto;
|
||||
client_payload.extend_from_slice(&trailing);
|
||||
client_side.write_all(&client_payload).await.unwrap();
|
||||
client_side.shutdown().await.unwrap();
|
||||
|
||||
let _ = tokio::time::timeout(Duration::from_secs(3), handler)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user