Compare commits

...

133 Commits
3.4.2 ... main

Author SHA1 Message Date
Alexey
01b0c5c6ce Merge pull request #786 from Dimasssss/patch-1
Update install.sh
2026-05-16 14:12:14 +03:00
Dimasssss
ad1bb5cc1a Update install.sh 2026-05-15 01:32:37 +03:00
Dimasssss
08cde1a255 Update install.sh 2026-05-15 01:29:13 +03:00
Dimasssss
faf1f28f9d Update install.sh 2026-05-15 01:23:45 +03:00
Dimasssss
32613c8e68 Update install.sh 2026-05-15 01:12:47 +03:00
Alexey
1fe621f743 Update CONFIG_PARAMS.en.md 2026-05-10 17:37:41 +03:00
Alexey
3b0ebf3c9e Update CONFIG_PARAMS.ru.md 2026-05-10 17:37:31 +03:00
Alexey
b41f6bc21e Update CONFIG_PARAMS.en.md 2026-05-10 17:37:15 +03:00
Alexey
0a9f599611 Update CONFIG_PARAMS.en.md 2026-05-10 17:37:03 +03:00
Alexey
cdb021fc71 Update CONFIG_PARAMS.ru.md 2026-05-10 17:22:39 +03:00
Alexey
6b61183b9d Update CONFIG_PARAMS.en.md 2026-05-10 17:22:21 +03:00
Alexey
7a284623d6 Update API.md 2026-05-10 15:38:27 +03:00
Alexey
3bd5637e47 ME + Admission + Cleanup Correctness: merge pull request #779 from telemt/flow
ME + Admission + Cleanup Correctness
2026-05-10 14:23:09 +03:00
Alexey
57b2aa0453 Rustfmt 2026-05-10 14:14:52 +03:00
Alexey
10c7cb2e0c Middle Relay Cancellation Errors 2026-05-10 14:12:15 +03:00
Alexey
900b574fb8 Harden ME Writer Cancellation paths 2026-05-10 14:09:10 +03:00
Alexey
beed6b4679 Middle Wait Deadlines + Tighten Session Release State 2026-05-10 13:58:02 +03:00
Alexey
eef2a38c75 Type Route Cutovers + Reduce IP Tracker cleanup pressure 2026-05-10 13:55:01 +03:00
Alexey
6cb72b3b6c Explicit Reasons of Session Fallback Cleanup + ME Close 2026-05-10 13:50:36 +03:00
Alexey
090b2ca636 Stats and Cleanup-proccess beyond Hot-path 2026-05-10 13:43:41 +03:00
Alexey
e10c070dc1 Observability + Cancellation for Middle Quota + Traffic Waits 2026-05-10 13:38:11 +03:00
Alexey
3f9ac87daf Bounded Rate Bursts + Cancel ME Waits 2026-05-10 13:33:54 +03:00
Alexey
36de807096 Merge branch 'flow' of https://github.com/telemt/telemt into flow 2026-05-10 13:31:11 +03:00
Alexey
844a912b38 Expose Quota Contention + Cleanup fallback metrics 2026-05-10 13:30:59 +03:00
Alexey
5c5a3fae06 Update AGENTS.md 2026-05-10 13:29:02 +03:00
Alexey
ba1d9be5d4 Hardened Relays and API Security paths 2026-05-10 13:22:54 +03:00
Alexey
b2aa9b8c9e Hardened API & Management-plane Admission
- bound API and metrics connection handling
- default metrics listener to localhost
- reject untrusted PROXY protocol peers before parsing headers
- cap API request body size and PROXY v2 payload allocation
- validate route usernames and TLS domains consistently
2026-05-09 20:50:23 +03:00
Alexey
73c82bda7a Update AGENTS.md 2026-05-09 16:34:54 +03:00
Alexey
b3510aa8b8 Bound HTTP API+Metrics Connection Admission 2026-05-09 16:29:30 +03:00
Alexey
dd3c5eff1c Merge pull request #776 from pavlozt/fix/sudo_path
Fix(installer): workaround for systems with secure PATH
2026-05-09 13:09:25 +03:00
PavelZ
dd4fb71959 Fix(installer): workaround for systems with secure PATH 2026-05-08 17:31:50 +03:00
Alexey
f0f2bc0482 Limit&Quota Saving as File + API 2026-05-08 14:38:24 +03:00
Alexey
86573be493 Event-driven Wakeup for ME Admission-gate 2026-05-08 13:34:41 +03:00
Alexey
658a565cb3 Merge pull request #770 from konstpic/feat/user-source-deny-list
feat(access): add per-user source IP deny list checks
2026-05-07 11:56:54 +03:00
Alexey
29fabcb199 Merge pull request #772 from agrofx1/user_check
Add root switch or login check
2026-05-07 11:53:50 +03:00
Alexey
efdf3bcc1b Fix root detection by checking UID 2026-05-07 11:53:29 +03:00
Agrofx
66c37ad6fd Merge branch 'flow' into user_check 2026-05-07 10:30:57 +03:00
Agrofx
0fcf67ca34 Update install.sh
Co-authored-by: Dimasssss <Dimasssss2000@gmail.com>
2026-05-07 10:30:47 +03:00
Agrofx
df14762a12 Add root switch or login check 2026-05-07 06:27:51 +03:00
Alexey
4995e83236 Config Strict and Validator 2026-05-06 20:38:55 +03:00
Alexey
e0f251ad82 TLS Domains masking fixes 2026-05-06 20:29:24 +03:00
Konstantin Pichugin
b605b1ba7c docs(access): document user_source_deny usage and API path
Add config examples and behavior notes for access.user_source_deny, and clarify that it is configured through config.toml rather than dedicated user API request fields.
2026-05-06 19:17:06 +03:00
Konstantin Pichugin
b859fb95c3 feat(access): add per-user source IP deny list checks
Add access.user_source_deny and enforce it in TLS and MTProto handshake paths after successful authentication to fail closed for blocked source IPs.
2026-05-06 19:11:18 +03:00
Alexey
8c303ab2b6 Merge pull request #765 from Misha20062006/patch-2
Correct saving instructions in QUICK_START_GUIDE.ru.md
2026-05-06 17:13:49 +03:00
Misha20062006
f70c2936c7 Correct saving instructions in QUICK_START_GUIDE.ru.md
Updated instructions for saving changes in the guide.
2026-05-06 00:07:14 +03:00
Alexey
d67c37afd7 Merge pull request #762 from astronaut808/feature/tls-front-profile-health
Add TLS Front Profile Health metrics
2026-05-05 15:23:01 +03:00
astronaut808
9f9ca9f270 Add TLS front profile health metrics 2026-05-03 18:07:24 +05:00
Alexey
cdd2239047 Merge pull request #758 from mammuthus/feature/metrics-bad-class-export-dashboard
Add class-based error metrics and dashboard panels
2026-05-02 00:46:53 +03:00
Alexey
9ee341a94f Merge pull request #757 from Dimasssss/docs
Update CONFIG_PARAMS
2026-05-02 00:36:46 +03:00
mamuthus
a7a2f4ab27 Adjust General metrics dashboard layout 2026-05-01 19:19:00 +00:00
mamuthus
9dae14aa66 Add class-based error metrics and dashboard panels 2026-05-01 18:26:32 +00:00
Dimasssss
f76c847c44 Update CONFIG_PARAMS.en.md 2026-05-01 21:10:34 +03:00
Dimasssss
1aaa9c0bc6 Update CONFIG_PARAMS.ru.md 2026-05-01 21:09:38 +03:00
Alexey
e50026e776 Update README.md 2026-04-30 19:41:40 +03:00
Alexey
7106f38fae Update Cargo.lock 2026-04-30 11:38:33 +03:00
Alexey
2a694470d5 Update Cargo.toml 2026-04-30 11:37:18 +03:00
Alexey
b98cd37211 TLS Full Certificate Budget Bookkeeping + Hot-path Cleanup and Timeout Invariants + IP-Tracker refactoring + Shard TLS Full-Cert Budget: merge pull request #753 from telemt/flow
TLS Full Certificate Budget Bookkeeping + Hot-path Cleanup and Timeout Invariants + IP-Tracker refactoring + Shard TLS Full-Cert Budget
2026-04-30 11:36:30 +03:00
Alexey
8b62965978 Stabilize unknown-DC symlink race test setup 2026-04-30 11:11:04 +03:00
Alexey
d46bda9880 Preserve synchronous IP cleanup queue contract + Rustfmt 2026-04-30 11:05:18 +03:00
Alexey
c3de07db6a Shard TLS full-cert budget tracking + Bound user-labeled metrics export cardinality 2026-04-30 11:01:10 +03:00
Alexey
61f9af7ffc Reduce Lock-free IP-Tracker Cleanup backlog 2026-04-30 10:51:04 +03:00
Alexey
1f90e28871 Cap scanner-sensitive Caches and IP-Tracker Cardinality 2026-04-30 10:43:27 +03:00
Alexey
876b74ebf7 Hot-path Cleanup and Timeout Invariants 2026-04-29 23:16:11 +03:00
Alexey
b34e1d71ae TLS Full Certificate Budget Bookkeeping 2026-04-29 23:00:25 +03:00
Alexey
b1c947e8e3 Limit only new ip when TimeWindow + Fix WorkingDirectory behavior + Atomically updates with Includes + Expose tls_domains links as domain-link pairs + TLS Fetcher on multiple tls_domains: merge pull request #751 from telemt/flow
Limit only new ip when TimeWindow + Fix WorkingDirectory behavior + Atomically updates with Includes + Expose tls_domains links as domain-link pairs + TLS Fetcher on multiple tls_domains
2026-04-29 16:04:36 +03:00
Alexey
cfe01dced2 Bump
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 15:54:22 +03:00
Alexey
8520955a5f Update helpers.rs
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 15:53:27 +03:00
Alexey
065786b839 TLS Fetcher on multiple tls_domains by #750
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 11:47:42 +03:00
Alexey
f0e1a6cf1c Expose tls_domains links as domain-link pairs
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 11:34:47 +03:00
Alexey
236bbb4970 Atomically updates with Includes
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-28 13:00:13 +03:00
Alexey
8ef5263fce Fix WorkingDirectory behavior
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
Co-Authored-By: mikhailnov <m@mikhailnov.ru>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-28 12:31:21 +03:00
Alexey
893cef22e3 Update README.md 2026-04-27 23:49:47 +03:00
Alexey
bdfa641843 Merge pull request #735 from sanekb/fix_timewindow_same_ip
fix: limit only new ip when TimeWindow mode enabled
2026-04-25 19:08:36 +03:00
Alexey
007fc86189 Merge branch 'flow' into fix_timewindow_same_ip 2026-04-25 18:56:27 +03:00
Alexey
10c9bcd97d Merge pull request #747 from telemt/flow
Restore active IP observability for users without unique-IP limits
2026-04-25 18:11:30 +03:00
Alexey
8ab9405dca Bump 2026-04-25 18:05:22 +03:00
Alexey
9412f089c0 Restore active IP observability for users without unique-IP limits 2026-04-25 15:49:28 +03:00
Alexey
4e57cee9b9 Merge pull request #745 from telemt/flow
API PATCH fixes + No IP tracking with disabled unique-IP limits + Bound hot-path pressure in ME Relay and Handshake + Bounded ME Route fairness and IP-Cleanup-Backlog + Bound relay queues by bytes
2026-04-25 14:45:34 +03:00
Alexey
e217371dc8 Bump 2026-04-25 14:36:51 +03:00
sanekb
d567dfe40b fix: limit only new ip when TimeWindow mode enabled 2026-04-25 14:36:43 +03:00
Alexey
37c916056a Rustfmt 2026-04-25 14:35:35 +03:00
Alexey
2f2fe9d5d3 Bound relay queues by bytes
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-25 13:54:20 +03:00
Alexey
1df668144c Bounded ME Route fairness and IP-Cleanup-Backlog
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-25 13:09:10 +03:00
Alexey
8494429690 Merge pull request #743 from amirotin/api/patch-user-null-removal
feat(api): support null-removal in PATCH /v1/users/{user}
2026-04-25 13:07:13 +03:00
Alexey
f25bb17b86 Merge branch 'flow' into api/patch-user-null-removal 2026-04-25 12:28:48 +03:00
Alexey
27b5d576c0 Bound hot-path pressure in ME Relay + Handshake
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-25 12:16:26 +03:00
Alexey
e78592ef9b Avoid IP tracking when unique-IP limits are disabled and cap beobachten memory
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-25 12:00:46 +03:00
Mirotin Artem
4ed87d1946 feat(api): support null-removal in PATCH /v1/users/{user}
PatchUserRequest now uses Patch<T> for the five removable fields
(user_ad_tag, max_tcp_conns, expiration_rfc3339, data_quota_bytes,
max_unique_ips). Sending JSON null drops the entry from the
corresponding access HashMap; sending 0 is preserved as a literal
limit; omitted fields stay untouched. The handler synchronises the
in-memory ip_tracker on both set and remove of max_unique_ips. A
helper parse_patch_expiration mirrors parse_optional_expiration for
the new three-state field. Runtime semantics are unchanged.
2026-04-25 00:49:34 +03:00
Mirotin Artem
635bea4de4 feat(api): add Patch<T> enum for JSON merge-patch semantics
Introduce a three-state Patch<T> (Unchanged / Remove / Set) and a
serde helper patch_field that distinguishes an omitted JSON field
from an explicit null. Wired up next as the field type for the
removable settings on PATCH /v1/users/{user}.
2026-04-25 00:49:34 +03:00
Alexey
8874396ba5 Merge pull request #739 from telemt/flow-test
Relays Tests Fixes
2026-04-24 15:51:47 +03:00
Alexey
033ebf5038 Relays Tests Fixes 2026-04-24 15:51:19 +03:00
Alexey
f7b918875c Close Errors Classification + TLS 1.2/1.3 Correctness in Fronting + Full ServerHello + ALPN in TLS Fetcher: merge pull request #738 from telemt/flow
Close Errors Classification + TLS 1.2/1.3 Correctness in Fronting + Full ServerHello + ALPN in TLS Fetcher
2026-04-24 15:48:39 +03:00
Alexey
8960fad8cd Сlassified Bad Connections and Handshake Failures in API
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-24 10:56:30 +03:00
Alexey
493f5c9680 ALPN in TLS Fetcher
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-23 22:22:05 +03:00
Alexey
67357310f7 TLS 1.2/1.3 Correctness + Full ServerHello + Rustfmt
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-23 21:29:18 +03:00
Alexey
8684378030 Human-readable Peer Close Classification 2026-04-21 15:46:18 +03:00
Alexey
db8d333ed6 Noisy-network peer Close Errors Classification 2026-04-21 15:35:11 +03:00
Alexey
30e73adaac Bump 2026-04-21 13:38:38 +03:00
Alexey
351f2c8458 Fairness Regression fixes + Unlimited mask_relay_max_bytes: merge pull request #726 from telemt/flow
Fairness Regression fixes + Unlimited mask_relay_max_bytes
2026-04-21 13:37:10 +03:00
Alexey
4ce6b14bd8 Rustfmt 2026-04-21 13:31:24 +03:00
Alexey
db114f09c3 Sync tests with code 2026-04-21 13:30:11 +03:00
Alexey
09310ff284 Unlimited mask_relay_max_bytes 2026-04-21 11:30:58 +03:00
Alexey
1e5b84c0ed Fairshare Disabled semantics fix 2026-04-21 11:21:58 +03:00
Alexey
926e3aa987 Fairness Regression fixes 2026-04-21 01:11:43 +03:00
Alexey
aace0129f8 Path for getProxyConfig/Secret + Active Ring and DRR Hardening + Weighted Fairness + 3-Leveled Pressure Model + Improve ME downstream retries + SNI handling: merge pull request #723 from telemt/flow
Path for getProxyConfig/Secret + Active Ring and DRR Hardening + Weighted Fairness + 3-Leveled Pressure Model + Improve ME downstream retries +  SNI handling
2026-04-19 19:12:10 +03:00
Alexey
2a7303c129 Bump 2026-04-19 19:10:19 +03:00
Alexey
9cb49bc024 Fix in Fairness tests 2026-04-19 19:03:45 +03:00
Alexey
8092283e8f Merge pull request #721 from lie-must-die/feat/unknown-sni-reject-handshake
Feat/unknown sni reject handshake
2026-04-19 14:19:22 +03:00
lie-must-die
132841da61 Update FAQ with SNI handling and metrics instructions
Added alternative configuration for unknown SNI handling and instructions for viewing metrics.
2026-04-19 12:50:26 +03:00
lie-must-die
372d288806 Добавить альтернативу для unknown_sni_action
Добавлена альтернатива для поведения telemt на неизвестный SNI.
2026-04-19 12:49:54 +03:00
lie-must-die
1c44d45fad Add 'reject_handshake' option to unknown_sni_action
Updated the `unknown_sni_action` parameter to include `reject_handshake` as a valid option. Expanded the description for `unknown_sni_action` to clarify its behavior.
2026-04-19 12:48:43 +03:00
lie-must-die
3a51a8d9aa Revise CONFIG_PARAMS.ru.md for clarity and detail
Updated descriptions and validation rules for various parameters in the Russian configuration documentation.
2026-04-19 12:47:26 +03:00
lie-must-die
dd27206104 Implement test for unknown SNI reject policy
Add test for unknown SNI rejection policy emitting TLS alert.
2026-04-19 12:44:39 +03:00
lie-must-die
f11c7880e6 Enhance unknown SNI action handling in handshake
Updated handling of unknown SNI actions in TLS handshake process. Added support for RejectHandshake action and adjusted delay application logic.
2026-04-19 12:43:54 +03:00
lie-must-die
5b07ffae7c Implement test for unknown_sni_action in ProxyConfig
Added test case for unknown_sni_action configuration.
2026-04-19 12:42:52 +03:00
lie-must-die
7bbed133ee Add RejectHandshake variant for TLS configuration
Added a new variant 'RejectHandshake' to handle TLS handshake rejection with a specific alert.
2026-04-19 12:40:10 +03:00
Alexey
f1bf95a7de Merge pull request #718 from astronaut808/fix/me-downstream-retry
Improve ME downstream retries for queued fairness backlog
2026-04-18 14:03:37 +03:00
Alexey
959a16af88 Merge pull request #716 from zarv1k/feature/configurable-proxy-confi-urls
feat: make URLS to obtain proxy_secret, getProxyConfig, getProxyConfgV6 files optionally configurable
2026-04-18 11:17:37 +03:00
Alexey
a54f9ba719 Merge branch 'flow' into feature/configurable-proxy-confi-urls 2026-04-18 11:16:38 +03:00
astronaut808
2d5cd9c8e1 Improve ME downstream retries for queued fairness backlog 2026-04-18 02:40:32 +05:00
Alexey
37b6f7b985 Weighted Fairness + 3-Leveled Pressure Model
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-18 00:37:04 +03:00
Alexey
50e9e5cf32 Active Ring and DRR Hardening
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
2026-04-18 00:34:35 +03:00
Alexey
d72cfd6bc4 Merge branch 'flow' into feature/configurable-proxy-confi-urls 2026-04-17 19:44:46 +03:00
Alexey
1b25bada29 ServerHello fixes + Docker Health-Check + Conntrack Control for Docker: merge pull request #717 from telemt/flow
ServerHello fixes + Docker Health-Check + Conntrack Control for Docker
2026-04-17 19:43:59 +03:00
Dmitry Zarva
fa3566a9cb - fix: fmt issues 2026-04-17 16:20:16 +00:00
Alexey
bde30eaf05 Update emulator.rs 2026-04-17 19:20:06 +03:00
Alexey
b447f60a72 Rustfmt + Bump 2026-04-17 19:08:57 +03:00
Alexey
093faed0c2 Conntrack Control for Docker 2026-04-17 19:06:18 +03:00
Dmitry Zarva
4e59e52454 - fix: ru docs 2026-04-17 14:10:20 +00:00
Alexey
3ca3e8ff0e Docker Health-Check 2026-04-17 16:36:15 +03:00
Dmitry Zarva
7b9b46291d - fix: param name in ru docs 2026-04-17 13:19:29 +00:00
Dmitry Zarva
2a168b2600 feat: make URLS to obtain proxy_secret, getProxyConfig, getProxyConfigV6 files optionally configurable 2026-04-17 13:04:46 +00:00
Alexey
6e3b4a1ce5 ServerHello fixes 2026-04-17 15:11:36 +03:00
94 changed files with 15462 additions and 6691 deletions

View File

@@ -191,6 +191,11 @@ When facing a non-trivial modification, follow this sequence:
4. **Implement**: Make the minimal, isolated change.
5. **Verify**: Explain why the change preserves existing behavior and architectural integrity.
When the repository contains a `PLAN.md` for the current task, maintain it as
a working checkbox plan while implementing changes. Mark completed and partial
items in `PLAN.md` as the code changes land, so the remaining work stays
explicit and future passes do not waste time rediscovering status.
---
### 9. Context Awareness
@@ -222,10 +227,9 @@ Your response MUST consist of two sections:
**Section 2: `## Changes`**
- For each modified or created file: the filename on a separate line in backticks, followed by the code block.
- For files **under 200 lines**: return the full file with all changes applied.
- For files **over 200 lines**: return only the changed functions/blocks with at least 3 lines of surrounding context above and below. If the user requests the full file, provide it.
- New files: full file content.
- For each modified or created file: the filename on a separate line in backticks, followed by a concise description of what changed.
- Do not include full file contents or long code blocks in `## Changes` unless the user explicitly asks for code text.
- If code snippets are necessary, include only the minimal relevant excerpt.
- End with a suggested git commit message in English.
#### Reporting Out-of-Scope Issues
@@ -429,4 +433,3 @@ Every patch must be **atomic and production-safe**.
* **No transitional states** — no placeholders, incomplete refactors, or temporary inconsistencies.
**Invariant:** After any single patch, the repository remains fully functional and buildable.

206
Cargo.lock generated
View File

@@ -90,9 +90,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "arc-swap"
version = "1.9.0"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
dependencies = [
"rustversion",
]
@@ -173,9 +173,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-lc-rs"
version = "1.16.2"
version = "1.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -183,9 +183,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.39.1"
version = "0.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
dependencies = [
"cc",
"cmake",
@@ -228,9 +228,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.11.0"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "blake3"
@@ -299,9 +299,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.58"
version = "1.2.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1"
checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -346,7 +346,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
"rand_core 0.10.0",
"rand_core 0.10.1",
]
[[package]]
@@ -416,9 +416,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.6.0"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
dependencies = [
"clap_builder",
]
@@ -805,9 +805,9 @@ dependencies = [
[[package]]
name = "fastrand"
version = "2.3.0"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "fiat-crypto"
@@ -997,7 +997,7 @@ dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
"rand_core 0.10.0",
"rand_core 0.10.1",
"wasip2",
"wasip3",
]
@@ -1068,6 +1068,12 @@ dependencies = [
"foldhash 0.2.0",
]
[[package]]
name = "hashbrown"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51"
[[package]]
name = "heck"
version = "0.5.0"
@@ -1096,7 +1102,7 @@ dependencies = [
"idna",
"ipnet",
"once_cell",
"rand 0.9.2",
"rand 0.9.4",
"ring",
"thiserror 2.0.18",
"tinyvec",
@@ -1118,7 +1124,7 @@ dependencies = [
"moka",
"once_cell",
"parking_lot",
"rand 0.9.2",
"rand 0.9.4",
"resolv-conf",
"smallvec",
"thiserror 2.0.18",
@@ -1213,15 +1219,14 @@ dependencies = [
[[package]]
name = "hyper-rustls"
version = "0.27.7"
version = "0.27.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
"http",
"hyper",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
@@ -1385,12 +1390,12 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.13.0"
version = "2.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
"hashbrown 0.17.0",
"serde",
"serde_core",
]
@@ -1401,7 +1406,7 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd5b3eaf1a28b758ac0faa5a4254e8ab2705605496f1b1f3fbbc3988ad73d199"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"inotify-sys",
"libc",
]
@@ -1534,9 +1539,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.94"
version = "0.3.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9"
checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca"
dependencies = [
"cfg-if",
"futures-util",
@@ -1578,9 +1583,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "libc"
version = "0.2.184"
version = "0.2.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
[[package]]
name = "linux-raw-sys"
@@ -1611,9 +1616,9 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lru"
version = "0.16.3"
version = "0.16.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
dependencies = [
"hashbrown 0.16.1",
]
@@ -1705,7 +1710,7 @@ version = "0.31.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"cfg-if",
"cfg_aliases",
"libc",
@@ -1728,7 +1733,7 @@ version = "8.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"fsevent-sys",
"inotify",
"kqueue",
@@ -1746,7 +1751,7 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
]
[[package]]
@@ -2012,9 +2017,9 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
dependencies = [
"bit-set",
"bit-vec",
"bitflags 2.11.0",
"bitflags 2.11.1",
"num-traits",
"rand 0.9.2",
"rand 0.9.4",
"rand_chacha",
"rand_xorshift",
"regex-syntax",
@@ -2059,7 +2064,7 @@ dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
"rand 0.9.2",
"rand 0.9.4",
"ring",
"rustc-hash",
"rustls",
@@ -2108,9 +2113,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
version = "0.9.2"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
dependencies = [
"rand_chacha",
"rand_core 0.9.5",
@@ -2118,13 +2123,13 @@ dependencies = [
[[package]]
name = "rand"
version = "0.10.0"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8"
checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
dependencies = [
"chacha20 0.10.0",
"getrandom 0.4.2",
"rand_core 0.10.0",
"rand_core 0.10.1",
]
[[package]]
@@ -2157,9 +2162,9 @@ dependencies = [
[[package]]
name = "rand_core"
version = "0.10.0"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rand_xorshift"
@@ -2172,9 +2177,9 @@ dependencies = [
[[package]]
name = "rayon"
version = "1.11.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
@@ -2196,7 +2201,7 @@ version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
]
[[package]]
@@ -2326,7 +2331,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"errno",
"libc",
"linux-raw-sys",
@@ -2335,9 +2340,9 @@ dependencies = [
[[package]]
name = "rustls"
version = "0.23.37"
version = "0.23.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -2399,9 +2404,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.10"
version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
@@ -2474,7 +2479,7 @@ version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"core-foundation",
"core-foundation-sys",
"libc",
@@ -2493,9 +2498,9 @@ dependencies = [
[[package]]
name = "semver"
version = "1.0.27"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "sendfd"
@@ -2615,7 +2620,7 @@ dependencies = [
"notify",
"percent-encoding",
"pin-project",
"rand 0.9.2",
"rand 0.9.4",
"sealed",
"sendfd",
"serde",
@@ -2646,7 +2651,7 @@ dependencies = [
"chacha20poly1305",
"hkdf",
"md-5",
"rand 0.9.2",
"rand 0.9.4",
"ring-compat",
"sha1",
]
@@ -2741,6 +2746,12 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symlink"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
[[package]]
name = "syn"
version = "2.0.117"
@@ -2780,7 +2791,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "telemt"
version = "3.4.2"
version = "3.4.11"
dependencies = [
"aes",
"anyhow",
@@ -2812,7 +2823,7 @@ dependencies = [
"num-traits",
"parking_lot",
"proptest",
"rand 0.10.0",
"rand 0.10.1",
"regex",
"reqwest",
"rustls",
@@ -2970,9 +2981,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.50.0"
version = "1.52.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6"
dependencies = [
"bytes",
"libc",
@@ -2988,9 +2999,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
version = "2.6.1"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
@@ -3123,7 +3134,7 @@ version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"bytes",
"futures-util",
"http",
@@ -3160,11 +3171,12 @@ dependencies = [
[[package]]
name = "tracing-appender"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf"
checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
dependencies = [
"crossbeam-channel",
"symlink",
"thiserror 2.0.18",
"time",
"tracing-subscriber",
@@ -3239,9 +3251,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "typenum"
version = "1.19.0"
version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
[[package]]
name = "unarray"
@@ -3297,9 +3309,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
version = "1.23.0"
version = "1.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
dependencies = [
"getrandom 0.4.2",
"js-sys",
@@ -3354,11 +3366,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasip2"
version = "1.0.2+wasi-0.2.9"
version = "1.0.3+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
dependencies = [
"wit-bindgen",
"wit-bindgen 0.57.1",
]
[[package]]
@@ -3367,14 +3379,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen",
"wit-bindgen 0.51.0",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.117"
version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0"
checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89"
dependencies = [
"cfg-if",
"once_cell",
@@ -3385,9 +3397,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.67"
version = "0.4.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e"
checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -3395,9 +3407,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.117"
version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be"
checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -3405,9 +3417,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.117"
version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2"
checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -3418,9 +3430,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.117"
version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b"
checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129"
dependencies = [
"unicode-ident",
]
@@ -3453,7 +3465,7 @@ version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags 2.11.0",
"bitflags 2.11.1",
"hashbrown 0.15.5",
"indexmap",
"semver",
@@ -3461,9 +3473,9 @@ dependencies = [
[[package]]
name = "web-sys"
version = "0.3.94"
version = "0.3.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a"
checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -3481,18 +3493,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.6"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca"
checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.6"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
dependencies = [
"rustls-pki-types",
]
@@ -3841,6 +3853,12 @@ dependencies = [
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen"
version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
@@ -3890,7 +3908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags 2.11.0",
"bitflags 2.11.1",
"indexmap",
"log",
"serde",
@@ -3922,9 +3940,9 @@ dependencies = [
[[package]]
name = "writeable"
version = "0.6.2"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
[[package]]
name = "x25519-dalek"

View File

@@ -1,6 +1,6 @@
[package]
name = "telemt"
version = "3.4.2"
version = "3.4.11"
edition = "2024"
[features]

View File

@@ -77,6 +77,34 @@ COPY config.toml /app/config.toml
EXPOSE 443 9090 9091
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
ENTRYPOINT ["/app/telemt"]
CMD ["config.toml"]
# ==========================
# Production Netfilter Profile
# ==========================
FROM debian:12-slim AS prod-netfilter
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends \
ca-certificates \
conntrack \
nftables \
iptables; \
rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=minimal /telemt /app/telemt
COPY config.toml /app/config.toml
EXPOSE 443 9090 9091
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
ENTRYPOINT ["/app/telemt"]
CMD ["config.toml"]
@@ -94,5 +122,7 @@ USER nonroot:nonroot
EXPOSE 443 9090 9091
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/app/telemt", "healthcheck", "/app/config.toml", "--mode", "liveness"]
ENTRYPOINT ["/app/telemt"]
CMD ["config.toml"]

View File

@@ -1,6 +1,6 @@
# Telemt - MTProxy on Rust + Tokio
[![Latest Release](https://img.shields.io/github/v/release/telemt/telemt?color=neon)](https://github.com/telemt/telemt/releases/latest) [![Stars](https://img.shields.io/github/stars/telemt/telemt?style=social)](https://github.com/telemt/telemt/stargazers) [![Forks](https://img.shields.io/github/forks/telemt/telemt?style=social)](https://github.com/telemt/telemt/network/members) [![Telegram](https://img.shields.io/badge/Telegram-Chat-24a1de?logo=telegram&logoColor=24a1de)](https://t.me/telemtrs)
[![Latest Release](https://img.shields.io/github/v/release/telemt/telemt?color=neon)](https://github.com/telemt/telemt/releases/latest) [![Stars](https://img.shields.io/github/stars/telemt/telemt?style=social)](https://github.com/telemt/telemt/stargazers) [![Forks](https://img.shields.io/github/forks/telemt/telemt?style=social)](https://github.com/telemt/telemt/network/members)
[🇷🇺 README на русском](https://github.com/telemt/telemt/blob/main/README.ru.md)
@@ -14,7 +14,7 @@
<p align="center">
<a href="https://t.me/telemtrs">
<img src="/docs/assets/telegram_button.svg" width="150"/>
<img src="https://github.com/user-attachments/assets/30b7e7b9-974a-4e3d-aab6-b58a85de4507" width="240"/>
</a>
</p>
@@ -99,7 +99,7 @@ Monero (XMR) directly:
8Bk4tZEYPQWSypeD2hrUXG2rKbAKF16GqEN942ZdAP5cFdSqW6h4DwkP5cJMAdszzuPeHeHZPTyjWWFwzeFdjuci3ktfMoB
```
All donations go toward infrastructure, development, and research.
All donations go toward infrastructure, development and research
![telemt_scheme](docs/assets/telemt.png)

View File

@@ -0,0 +1,10 @@
services:
telemt:
build:
context: .
target: prod-netfilter
network_mode: host
ports: []
cap_add:
- NET_BIND_SERVICE
- NET_ADMIN

View File

@@ -0,0 +1,8 @@
services:
telemt:
build:
context: .
target: prod-netfilter
cap_add:
- NET_BIND_SERVICE
- NET_ADMIN

View File

@@ -1,7 +1,9 @@
services:
telemt:
image: ghcr.io/telemt/telemt:latest
build: .
build:
context: .
target: prod
container_name: telemt
restart: unless-stopped
ports:
@@ -16,13 +18,18 @@ services:
- /etc/telemt:rw,mode=1777,size=4m
environment:
- RUST_LOG=info
healthcheck:
test: [ "CMD", "/app/telemt", "healthcheck", "/etc/telemt/config.toml", "--mode", "liveness" ]
interval: 30s
timeout: 5s
retries: 3
start_period: 20s
# Uncomment this line if you want to use host network for IPv6, but bridge is default and usually better
# network_mode: host
cap_drop:
- ALL
cap_add:
- NET_BIND_SERVICE
- NET_ADMIN
read_only: true
security_opt:
- no-new-privileges:true

View File

@@ -13,7 +13,7 @@ API runtime is configured in `[server.api]`.
| `listen` | `string` (`IP:PORT`) | `0.0.0.0:9091` | API bind address. |
| `whitelist` | `CIDR[]` | `127.0.0.0/8` | Source IP allowlist. Empty list means allow all. |
| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. |
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be within `[1, 1048576]`. |
| `minimal_runtime_enabled` | `bool` | `true` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
@@ -26,7 +26,7 @@ API runtime is configured in `[server.api]`.
Runtime validation for API config:
- `server.api.listen` must be a valid `IP:PORT`.
- `server.api.request_body_limit_bytes` must be `> 0`.
- `server.api.request_body_limit_bytes` must be within `[1, 1048576]`.
- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`.
- `server.api.runtime_edge_cache_ttl_ms` must be within `[0, 60000]`.
- `server.api.runtime_edge_top_n` must be within `[1, 1000]`.
@@ -76,13 +76,14 @@ Requests are processed in this order:
Notes:
- Whitelist is evaluated against the direct TCP peer IP (`SocketAddr::ip`), without `X-Forwarded-For` support.
- `Authorization` check is exact string equality against configured `auth_header`.
- `Authorization` check is exact constant-time byte equality against configured `auth_header`.
## Endpoint Matrix
| Method | Path | Body | Success | `data` contract |
| --- | --- | --- | --- | --- |
| `GET` | `/v1/health` | none | `200` | `HealthData` |
| `GET` | `/v1/health/ready` | none | `200` or `503` | `HealthReadyData` |
| `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` |
| `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` |
| `GET` | `/v1/runtime/initialization` | none | `200` | `RuntimeInitializationData` |
@@ -102,13 +103,50 @@ Notes:
| `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` |
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
| `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` |
| `POST` | `/v1/users` | `CreateUserRequest` | `201` or `202` | `CreateUserResponse` |
| `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` |
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` | `UserInfo` |
| `DELETE` | `/v1/users/{username}` | none | `200` | `string` (deleted username) |
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `404` | `ErrorResponse` (`not_found`, current runtime behavior) |
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` or `202` | `UserInfo` |
| `DELETE` | `/v1/users/{username}` | none | `200` or `202` | `DeleteUserResponse` |
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` or `202` | `CreateUserResponse` |
| `POST` | `/v1/users/{username}/reset-quota` | empty body | `200` | `ResetUserQuotaResponse` |
## Endpoint Behavior
| Endpoint | Function |
| --- | --- |
| `GET /v1/health` | Returns basic API liveness and current `read_only` flag. |
| `GET /v1/health/ready` | Returns readiness based on admission state and upstream health; returns `503` when not ready. |
| `GET /v1/system/info` | Returns binary/build metadata, process uptime, config path/hash, and reload counters. |
| `GET /v1/runtime/gates` | Returns admission, ME readiness, fallback/reroute, and startup gate state. |
| `GET /v1/runtime/initialization` | Returns startup progress, ME initialization status, and per-component timeline. |
| `GET /v1/limits/effective` | Returns effective timeout, upstream, ME, unique-IP, and TCP policy values after config defaults/resolution. |
| `GET /v1/security/posture` | Returns current API/security/telemetry posture flags. |
| `GET /v1/security/whitelist` | Returns configured API whitelist CIDRs. |
| `GET /v1/stats/summary` | Returns compact core counters and classed failure counters. |
| `GET /v1/stats/zero/all` | Returns zero-cost core, upstream, ME, pool, and desync counters. |
| `GET /v1/stats/upstreams` | Returns upstream zero counters and, when enabled/available, runtime upstream health rows. |
| `GET /v1/stats/minimal/all` | Returns cached minimal ME writer/DC/runtime/network-path snapshot. |
| `GET /v1/stats/me-writers` | Returns cached ME writer coverage and per-writer status rows. |
| `GET /v1/stats/dcs` | Returns cached per-DC endpoint/writer/load status rows. |
| `GET /v1/runtime/me_pool_state` | Returns active/warm/pending/draining generation state, writer contour/health, and refill state. |
| `GET /v1/runtime/me_quality` | Returns ME lifecycle counters, route-drop counters, family states, drain gate, and per-DC RTT/coverage. |
| `GET /v1/runtime/upstream_quality` | Returns upstream policy/counters plus runtime upstream health rows when available. |
| `GET /v1/runtime/nat_stun` | Returns NAT/STUN runtime flags, configured/live STUN servers, reflection cache, and backoff. |
| `GET /v1/runtime/me-selftest` | Returns ME self-test state for KDF, time skew, IP family, PID, and SOCKS BND observations. |
| `GET /v1/runtime/connections/summary` | Returns runtime-edge connection totals and top-N users by connections/throughput. |
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
| `GET /v1/users` | Returns disk-first user views sorted by username. |
| `POST /v1/users` | Creates a user and returns the effective user view plus secret. |
| `GET /v1/users/{username}` | Returns one disk-first user view or `404` when absent. |
| `PATCH /v1/users/{username}` | Updates selected per-user fields with JSON Merge Patch semantics. |
| `DELETE /v1/users/{username}` | Deletes one user and related per-user access-map entries. |
| `POST /v1/users/{username}/rotate-secret` | Rotates one user's secret and returns the effective secret. |
| `POST /v1/users/{username}/reset-quota` | Resets one user's runtime quota counter and persists quota state. |
## Common Error Codes
@@ -118,7 +156,7 @@ Notes:
| `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. |
| `403` | `forbidden` | Source IP is not allowed by whitelist. |
| `403` | `read_only` | Mutating endpoint called while `read_only=true`. |
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route (including current `rotate-secret` route). |
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route. |
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
| `409` | `user_exists` | User already exists on create. |
@@ -132,11 +170,12 @@ Notes:
| Case | Behavior |
| --- | --- |
| Path matching | Exact match on `req.uri().path()`. Query string does not affect route matching. |
| Trailing slash | Not normalized. Example: `/v1/users/` is `404`. |
| Trailing slash | Trimmed for route matching when path length is greater than 1. Example: `/v1/users/` matches `/v1/users`. |
| Username route with extra slash | `/v1/users/{username}/...` is not treated as user route and returns `404`. |
| `PUT /v1/users/{username}` | `405 method_not_allowed`. |
| `POST /v1/users/{username}` | `404 not_found`. |
| `POST /v1/users/{username}/rotate-secret` | `404 not_found` in current release due route matcher limitation. |
| `POST /v1/users/{username}/rotate-secret/` | Trailing slash is trimmed and the route matches `rotate-secret`. |
| `POST /v1/users/{username}/reset-quota/` | Trailing slash is trimmed and the route matches `reset-quota`. |
## Body and JSON Semantics
@@ -146,7 +185,7 @@ Notes:
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
- `Content-Type` is not required for JSON parsing.
- Unknown JSON fields are ignored by deserialization.
- `PATCH` updates only provided fields and does not support explicit clearing of optional fields.
- `PATCH` uses JSON Merge Patch semantics for optional per-user fields: omitted means unchanged, explicit `null` removes the config entry, and a non-null value sets it.
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
## Query Parameters
@@ -172,18 +211,33 @@ Notes:
| Field | Type | Required | Description |
| --- | --- | --- | --- |
| `secret` | `string` | no | Exactly 32 hex chars. |
| `user_ad_tag` | `string` | no | Exactly 32 hex chars. |
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
| `user_ad_tag` | `string|null` | no | Exactly 32 hex chars; `null` removes the per-user ad tag. |
| `max_tcp_conns` | `usize|null` | no | Per-user concurrent TCP limit; `null` removes the per-user override. |
| `expiration_rfc3339` | `string|null` | no | RFC3339 expiration timestamp; `null` removes the expiration. |
| `data_quota_bytes` | `u64|null` | no | Per-user traffic quota; `null` removes the per-user quota. |
| `max_unique_ips` | `usize|null` | no | Per-user unique source IP limit; `null` removes the per-user override. |
### `access.user_source_deny` via API
- In current API surface, per-user deny-list is **not** exposed as a dedicated field in `CreateUserRequest` / `PatchUserRequest`.
- Configure it in `config.toml` under `[access.user_source_deny]` and apply via normal config reload path.
- Runtime behavior after apply:
- auth succeeds for username/secret
- source IP is checked against `access.user_source_deny[username]`
- on match, handshake is rejected with the same fail-closed outcome as invalid auth
Example config:
```toml
[access.user_source_deny]
alice = ["203.0.113.0/24", "2001:db8:abcd::/48"]
bob = ["198.51.100.42/32"]
```
### `RotateSecretRequest`
| Field | Type | Required | Description |
| --- | --- | --- | --- |
| `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. |
Note: the request contract is defined, but the corresponding route currently returns `404` (see routing edge cases).
An empty request body is accepted and generates a new secret automatically.
## Response Data Contracts
@@ -193,15 +247,33 @@ Note: the request contract is defined, but the corresponding route currently ret
| `status` | `string` | Always `"ok"`. |
| `read_only` | `bool` | Mirrors current API `read_only` mode. |
### `HealthReadyData`
| Field | Type | Description |
| --- | --- | --- |
| `ready` | `bool` | `true` when admission is open and at least one upstream is healthy. |
| `status` | `string` | `"ready"` or `"not_ready"`. |
| `reason` | `string?` | `admission_closed` or `no_healthy_upstreams` when not ready. |
| `admission_open` | `bool` | Current admission-gate state. |
| `healthy_upstreams` | `usize` | Number of healthy upstream entries. |
| `total_upstreams` | `usize` | Number of configured upstream entries. |
### `SummaryData`
| Field | Type | Description |
| --- | --- | --- |
| `uptime_seconds` | `f64` | Process uptime in seconds. |
| `connections_total` | `u64` | Total accepted client connections. |
| `connections_bad_total` | `u64` | Failed/invalid client connections. |
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
| `configured_users` | `usize` | Number of configured users in config. |
#### `ClassCount`
| Field | Type | Description |
| --- | --- | --- |
| `class` | `string` | Failure class label. |
| `total` | `u64` | Counter value for this class. |
### `SystemInfoData`
| Field | Type | Description |
| --- | --- | --- |
@@ -226,7 +298,12 @@ Note: the request contract is defined, but the corresponding route currently ret
| `conditional_cast_enabled` | `bool` | Whether conditional ME admission logic is enabled (`general.use_middle_proxy`). |
| `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. |
| `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. |
| `me2dc_fast_enabled` | `bool` | Whether fast ME -> direct fallback is enabled. |
| `use_middle_proxy` | `bool` | Current transport mode preference. |
| `route_mode` | `string` | Current route mode label from route runtime controller. |
| `reroute_active` | `bool` | `true` when ME fallback currently routes new sessions to Direct-DC. |
| `reroute_to_direct_at_epoch_secs` | `u64?` | Unix timestamp when current direct reroute began. |
| `reroute_reason` | `string?` | `fast_not_ready_fallback` or `strict_grace_fallback` while reroute is active. |
| `startup_status` | `string` | Startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). |
| `startup_stage` | `string` | Current startup stage identifier. |
| `startup_progress_pct` | `f64` | Startup progress percentage (`0..100`). |
@@ -277,11 +354,13 @@ Note: the request contract is defined, but the corresponding route currently ret
| `upstream` | `EffectiveUpstreamLimits` | Effective upstream connect/retry limits. |
| `middle_proxy` | `EffectiveMiddleProxyLimits` | Effective ME pool/floor/reconnect limits. |
| `user_ip_policy` | `EffectiveUserIpPolicyLimits` | Effective unique-IP policy mode/window. |
| `user_tcp_policy` | `EffectiveUserTcpPolicyLimits` | Effective per-user TCP connection policy. |
#### `EffectiveTimeoutLimits`
| Field | Type | Description |
| --- | --- | --- |
| `client_handshake_secs` | `u64` | Client handshake timeout. |
| `client_first_byte_idle_secs` | `u64` | First-byte idle timeout before protocol classification. |
| `tg_connect_secs` | `u64` | Upstream Telegram connect timeout. |
| `client_keepalive_secs` | `u64` | Client keepalive interval. |
| `client_ack_secs` | `u64` | ACK timeout. |
@@ -320,13 +399,20 @@ Note: the request contract is defined, but the corresponding route currently ret
| `writer_pick_mode` | `string` | Writer picker mode (`sorted_rr`, `p2c`). |
| `writer_pick_sample_size` | `u8` | Candidate sample size for `p2c` picker mode. |
| `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. |
| `me2dc_fast` | `bool` | Effective fast fallback flag. |
#### `EffectiveUserIpPolicyLimits`
| Field | Type | Description |
| --- | --- | --- |
| `global_each` | `usize` | Global per-user unique-IP limit applied when no per-user override exists. |
| `mode` | `string` | Unique-IP policy mode (`active_window`, `time_window`, `combined`). |
| `window_secs` | `u64` | Time window length used by unique-IP policy. |
#### `EffectiveUserTcpPolicyLimits`
| Field | Type | Description |
| --- | --- | --- |
| `global_each` | `usize` | Global per-user concurrent TCP limit applied when no per-user override exists. |
### `SecurityPostureData`
| Field | Type | Description |
| --- | --- | --- |
@@ -430,6 +516,8 @@ Note: the request contract is defined, but the corresponding route currently ret
| --- | --- | --- |
| `counters` | `RuntimeMeQualityCountersData` | Key ME lifecycle/error counters. |
| `route_drops` | `RuntimeMeQualityRouteDropData` | Route drop counters by reason. |
| `family_states` | `RuntimeMeQualityFamilyStateData[]` | Per-family ME route/recovery state rows. |
| `drain_gate` | `RuntimeMeQualityDrainGateData` | Current ME drain-gate decision state. |
| `dc_rtt` | `RuntimeMeQualityDcRttData[]` | Per-DC RTT and writer coverage rows. |
#### `RuntimeMeQualityCountersData`
@@ -451,6 +539,24 @@ Note: the request contract is defined, but the corresponding route currently ret
| `queue_full_base_total` | `u64` | Route drops in base-queue path. |
| `queue_full_high_total` | `u64` | Route drops in high-priority queue path. |
#### `RuntimeMeQualityFamilyStateData`
| Field | Type | Description |
| --- | --- | --- |
| `family` | `string` | Address family label. |
| `state` | `string` | Current family state label. |
| `state_since_epoch_secs` | `u64` | Unix timestamp when current state began. |
| `suppressed_until_epoch_secs` | `u64?` | Unix timestamp until suppression remains active. |
| `fail_streak` | `u32` | Consecutive failure count. |
| `recover_success_streak` | `u32` | Consecutive recovery success count. |
#### `RuntimeMeQualityDrainGateData`
| Field | Type | Description |
| --- | --- | --- |
| `route_quorum_ok` | `bool` | Whether route quorum condition allows drain. |
| `redundancy_ok` | `bool` | Whether redundancy condition allows drain. |
| `block_reason` | `string` | Current drain block reason label. |
| `updated_at_epoch_secs` | `u64` | Unix timestamp of the latest gate update. |
#### `RuntimeMeQualityDcRttData`
| Field | Type | Description |
| --- | --- | --- |
@@ -713,11 +819,24 @@ Note: the request contract is defined, but the corresponding route currently ret
| `uptime_seconds` | `f64` | Process uptime. |
| `connections_total` | `u64` | Total accepted connections. |
| `connections_bad_total` | `u64` | Failed/invalid connections. |
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
| `handshake_timeouts_total` | `u64` | Handshake timeouts. |
| `accept_permit_timeout_total` | `u64` | Listener admission permit acquisition timeouts. |
| `configured_users` | `usize` | Configured user count. |
| `telemetry_core_enabled` | `bool` | Core telemetry toggle. |
| `telemetry_user_enabled` | `bool` | User telemetry toggle. |
| `telemetry_me_level` | `string` | ME telemetry level (`off|normal|verbose`). |
| `conntrack_control_enabled` | `bool` | Whether conntrack control is enabled by policy. |
| `conntrack_control_available` | `bool` | Whether conntrack control backend is currently available. |
| `conntrack_pressure_active` | `bool` | Current conntrack pressure flag. |
| `conntrack_event_queue_depth` | `u64` | Current conntrack close-event queue depth. |
| `conntrack_rule_apply_ok` | `bool` | Last conntrack rule application state. |
| `conntrack_delete_attempt_total` | `u64` | Conntrack delete attempts. |
| `conntrack_delete_success_total` | `u64` | Successful conntrack deletes. |
| `conntrack_delete_not_found_total` | `u64` | Conntrack delete misses. |
| `conntrack_delete_error_total` | `u64` | Conntrack delete errors. |
| `conntrack_close_event_drop_total` | `u64` | Dropped conntrack close events. |
#### `ZeroUpstreamData`
| Field | Type | Description |
@@ -804,6 +923,24 @@ Note: the request contract is defined, but the corresponding route currently ret
| `route_drop_queue_full_total` | `u64` | Route drops due to full queue (total). |
| `route_drop_queue_full_base_total` | `u64` | Route drops in base queue mode. |
| `route_drop_queue_full_high_total` | `u64` | Route drops in high queue mode. |
| `d2c_batches_total` | `u64` | ME D->C batch flushes. |
| `d2c_batch_frames_total` | `u64` | ME D->C frames included in batches. |
| `d2c_batch_bytes_total` | `u64` | ME D->C payload bytes included in batches. |
| `d2c_flush_reason_queue_drain_total` | `u64` | Flushes caused by queue drain. |
| `d2c_flush_reason_batch_frames_total` | `u64` | Flushes caused by frame-count batch limit. |
| `d2c_flush_reason_batch_bytes_total` | `u64` | Flushes caused by byte-count batch limit. |
| `d2c_flush_reason_max_delay_total` | `u64` | Flushes caused by max-delay budget. |
| `d2c_flush_reason_ack_immediate_total` | `u64` | Flushes caused by immediate ACK policy. |
| `d2c_flush_reason_close_total` | `u64` | Flushes caused by close path. |
| `d2c_data_frames_total` | `u64` | ME D->C data frames. |
| `d2c_ack_frames_total` | `u64` | ME D->C ACK frames. |
| `d2c_payload_bytes_total` | `u64` | ME D->C payload bytes. |
| `d2c_write_mode_coalesced_total` | `u64` | Coalesced D->C writes. |
| `d2c_write_mode_split_total` | `u64` | Split D->C writes. |
| `d2c_quota_reject_pre_write_total` | `u64` | D->C quota rejects before write. |
| `d2c_quota_reject_post_write_total` | `u64` | D->C quota rejects after write. |
| `d2c_frame_buf_shrink_total` | `u64` | D->C frame-buffer shrink operations. |
| `d2c_frame_buf_shrink_bytes_total` | `u64` | Bytes released by D->C frame-buffer shrink operations. |
| `socks_kdf_strict_reject_total` | `u64` | SOCKS KDF strict rejects. |
| `socks_kdf_compat_fallback_total` | `u64` | SOCKS KDF compat fallbacks. |
| `endpoint_quarantine_total` | `u64` | Endpoint quarantine activations. |
@@ -963,6 +1100,8 @@ Note: the request contract is defined, but the corresponding route currently ret
| `required_writers` | `usize` | Required writers based on current floor policy. |
| `alive_writers` | `usize` | Writers currently alive. |
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
| `fresh_alive_writers` | `usize` | Alive writers that match freshness requirements. |
| `fresh_coverage_pct` | `f64` | `fresh_alive_writers / required_writers * 100`. |
#### `MeWriterStatus`
| Field | Type | Description |
@@ -977,6 +1116,12 @@ Note: the request contract is defined, but the corresponding route currently ret
| `bound_clients` | `usize` | Number of currently bound clients. |
| `idle_for_secs` | `u64?` | Idle age in seconds if idle. |
| `rtt_ema_ms` | `f64?` | RTT exponential moving average. |
| `matches_active_generation` | `bool` | Whether this writer belongs to the active pool generation. |
| `in_desired_map` | `bool` | Whether this writer's endpoint remains in desired topology. |
| `allow_drain_fallback` | `bool` | Whether drain fallback is allowed for this writer. |
| `drain_started_at_epoch_secs` | `u64?` | Unix timestamp when drain started. |
| `drain_deadline_epoch_secs` | `u64?` | Unix timestamp of drain deadline. |
| `drain_over_ttl` | `bool` | Whether drain has exceeded its TTL. |
### `DcStatusData`
| Field | Type | Description |
@@ -1001,6 +1146,8 @@ Note: the request contract is defined, but the corresponding route currently ret
| `floor_capped` | `bool` | `true` when computed floor target was capped by active limits. |
| `alive_writers` | `usize` | Alive writers in this DC. |
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
| `fresh_alive_writers` | `usize` | Fresh alive writers in this DC. |
| `fresh_coverage_pct` | `f64` | `fresh_alive_writers / required_writers * 100`. |
| `rtt_ms` | `f64?` | Aggregated RTT for DC. |
| `load` | `usize` | Active client sessions bound to this DC. |
@@ -1014,6 +1161,7 @@ Note: the request contract is defined, but the corresponding route currently ret
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | Username. |
| `in_runtime` | `bool` | Whether current runtime config already contains this user. |
| `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). |
| `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. |
| `expiration_rfc3339` | `string?` | Optional expiration timestamp. |
@@ -1027,12 +1175,25 @@ Note: the request contract is defined, but the corresponding route currently ret
| `total_octets` | `u64` | Total traffic octets for this user. |
| `links` | `UserLinks` | Active connection links derived from current config. |
### `UserActiveIps`
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | Username with at least one active tracked source IP. |
| `active_ips` | `ip[]` | Active source IPs for this user. |
#### `UserLinks`
| Field | Type | Description |
| --- | --- | --- |
| `classic` | `string[]` | Active `tg://proxy` links for classic mode. |
| `secure` | `string[]` | Active `tg://proxy` links for secure/DD mode. |
| `tls` | `string[]` | Active `tg://proxy` links for EE-TLS mode (for each host+TLS domain). |
| `tls_domains` | `TlsDomainLink[]` | Extra TLS-domain links as explicit domain/link pairs for `censorship.tls_domains`. |
#### `TlsDomainLink`
| Field | Type | Description |
| --- | --- | --- |
| `domain` | `string` | TLS domain represented by the link. |
| `link` | `string` | `tg://proxy` link for this domain. |
Link generation uses active config and enabled modes:
- Link port is `general.links.public_port` when configured; otherwise `server.port`.
@@ -1052,13 +1213,27 @@ Link generation uses active config and enabled modes:
| `user` | `UserInfo` | Created or updated user view. |
| `secret` | `string` | Effective user secret. |
### `DeleteUserResponse`
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | Deleted username. |
| `in_runtime` | `bool` | `true` when runtime config still contains the user and hot-reload has not applied deletion yet. |
### `ResetUserQuotaResponse`
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | User whose runtime quota counter was reset. |
| `used_bytes` | `u64` | Current used bytes after reset; always `0` on success. |
| `last_reset_epoch_secs` | `u64` | Unix timestamp of the reset operation. |
## Mutation Semantics
| Endpoint | Notes |
| --- | --- |
| `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). |
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. Current implementation persists full config document on success. |
| `POST /v1/users/{username}/rotate-secret` | Currently returns `404` in runtime route matcher; request schema is reserved for intended behavior. |
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged; explicit `null` removes optional per-user entries. The write path updates only affected `access.*` TOML tables. |
| `POST /v1/users/{username}/rotate-secret` | Replaces the user's secret with a provided valid 32-hex value or a generated value, then returns the effective secret in `CreateUserResponse`. |
| `POST /v1/users/{username}/reset-quota` | Resets the runtime quota counter for the route username, persists quota state to `general.quota_state_path`, and does not modify user config. |
| `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. |
All mutating endpoints:
@@ -1133,5 +1308,4 @@ When `general.use_middle_proxy=true` and `general.me2dc_fallback=true`:
## Known Limitations (Current Release)
- `POST /v1/users/{username}/rotate-secret` is currently unreachable in route matcher and returns `404`.
- API runtime controls under `server.api` are documented as restart-required; hot-reload behavior for these fields is not strictly uniform in all change combinations.

View File

@@ -128,7 +128,48 @@ Recommended for cleaner testing:
Persisted cache artifacts are useful, but they are not required if packet captures already demonstrate the runtime result.
### 4. Capture a direct-origin trace
### 4. Check TLS-front profile health metrics
If the metrics endpoint is enabled, check the TLS-front profile health before packet-capture validation:
```bash
curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert'
```
The profile-health metrics expose the runtime state of configured TLS front domains:
- `telemt_tls_front_profile_domains` shows configured, emitted, and suppressed domain series.
- `telemt_tls_front_profile_info` shows profile source and feature flags per domain.
- `telemt_tls_front_profile_age_seconds` shows cached profile age.
- `telemt_tls_front_profile_app_data_records` shows cached AppData record count.
- `telemt_tls_front_profile_ticket_records` shows cached ticket-like tail record count.
- `telemt_tls_front_profile_change_cipher_spec_records` shows cached ChangeCipherSpec count.
- `telemt_tls_front_profile_app_data_bytes` shows total cached AppData bytes.
Interpretation:
- `source="merged"` or `source="raw"` means real TLS profile data is being used.
- `source="default"` or `is_default="true"` means the domain currently uses the synthetic default fallback.
- `has_cert_payload="true"` means certificate payload data is available for TLS emulation.
- Non-zero AppData/ticket/CCS counters show captured server-flight shape.
Example healthy output:
```text
telemt_tls_front_profile_domains{status="configured"} 1
telemt_tls_front_profile_domains{status="emitted"} 1
telemt_tls_front_profile_domains{status="suppressed"} 0
telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1
telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20
telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3
telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240
```
These metrics do not prove byte-level origin equivalence. They are an operational health signal that the configured domain is backed by real cached profile data instead of default fallback data.
### 5. Capture a direct-origin trace
From a separate client host, connect directly to the origin:
@@ -142,7 +183,7 @@ Capture with:
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
```
### 5. Capture a Telemt FakeTLS success-path trace
### 6. Capture a Telemt FakeTLS success-path trace
Now connect to Telemt with a real Telegram client through an `ee` proxy link that targets the Telemt instance.
@@ -154,7 +195,7 @@ Capture with:
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
```
### 6. Decode TLS record structure
### 7. Decode TLS record structure
Use `tshark` to print record-level structure:
@@ -182,7 +223,7 @@ Focus on the server flight after ClientHello:
- `20` = ChangeCipherSpec
- `23` = ApplicationData
### 7. Build a comparison table
### 8. Build a comparison table
A compact table like the following is usually enough:

View File

@@ -126,9 +126,50 @@ openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
2. Дайте ему получить TLS front profile data для выбранного домена.
3. Если `tls_front_dir` хранится persistently, убедитесь, что TLS front cache заполнен.
Persisted cache artifacts полезны, но не обязательны, если packet capture уже показывают runtime result.
Сохранённые артефакты кэша полезны, но не обязательны, если packet capture уже показывает результат в runtime.
### 4. Снять direct-origin trace
### 4. Проверить метрики состояния TLS-front profile
Если endpoint метрик включён, перед проверкой через packet capture можно быстро проверить состояние TLS-front profile:
```bash
curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert'
```
Метрики состояния профиля показывают runtime-состояние настроенных TLS-front доменов:
- `telemt_tls_front_profile_domains` показывает количество настроенных, экспортируемых и скрытых из-за лимита доменов.
- `telemt_tls_front_profile_info` показывает источник профиля и флаги доступных данных по каждому домену.
- `telemt_tls_front_profile_age_seconds` показывает возраст закешированного профиля.
- `telemt_tls_front_profile_app_data_records` показывает количество закешированных AppData records.
- `telemt_tls_front_profile_ticket_records` показывает количество закешированных ticket-like tail records.
- `telemt_tls_front_profile_change_cipher_spec_records` показывает закешированное количество ChangeCipherSpec records.
- `telemt_tls_front_profile_app_data_bytes` показывает общий размер закешированных AppData bytes.
Интерпретация:
- `source="merged"` или `source="raw"` означает, что используются реальные данные TLS-профиля.
- `source="default"` или `is_default="true"` означает, что домен сейчас работает на synthetic default fallback.
- `has_cert_payload="true"` означает, что certificate payload доступен для TLS emulation.
- Ненулевые AppData/ticket/CCS counters показывают захваченную форму server flight.
Пример здорового состояния:
```text
telemt_tls_front_profile_domains{status="configured"} 1
telemt_tls_front_profile_domains{status="emitted"} 1
telemt_tls_front_profile_domains{status="suppressed"} 0
telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1
telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20
telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3
telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240
```
Эти метрики не доказывают побайтную эквивалентность с origin. Это эксплуатационный сигнал состояния: настроенный домен действительно основан на реальных закешированных данных профиля, а не на default fallback.
### 5. Снять direct-origin trace
С отдельной клиентской машины подключитесь напрямую к origin:
@@ -142,7 +183,7 @@ Capture:
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
```
### 5. Снять Telemt FakeTLS success-path trace
### 6. Снять Telemt FakeTLS success-path trace
Теперь подключитесь к Telemt через реальный Telegram client с `ee` proxy link, который указывает на Telemt instance.
@@ -154,7 +195,7 @@ Capture:
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
```
### 6. Декодировать структуру TLS records
### 7. Декодировать структуру TLS records
Используйте `tshark`, чтобы вывести record-level structure:
@@ -182,7 +223,7 @@ tshark -r telemt-emulated.pcap -Y "tls.record" -T fields \
- `20` = ChangeCipherSpec
- `23` = ApplicationData
### 7. Собрать сравнительную таблицу
### 8. Собрать сравнительную таблицу
Обычно достаточно короткой таблицы такого вида:

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -210,6 +210,13 @@ If you need to allow connections with any domains (ignoring SNI mismatches), add
unknown_sni_action = "mask"
```
Alternatively, if you want telemt to behave like a vanilla nginx with `ssl_reject_handshake on;` on unknown SNI (emit a TLS `unrecognized_name` alert and close the connection), use:
```toml
[censorship]
unknown_sni_action = "reject_handshake"
```
This does not recover stale clients, but it makes port 443 wire-indistinguishable from a stock web server that simply does not host the requested vhost.
### How to view metrics
1. Open the configuration file: `nano /etc/telemt/telemt.toml`.

View File

@@ -227,6 +227,13 @@ curl -s http://127.0.0.1:9091/v1/users | jq
unknown_sni_action = "mask"
```
Альтернатива: если вы хотите, чтобы telemt на неизвестный SNI вёл себя как обычный nginx с `ssl_reject_handshake on;` (отдавал TLS-alert `unrecognized_name` и закрывал соединение), используйте:
```toml
[censorship]
unknown_sni_action = "reject_handshake"
```
Это не пропускает старых клиентов, но делает поведение на 443-м порту неотличимым от стокового веб-сервера, у которого просто нет такого виртуального хоста.
## Как посмотреть метрики
1. Откройте файл конфигурации: `nano /etc/telemt/telemt.toml`.

View File

@@ -164,7 +164,7 @@ tls_front_dir = "tlsfront" # Директория кэша для эмуляц
hello = "00000000000000000000000000000000"
```
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
Затем нажмите Ctrl+O -> Ctrl+X, чтобы сохранить
> [!WARNING]
> Замените значение параметра `hello` на значение, которое вы получили в пункте 0.

View File

@@ -27,6 +27,8 @@ ACTION="install"
TARGET_VERSION="${VERSION:-latest}"
LANG_CHOICE="en"
PATH="${PATH}:/usr/sbin:/sbin"
set_language() {
case "$1" in
ru)
@@ -102,6 +104,8 @@ set_language() {
L_OUT_SUCC_H="УСТАНОВКА УСПЕШНО ЗАВЕРШЕНА"
L_OUT_UNINST_H="УДАЛЕНИЕ ЗАВЕРШЕНО"
L_OUT_LINK="Ваша ссылка для подключения к Telegram Proxy:\n"
L_ERR_INCORR_ROOT_LOGIN="Используйте 'su -' или 'sudo -i' для входа под пользователем root"
L_OUT_LOGS="Чтобы посмотреть логи (в случае проблем), используйте команду:"
;;
*)
L_ERR_DOMAIN_REQ="requires a domain argument."
@@ -176,6 +180,8 @@ set_language() {
L_OUT_SUCC_H="INSTALLATION SUCCESS"
L_OUT_UNINST_H="UNINSTALLATION COMPLETE"
L_OUT_LINK="Your Telegram Proxy connection link:\n"
L_ERR_INCORR_ROOT_LOGIN="Use 'su -' or 'sudo -i' to login under root"
L_OUT_LOGS="To view logs (in case of issues), use the following command:"
;;
esac
}
@@ -388,6 +394,9 @@ verify_common() {
if [ "$(id -u)" -eq 0 ]; then
SUDO=""
if [ "${USER:-}" != "root" ] && [ "${LOGNAME:-}" != "root" ]; then
die "$L_ERR_INCORR_ROOT_LOGIN"
fi
else
command -v sudo >/dev/null 2>&1 || die "$L_ERR_ROOT"
SUDO="sudo"
@@ -532,7 +541,7 @@ install_binary() {
fi
$SUDO mkdir -p "$INSTALL_DIR" || die "$L_ERR_MKDIR"
$SUDO rm -f "$bin_dst" 2>/dev/null || true
if command -v install >/dev/null 2>&1; then
@@ -602,33 +611,33 @@ install_config() {
tmp_conf="${TEMP_DIR}/config.tmp"
$SUDO cat "$CONFIG_FILE" > "$tmp_conf"
escaped_domain="$(printf '%s\n' "$TLS_DOMAIN" | tr -d '[:cntrl:]' | sed 's/\\/\\\\/g; s/"/\\"/g')"
awk -v port="$SERVER_PORT" -v secret="$USER_SECRET" -v domain="$escaped_domain" -v ad_tag="$AD_TAG" \
-v flag_p="$PORT_PROVIDED" -v flag_s="$SECRET_PROVIDED" -v flag_d="$DOMAIN_PROVIDED" -v flag_a="$AD_TAG_PROVIDED" '
BEGIN { ad_tag_handled = 0 }
flag_p == "1" && /^[ \t]*port[ \t]*=/ { print "port = " port; next }
flag_s == "1" && /^[ \t]*hello[ \t]*=/ { print "hello = \"" secret "\""; next }
flag_d == "1" && /^[ \t]*tls_domain[ \t]*=/ { print "tls_domain = \"" domain "\""; next }
flag_a == "1" && /^[ \t]*ad_tag[ \t]*=/ {
if (!ad_tag_handled) {
print "ad_tag = \"" ad_tag "\"";
ad_tag_handled = 1;
}
next
flag_a == "1" && /^[ \t]*ad_tag[ \t]*=/ {
if (!ad_tag_handled) {
print "ad_tag = \"" ad_tag "\"";
ad_tag_handled = 1;
}
next
}
flag_a == "1" && /^\[general\]/ {
print;
if (!ad_tag_handled) {
print "ad_tag = \"" ad_tag "\"";
ad_tag_handled = 1;
}
next
flag_a == "1" && /^\[general\]/ {
print;
if (!ad_tag_handled) {
print "ad_tag = \"" ad_tag "\"";
ad_tag_handled = 1;
}
next
}
{ print }
' "$tmp_conf" > "${tmp_conf}.new" && mv "${tmp_conf}.new" "$tmp_conf"
@@ -778,11 +787,11 @@ uninstall() {
say "$L_U_STAGE_5"
$SUDO rm -rf "$CONFIG_DIR" "$WORK_DIR"
$SUDO rm -f "$CONFIG_FILE"
if check_os_entity passwd telemt; then
$SUDO userdel telemt 2>/dev/null || $SUDO deluser telemt 2>/dev/null || true
fi
if check_os_entity group telemt; then
$SUDO groupdel telemt 2>/dev/null || $SUDO delgroup telemt 2>/dev/null || true
fi
@@ -909,7 +918,7 @@ case "$ACTION" in
if command -v curl >/dev/null 2>&1; then SERVER_IP="$(curl -s4 -m 3 ifconfig.me 2>/dev/null || curl -s4 -m 3 api.ipify.org 2>/dev/null || true)"
elif command -v wget >/dev/null 2>&1; then SERVER_IP="$(wget -qO- -T 3 ifconfig.me 2>/dev/null || wget -qO- -T 3 api.ipify.org 2>/dev/null || true)"; fi
[ -z "$SERVER_IP" ] && SERVER_IP="<YOUR_SERVER_IP>"
if command -v xxd >/dev/null 2>&1; then HEX_DOMAIN="$(printf '%s' "$TLS_DOMAIN" | xxd -p | tr -d '\n')"
elif command -v hexdump >/dev/null 2>&1; then HEX_DOMAIN="$(printf '%s' "$TLS_DOMAIN" | hexdump -v -e '/1 "%02x"')"
elif command -v od >/dev/null 2>&1; then HEX_DOMAIN="$(printf '%s' "$TLS_DOMAIN" | od -A n -t x1 | tr -d ' \n')"
@@ -920,6 +929,15 @@ case "$ACTION" in
printf '%b\n' "$L_OUT_LINK"
printf ' tg://proxy?server=%s&port=%s&secret=%s\n\n' "$SERVER_IP" "$SERVER_PORT" "$CLIENT_SECRET"
svc="$(get_svc_mgr)"
if [ "$svc" = "systemd" ]; then
printf '%s\n' "$L_OUT_LOGS"
printf ' sudo journalctl -u %s -f\n\n' "$SERVICE_NAME"
elif [ "$svc" = "openrc" ]; then
printf '%s\n' "$L_OUT_LOGS"
printf ' sudo tail -f /var/log/messages /var/log/syslog 2>/dev/null | grep -i %s\n\n' "$SERVICE_NAME"
fi
printf '====================================================================\n'
;;
esac

View File

@@ -82,6 +82,7 @@ pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyCon
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
}
#[allow(dead_code)]
pub(super) async fn save_config_to_disk(
config_path: &Path,
cfg: &ProxyConfig,
@@ -106,6 +107,12 @@ pub(super) async fn save_access_sections_to_disk(
if applied.contains(section) {
continue;
}
if find_toml_table_bounds(&content, section.table_name()).is_none()
&& access_section_is_empty(cfg, *section)
{
applied.push(*section);
continue;
}
let rendered = render_access_section(cfg, *section)?;
content = upsert_toml_table(&content, section.table_name(), &rendered);
applied.push(*section);
@@ -183,6 +190,17 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result<St
Ok(out)
}
fn access_section_is_empty(cfg: &ProxyConfig, section: AccessSection) -> bool {
match section {
AccessSection::Users => cfg.access.users.is_empty(),
AccessSection::UserAdTags => cfg.access.user_ad_tags.is_empty(),
AccessSection::UserMaxTcpConns => cfg.access.user_max_tcp_conns.is_empty(),
AccessSection::UserExpirations => cfg.access.user_expirations.is_empty(),
AccessSection::UserDataQuota => cfg.access.user_data_quota.is_empty(),
AccessSection::UserMaxUniqueIps => cfg.access.user_max_unique_ips.is_empty(),
}
}
fn serialize_table_body<T: Serialize>(value: &T) -> Result<String, ApiFailure> {
toml::to_string(value)
.map_err(|e| ApiFailure::internal(format!("failed to serialize access section: {}", e)))

View File

@@ -5,6 +5,7 @@ use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::Duration;
use http_body_util::Full;
use hyper::body::{Bytes, Incoming};
@@ -12,8 +13,10 @@ use hyper::header::AUTHORIZATION;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Method, Request, Response, StatusCode};
use subtle::ConstantTimeEq;
use tokio::net::TcpListener;
use tokio::sync::{Mutex, RwLock, watch};
use tokio::sync::{Mutex, RwLock, Semaphore, watch};
use tokio::time::timeout;
use tracing::{debug, info, warn};
use crate::config::{ApiGrayAction, ProxyConfig};
@@ -28,6 +31,7 @@ mod config_store;
mod events;
mod http_utils;
mod model;
mod patch;
mod runtime_edge;
mod runtime_init;
mod runtime_min;
@@ -41,8 +45,9 @@ use config_store::{current_revision, load_config_from_disk, parse_if_match};
use events::ApiEventStore;
use http_utils::{error_response, read_json, read_optional_json, success_response};
use model::{
ApiFailure, CreateUserRequest, DeleteUserResponse, HealthData, PatchUserRequest,
RotateSecretRequest, SummaryData, UserActiveIps,
ApiFailure, ClassCount, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
PatchUserRequest, ResetUserQuotaResponse, RotateSecretRequest, SummaryData, UserActiveIps,
is_valid_username,
};
use runtime_edge::{
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
@@ -65,6 +70,10 @@ use runtime_zero::{
};
use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config};
const API_MAX_CONTROL_CONNECTIONS: usize = 1024;
const API_HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
const ROUTE_USERNAME_ERROR: &str = "username must match [A-Za-z0-9_.-] and be 1..64 chars";
pub(super) struct ApiRuntimeState {
pub(super) process_started_at_epoch_secs: u64,
pub(super) config_reload_count: AtomicU64,
@@ -79,6 +88,7 @@ pub(super) struct ApiShared {
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
pub(super) upstream_manager: Arc<UpstreamManager>,
pub(super) config_path: PathBuf,
pub(super) quota_state_path: PathBuf,
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
pub(super) mutation_lock: Arc<Mutex<()>>,
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
@@ -101,6 +111,18 @@ impl ApiShared {
}
}
fn auth_header_matches(actual: &str, expected: &str) -> bool {
actual.as_bytes().ct_eq(expected.as_bytes()).into()
}
fn parse_route_username(user: &str) -> Result<&str, ApiFailure> {
if is_valid_username(user) {
Ok(user)
} else {
Err(ApiFailure::bad_request(ROUTE_USERNAME_ERROR))
}
}
pub async fn serve(
listen: SocketAddr,
stats: Arc<Stats>,
@@ -111,6 +133,7 @@ pub async fn serve(
config_rx: watch::Receiver<Arc<ProxyConfig>>,
admission_rx: watch::Receiver<bool>,
config_path: PathBuf,
quota_state_path: PathBuf,
detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
process_started_at_epoch_secs: u64,
startup_tracker: Arc<StartupTracker>,
@@ -142,6 +165,7 @@ pub async fn serve(
me_pool,
upstream_manager,
config_path,
quota_state_path,
detected_ips_rx,
mutation_lock: Arc::new(Mutex::new(())),
minimal_cache: Arc::new(Mutex::new(None)),
@@ -163,6 +187,8 @@ pub async fn serve(
shared.runtime_events.clone(),
);
let connection_permits = Arc::new(Semaphore::new(API_MAX_CONTROL_CONNECTIONS));
loop {
let (stream, peer) = match listener.accept().await {
Ok(v) => v,
@@ -172,20 +198,45 @@ pub async fn serve(
}
};
let connection_permit = match connection_permits.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
debug!(
peer = %peer,
max_connections = API_MAX_CONTROL_CONNECTIONS,
"Dropping API connection: control-plane connection budget exhausted"
);
continue;
}
};
let shared_conn = shared.clone();
let config_rx_conn = config_rx.clone();
tokio::spawn(async move {
let _connection_permit = connection_permit;
let svc = service_fn(move |req: Request<Incoming>| {
let shared_req = shared_conn.clone();
let config_rx_req = config_rx_conn.clone();
async move { handle(req, peer, shared_req, config_rx_req).await }
});
if let Err(error) = http1::Builder::new()
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
.await
match timeout(
API_HTTP_CONNECTION_TIMEOUT,
http1::Builder::new().serve_connection(hyper_util::rt::TokioIo::new(stream), svc),
)
.await
{
if !error.is_user() {
debug!(error = %error, "API connection error");
Ok(Ok(())) => {}
Ok(Err(error)) => {
if !error.is_user() {
debug!(error = %error, "API connection error");
}
}
Err(_) => {
debug!(
peer = %peer,
timeout_ms = API_HTTP_CONNECTION_TIMEOUT.as_millis() as u64,
"API connection timed out"
);
}
}
});
@@ -241,7 +292,7 @@ async fn handle(
.headers()
.get(AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.map(|v| v == api_cfg.auth_header)
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
.unwrap_or(false);
if !auth_ok {
return Ok(error_response(
@@ -275,6 +326,33 @@ async fn handle(
};
Ok(success_response(StatusCode::OK, data, revision))
}
("GET", "/v1/health/ready") => {
let revision = current_revision(&shared.config_path).await?;
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
let upstream_health = shared.upstream_manager.api_health_summary().await;
let ready = admission_open && upstream_health.healthy_total > 0;
let reason = if ready {
None
} else if !admission_open {
Some("admission_closed")
} else {
Some("no_healthy_upstreams")
};
let data = HealthReadyData {
ready,
status: if ready { "ready" } else { "not_ready" },
reason,
admission_open,
healthy_upstreams: upstream_health.healthy_total,
total_upstreams: upstream_health.configured_total,
};
let status_code = if ready {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
};
Ok(success_response(status_code, data, revision))
}
("GET", "/v1/system/info") => {
let revision = current_revision(&shared.config_path).await?;
let data = build_system_info_data(shared.as_ref(), cfg.as_ref(), &revision);
@@ -307,10 +385,24 @@ async fn handle(
}
("GET", "/v1/stats/summary") => {
let revision = current_revision(&shared.config_path).await?;
let connections_bad_by_class = shared
.stats
.get_connects_bad_class_counts()
.into_iter()
.map(|(class, total)| ClassCount { class, total })
.collect();
let handshake_failures_by_class = shared
.stats
.get_handshake_failure_class_counts()
.into_iter()
.map(|(class, total)| ClassCount { class, total })
.collect();
let data = SummaryData {
uptime_seconds: shared.stats.uptime_secs(),
connections_total: shared.stats.get_connects_all(),
connections_bad_total: shared.stats.get_connects_bad(),
connections_bad_by_class,
handshake_failures_by_class,
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
configured_users: cfg.access.users.len(),
};
@@ -449,10 +541,115 @@ async fn handle(
Ok(success_response(status, data, revision))
}
_ => {
if method == Method::POST
&& let Some(user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/reset-quota"))
&& !user.is_empty()
&& !user.contains('/')
{
let user = parse_route_username(user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let snapshot = match crate::quota_state::reset_user_quota(
&shared.quota_state_path,
shared.stats.as_ref(),
user,
)
.await
{
Ok(snapshot) => snapshot,
Err(error) => {
shared.runtime_events.record(
"api.user.reset_quota.failed",
format!("username={} error={}", user, error),
);
return Err(ApiFailure::internal(format!(
"Failed to reset user quota: {}",
error
)));
}
};
shared
.runtime_events
.record("api.user.reset_quota.ok", format!("username={}", user));
let revision = current_revision(&shared.config_path).await?;
return Ok(success_response(
StatusCode::OK,
ResetUserQuotaResponse {
username: user.to_string(),
used_bytes: snapshot.used_bytes,
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
},
revision,
));
}
if method == Method::POST
&& let Some(base_user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/rotate-secret"))
&& !base_user.is_empty()
&& !base_user.contains('/')
{
let base_user = parse_route_username(base_user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body =
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
.await?;
let result = rotate_secret(
base_user,
body.unwrap_or_default(),
expected_revision,
&shared,
)
.await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.rotate_secret.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.user.in_runtime =
runtime_cfg.access.users.contains_key(&data.user.username);
shared.runtime_events.record(
"api.user.rotate_secret.ok",
format!("username={}", base_user),
);
let status = if data.user.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
&& !user.is_empty()
&& !user.contains('/')
{
let user = parse_route_username(user)?;
if method == Method::GET {
let revision = current_revision(&shared.config_path).await?;
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
@@ -553,56 +750,6 @@ async fn handle(
};
return Ok(success_response(status, response, revision));
}
if method == Method::POST
&& let Some(base_user) = user.strip_suffix("/rotate-secret")
&& !base_user.is_empty()
&& !base_user.contains('/')
{
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body =
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
.await?;
let result = rotate_secret(
base_user,
body.unwrap_or_default(),
expected_revision,
&shared,
)
.await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.rotate_secret.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.user.in_runtime =
runtime_cfg.access.users.contains_key(&data.user.username);
shared.runtime_events.record(
"api.user.rotate_secret.ok",
format!("username={}", base_user),
);
let status = if data.user.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if method == Method::POST {
return Ok(error_response(
request_id,

View File

@@ -5,6 +5,7 @@ use chrono::{DateTime, Utc};
use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use super::patch::{Patch, patch_field};
use crate::crypto::SecureRandom;
const MAX_USERNAME_LEN: usize = 64;
@@ -60,11 +61,30 @@ pub(super) struct HealthData {
pub(super) read_only: bool,
}
#[derive(Serialize)]
pub(super) struct HealthReadyData {
pub(super) ready: bool,
pub(super) status: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
pub(super) reason: Option<&'static str>,
pub(super) admission_open: bool,
pub(super) healthy_upstreams: usize,
pub(super) total_upstreams: usize,
}
#[derive(Serialize, Clone)]
pub(super) struct ClassCount {
pub(super) class: String,
pub(super) total: u64,
}
#[derive(Serialize)]
pub(super) struct SummaryData {
pub(super) uptime_seconds: f64,
pub(super) connections_total: u64,
pub(super) connections_bad_total: u64,
pub(super) connections_bad_by_class: Vec<ClassCount>,
pub(super) handshake_failures_by_class: Vec<ClassCount>,
pub(super) handshake_timeouts_total: u64,
pub(super) configured_users: usize,
}
@@ -80,6 +100,8 @@ pub(super) struct ZeroCoreData {
pub(super) uptime_seconds: f64,
pub(super) connections_total: u64,
pub(super) connections_bad_total: u64,
pub(super) connections_bad_by_class: Vec<ClassCount>,
pub(super) handshake_failures_by_class: Vec<ClassCount>,
pub(super) handshake_timeouts_total: u64,
pub(super) accept_permit_timeout_total: u64,
pub(super) configured_users: usize,
@@ -434,6 +456,13 @@ pub(super) struct UserLinks {
pub(super) classic: Vec<String>,
pub(super) secure: Vec<String>,
pub(super) tls: Vec<String>,
pub(super) tls_domains: Vec<TlsDomainLink>,
}
#[derive(Serialize)]
pub(super) struct TlsDomainLink {
pub(super) domain: String,
pub(super) link: String,
}
#[derive(Serialize)]
@@ -472,6 +501,13 @@ pub(super) struct DeleteUserResponse {
pub(super) in_runtime: bool,
}
#[derive(Serialize)]
pub(super) struct ResetUserQuotaResponse {
pub(super) username: String,
pub(super) used_bytes: u64,
pub(super) last_reset_epoch_secs: u64,
}
#[derive(Deserialize)]
pub(super) struct CreateUserRequest {
pub(super) username: String,
@@ -486,11 +522,16 @@ pub(super) struct CreateUserRequest {
#[derive(Deserialize)]
pub(super) struct PatchUserRequest {
pub(super) secret: Option<String>,
pub(super) user_ad_tag: Option<String>,
pub(super) max_tcp_conns: Option<usize>,
pub(super) expiration_rfc3339: Option<String>,
pub(super) data_quota_bytes: Option<u64>,
pub(super) max_unique_ips: Option<usize>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) user_ad_tag: Patch<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) max_tcp_conns: Patch<usize>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) expiration_rfc3339: Patch<String>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) data_quota_bytes: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) max_unique_ips: Patch<usize>,
}
#[derive(Default, Deserialize)]
@@ -509,6 +550,20 @@ pub(super) fn parse_optional_expiration(
Ok(Some(parsed.with_timezone(&Utc)))
}
pub(super) fn parse_patch_expiration(
value: &Patch<String>,
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
match value {
Patch::Unchanged => Ok(Patch::Unchanged),
Patch::Remove => Ok(Patch::Remove),
Patch::Set(raw) => {
let parsed = DateTime::parse_from_rfc3339(raw)
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
Ok(Patch::Set(parsed.with_timezone(&Utc)))
}
}
}
pub(super) fn is_valid_user_secret(secret: &str) -> bool {
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
}

130
src/api/patch.rs Normal file
View File

@@ -0,0 +1,130 @@
use serde::Deserialize;
/// Three-state field for JSON Merge Patch semantics on the `PATCH /v1/users/{user}`
/// endpoint.
///
/// `Unchanged` is produced when the JSON body omits the field entirely and tells the
/// handler to leave the corresponding configuration entry untouched. `Remove` is
/// produced when the JSON body sets the field to `null` and instructs the handler to
/// drop the entry from the corresponding access HashMap. `Set` carries an explicit
/// new value, including zero, which is preserved verbatim in the configuration.
#[derive(Debug)]
pub(super) enum Patch<T> {
Unchanged,
Remove,
Set(T),
}
impl<T> Default for Patch<T> {
fn default() -> Self {
Self::Unchanged
}
}
/// Serde deserializer adapter for fields that follow JSON Merge Patch semantics.
///
/// Pair this with `#[serde(default, deserialize_with = "patch_field")]` on a
/// `Patch<T>` field. An omitted field falls back to `Patch::Unchanged` via
/// `Default`; an explicit JSON `null` becomes `Patch::Remove`; any other value
/// becomes `Patch::Set(v)`.
pub(super) fn patch_field<'de, D, T>(deserializer: D) -> Result<Patch<T>, D::Error>
where
D: serde::Deserializer<'de>,
T: serde::Deserialize<'de>,
{
Option::<T>::deserialize(deserializer).map(|opt| match opt {
Some(value) => Patch::Set(value),
None => Patch::Remove,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::api::model::{PatchUserRequest, parse_patch_expiration};
use chrono::{TimeZone, Utc};
use serde::Deserialize;
#[derive(Deserialize)]
struct Holder {
#[serde(default, deserialize_with = "patch_field")]
value: Patch<u64>,
}
fn parse(json: &str) -> Holder {
serde_json::from_str(json).expect("valid json")
}
#[test]
fn omitted_field_yields_unchanged() {
let h = parse("{}");
assert!(matches!(h.value, Patch::Unchanged));
}
#[test]
fn explicit_null_yields_remove() {
let h = parse(r#"{"value": null}"#);
assert!(matches!(h.value, Patch::Remove));
}
#[test]
fn explicit_value_yields_set() {
let h = parse(r#"{"value": 42}"#);
assert!(matches!(h.value, Patch::Set(42)));
}
#[test]
fn explicit_zero_yields_set_zero() {
let h = parse(r#"{"value": 0}"#);
assert!(matches!(h.value, Patch::Set(0)));
}
#[test]
fn parse_patch_expiration_passes_unchanged_and_remove_through() {
assert!(matches!(
parse_patch_expiration(&Patch::Unchanged),
Ok(Patch::Unchanged)
));
assert!(matches!(
parse_patch_expiration(&Patch::Remove),
Ok(Patch::Remove)
));
}
#[test]
fn parse_patch_expiration_parses_set_value() {
let parsed =
parse_patch_expiration(&Patch::Set("2030-01-02T03:04:05Z".into())).expect("valid");
match parsed {
Patch::Set(dt) => {
assert_eq!(dt, Utc.with_ymd_and_hms(2030, 1, 2, 3, 4, 5).unwrap());
}
other => panic!("expected Patch::Set, got {:?}", other),
}
}
#[test]
fn parse_patch_expiration_rejects_invalid_set_value() {
assert!(parse_patch_expiration(&Patch::Set("not-a-date".into())).is_err());
}
#[test]
fn patch_user_request_deserializes_mixed_states() {
let raw = r#"{
"secret": "00112233445566778899aabbccddeeff",
"max_tcp_conns": 0,
"max_unique_ips": null,
"data_quota_bytes": 1024
}"#;
let req: PatchUserRequest = serde_json::from_str(raw).expect("valid json");
assert_eq!(
req.secret.as_deref(),
Some("00112233445566778899aabbccddeeff")
);
assert!(matches!(req.max_tcp_conns, Patch::Set(0)));
assert!(matches!(req.max_unique_ips, Patch::Remove));
assert!(matches!(req.data_quota_bytes, Patch::Set(1024)));
assert!(matches!(req.expiration_rfc3339, Patch::Unchanged));
assert!(matches!(req.user_ad_tag, Patch::Unchanged));
}
}

View File

@@ -7,8 +7,8 @@ use crate::transport::upstream::IpPreference;
use super::ApiShared;
use super::model::{
DcEndpointWriters, DcStatus, DcStatusData, MeWriterStatus, MeWritersData, MeWritersSummary,
MinimalAllData, MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData,
ClassCount, DcEndpointWriters, DcStatus, DcStatusData, MeWriterStatus, MeWritersData,
MeWritersSummary, MinimalAllData, MinimalAllPayload, MinimalDcPathData, MinimalMeRuntimeData,
MinimalQuarantineData, UpstreamDcStatus, UpstreamStatus, UpstreamSummaryData, UpstreamsData,
ZeroAllData, ZeroCodeCount, ZeroCoreData, ZeroDesyncData, ZeroMiddleProxyData, ZeroPoolData,
ZeroUpstreamData,
@@ -26,6 +26,16 @@ pub(crate) struct MinimalCacheEntry {
pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> ZeroAllData {
let telemetry = stats.telemetry_policy();
let bad_connection_classes = stats
.get_connects_bad_class_counts()
.into_iter()
.map(|(class, total)| ClassCount { class, total })
.collect();
let handshake_failure_classes = stats
.get_handshake_failure_class_counts()
.into_iter()
.map(|(class, total)| ClassCount { class, total })
.collect();
let handshake_error_codes = stats
.get_me_handshake_error_code_counts()
.into_iter()
@@ -38,6 +48,8 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
uptime_seconds: stats.uptime_secs(),
connections_total: stats.get_connects_all(),
connections_bad_total: stats.get_connects_bad(),
connections_bad_by_class: bad_connection_classes,
handshake_failures_by_class: handshake_failure_classes,
handshake_timeouts_total: stats.get_handshake_timeouts(),
accept_permit_timeout_total: stats.get_accept_permit_timeout_total(),
configured_users,

View File

@@ -8,14 +8,15 @@ use crate::stats::Stats;
use super::ApiShared;
use super::config_store::{
AccessSection, ensure_expected_revision, load_config_from_disk, save_access_sections_to_disk,
save_config_to_disk,
AccessSection, current_revision, ensure_expected_revision, load_config_from_disk,
save_access_sections_to_disk,
};
use super::model::{
ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest,
UserInfo, UserLinks, is_valid_ad_tag, is_valid_user_secret, is_valid_username,
parse_optional_expiration, random_user_secret,
TlsDomainLink, UserInfo, UserLinks, is_valid_ad_tag, is_valid_user_secret, is_valid_username,
parse_optional_expiration, parse_patch_expiration, random_user_secret,
};
use super::patch::Patch;
pub(super) async fn create_user(
body: CreateUserRequest,
@@ -175,6 +176,13 @@ pub(super) async fn patch_user(
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(UserInfo, String), ApiFailure> {
let touches_users = body.secret.is_some();
let touches_user_ad_tags = !matches!(&body.user_ad_tag, Patch::Unchanged);
let touches_user_max_tcp_conns = !matches!(&body.max_tcp_conns, Patch::Unchanged);
let touches_user_expirations = !matches!(&body.expiration_rfc3339, Patch::Unchanged);
let touches_user_data_quota = !matches!(&body.data_quota_bytes, Patch::Unchanged);
let touches_user_max_unique_ips = !matches!(&body.max_unique_ips, Patch::Unchanged);
if let Some(secret) = body.secret.as_ref()
&& !is_valid_user_secret(secret)
{
@@ -182,14 +190,14 @@ pub(super) async fn patch_user(
"secret must be exactly 32 hex characters",
));
}
if let Some(ad_tag) = body.user_ad_tag.as_ref()
if let Patch::Set(ad_tag) = &body.user_ad_tag
&& !is_valid_ad_tag(ad_tag)
{
return Err(ApiFailure::bad_request(
"user_ad_tag must be exactly 32 hex characters",
));
}
let expiration = parse_optional_expiration(body.expiration_rfc3339.as_deref())?;
let expiration = parse_patch_expiration(&body.expiration_rfc3339)?;
let _guard = shared.mutation_lock.lock().await;
let mut cfg = load_config_from_disk(&shared.config_path).await?;
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
@@ -205,38 +213,95 @@ pub(super) async fn patch_user(
if let Some(secret) = body.secret {
cfg.access.users.insert(user.to_string(), secret);
}
if let Some(ad_tag) = body.user_ad_tag {
cfg.access.user_ad_tags.insert(user.to_string(), ad_tag);
match body.user_ad_tag {
Patch::Unchanged => {}
Patch::Remove => {
cfg.access.user_ad_tags.remove(user);
}
Patch::Set(ad_tag) => {
cfg.access.user_ad_tags.insert(user.to_string(), ad_tag);
}
}
if let Some(limit) = body.max_tcp_conns {
cfg.access
.user_max_tcp_conns
.insert(user.to_string(), limit);
match body.max_tcp_conns {
Patch::Unchanged => {}
Patch::Remove => {
cfg.access.user_max_tcp_conns.remove(user);
}
Patch::Set(limit) => {
cfg.access
.user_max_tcp_conns
.insert(user.to_string(), limit);
}
}
if let Some(expiration) = expiration {
cfg.access
.user_expirations
.insert(user.to_string(), expiration);
match expiration {
Patch::Unchanged => {}
Patch::Remove => {
cfg.access.user_expirations.remove(user);
}
Patch::Set(expiration) => {
cfg.access
.user_expirations
.insert(user.to_string(), expiration);
}
}
if let Some(quota) = body.data_quota_bytes {
cfg.access.user_data_quota.insert(user.to_string(), quota);
}
let mut updated_limit = None;
if let Some(limit) = body.max_unique_ips {
cfg.access
.user_max_unique_ips
.insert(user.to_string(), limit);
updated_limit = Some(limit);
match body.data_quota_bytes {
Patch::Unchanged => {}
Patch::Remove => {
cfg.access.user_data_quota.remove(user);
}
Patch::Set(quota) => {
cfg.access.user_data_quota.insert(user.to_string(), quota);
}
}
// Capture how the per-user IP limit changed, so the in-memory ip_tracker
// can be synced (set or removed) after the config is persisted.
let max_unique_ips_change = match body.max_unique_ips {
Patch::Unchanged => None,
Patch::Remove => {
cfg.access.user_max_unique_ips.remove(user);
Some(None)
}
Patch::Set(limit) => {
cfg.access
.user_max_unique_ips
.insert(user.to_string(), limit);
Some(Some(limit))
}
};
cfg.validate()
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
let revision = save_config_to_disk(&shared.config_path, &cfg).await?;
let mut touched_sections = Vec::new();
if touches_users {
touched_sections.push(AccessSection::Users);
}
if touches_user_ad_tags {
touched_sections.push(AccessSection::UserAdTags);
}
if touches_user_max_tcp_conns {
touched_sections.push(AccessSection::UserMaxTcpConns);
}
if touches_user_expirations {
touched_sections.push(AccessSection::UserExpirations);
}
if touches_user_data_quota {
touched_sections.push(AccessSection::UserDataQuota);
}
if touches_user_max_unique_ips {
touched_sections.push(AccessSection::UserMaxUniqueIps);
}
let revision = if touched_sections.is_empty() {
current_revision(&shared.config_path).await?
} else {
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?
};
drop(_guard);
if let Some(limit) = updated_limit {
shared.ip_tracker.set_user_limit(user, limit).await;
match max_unique_ips_change {
Some(Some(limit)) => shared.ip_tracker.set_user_limit(user, limit).await,
Some(None) => shared.ip_tracker.remove_user_limit(user).await,
None => {}
}
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
let users = users_from_config(
@@ -400,11 +465,7 @@ pub(super) async fn users_from_config(
.map(|secret| {
build_user_links(cfg, secret, startup_detected_ip_v4, startup_detected_ip_v6)
})
.unwrap_or(UserLinks {
classic: Vec::new(),
secure: Vec::new(),
tls: Vec::new(),
});
.unwrap_or_else(empty_user_links);
users.push(UserInfo {
in_runtime: runtime_cfg
.map(|runtime| runtime.access.users.contains_key(&username))
@@ -445,6 +506,15 @@ pub(super) async fn users_from_config(
users
}
fn empty_user_links() -> UserLinks {
UserLinks {
classic: Vec::new(),
secure: Vec::new(),
tls: Vec::new(),
tls_domains: Vec::new(),
}
}
fn build_user_links(
cfg: &ProxyConfig,
secret: &str,
@@ -458,10 +528,12 @@ fn build_user_links(
.public_port
.unwrap_or(resolve_default_link_port(cfg));
let tls_domains = resolve_tls_domains(cfg);
let extra_tls_domains = resolve_extra_tls_domains(cfg);
let mut classic = Vec::new();
let mut secure = Vec::new();
let mut tls = Vec::new();
let mut tls_domain_links = Vec::new();
for host in &hosts {
if cfg.general.modes.classic {
@@ -484,6 +556,17 @@ fn build_user_links(
host, port, secret, domain_hex
));
}
for domain in &extra_tls_domains {
let domain_hex = hex::encode(domain);
let link = format!(
"tg://proxy?server={}&port={}&secret=ee{}{}",
host, port, secret, domain_hex
);
tls_domain_links.push(TlsDomainLink {
domain: (*domain).to_string(),
link,
});
}
}
}
@@ -491,6 +574,7 @@ fn build_user_links(
classic,
secure,
tls,
tls_domains: tls_domain_links,
}
}
@@ -607,6 +691,19 @@ fn resolve_tls_domains(cfg: &ProxyConfig) -> Vec<&str> {
domains
}
fn resolve_extra_tls_domains(cfg: &ProxyConfig) -> Vec<&str> {
let mut domains = Vec::with_capacity(cfg.censorship.tls_domains.len());
let primary = cfg.censorship.tls_domain.as_str();
for domain in &cfg.censorship.tls_domains {
let value = domain.as_str();
if value.is_empty() || value == primary || domains.contains(&value) {
continue;
}
domains.push(value);
}
domains
}
#[cfg(test)]
mod tests {
use super::*;
@@ -696,4 +793,80 @@ mod tests {
assert!(alice.in_runtime);
assert!(!bob.in_runtime);
}
#[tokio::test]
async fn users_from_config_returns_tls_link_for_each_tls_domain() {
let mut cfg = ProxyConfig::default();
cfg.access.users.insert(
"alice".to_string(),
"0123456789abcdef0123456789abcdef".to_string(),
);
cfg.general.modes.classic = false;
cfg.general.modes.secure = false;
cfg.general.modes.tls = true;
cfg.general.links.public_host = Some("proxy.example.net".to_string());
cfg.general.links.public_port = Some(443);
cfg.censorship.tls_domain = "front-a.example.com".to_string();
cfg.censorship.tls_domains = vec![
"front-b.example.com".to_string(),
"front-c.example.com".to_string(),
"front-b.example.com".to_string(),
"front-a.example.com".to_string(),
];
let stats = Stats::new();
let tracker = UserIpTracker::new();
let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await;
let alice = users
.iter()
.find(|entry| entry.username == "alice")
.expect("alice must be present");
assert_eq!(alice.links.tls.len(), 3);
assert!(
alice
.links
.tls
.iter()
.any(|link| link.ends_with(&hex::encode("front-a.example.com")))
);
assert!(
alice
.links
.tls
.iter()
.any(|link| link.ends_with(&hex::encode("front-b.example.com")))
);
assert!(
alice
.links
.tls
.iter()
.any(|link| link.ends_with(&hex::encode("front-c.example.com")))
);
assert_eq!(alice.links.tls_domains.len(), 2);
assert!(
alice
.links
.tls_domains
.iter()
.any(|entry| entry.domain == "front-b.example.com"
&& entry.link.ends_with(&hex::encode("front-b.example.com")))
);
assert!(
alice
.links
.tls_domains
.iter()
.any(|entry| entry.domain == "front-c.example.com"
&& entry.link.ends_with(&hex::encode("front-c.example.com")))
);
assert!(
!alice
.links
.tls_domains
.iter()
.any(|entry| entry.domain == "front-a.example.com")
);
}
}

View File

@@ -6,12 +6,15 @@
//! - `reload [--pid-file PATH]` - Reload configuration (SIGHUP)
//! - `status [--pid-file PATH]` - Check daemon status
//! - `run [OPTIONS] [config.toml]` - Run in foreground (default behavior)
//! - `healthcheck [OPTIONS] [config.toml]` - Run control-plane health probe
use rand::RngExt;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use crate::healthcheck::{self, HealthcheckMode};
#[cfg(unix)]
use crate::daemon::{self, DEFAULT_PID_FILE, DaemonOptions};
@@ -28,6 +31,8 @@ pub enum Subcommand {
Reload,
/// Check daemon status (`status` subcommand).
Status,
/// Run health probe and exit with status code.
Healthcheck,
/// Fire-and-forget setup (`--init`).
Init,
}
@@ -38,6 +43,8 @@ pub struct ParsedCommand {
pub subcommand: Subcommand,
pub pid_file: PathBuf,
pub config_path: String,
pub healthcheck_mode: HealthcheckMode,
pub healthcheck_mode_invalid: Option<String>,
#[cfg(unix)]
pub daemon_opts: DaemonOptions,
pub init_opts: Option<InitOptions>,
@@ -52,6 +59,8 @@ impl Default for ParsedCommand {
#[cfg(not(unix))]
pid_file: PathBuf::from("/var/run/telemt.pid"),
config_path: "config.toml".to_string(),
healthcheck_mode: HealthcheckMode::Liveness,
healthcheck_mode_invalid: None,
#[cfg(unix)]
daemon_opts: DaemonOptions::default(),
init_opts: None,
@@ -91,6 +100,9 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
"status" => {
cmd.subcommand = Subcommand::Status;
}
"healthcheck" => {
cmd.subcommand = Subcommand::Healthcheck;
}
"run" => {
cmd.subcommand = Subcommand::Run;
#[cfg(unix)]
@@ -113,7 +125,35 @@ pub fn parse_command(args: &[String]) -> ParsedCommand {
while i < args.len() {
match args[i].as_str() {
// Skip subcommand names
"start" | "stop" | "reload" | "status" | "run" => {}
"start" | "stop" | "reload" | "status" | "run" | "healthcheck" => {}
"--mode" => {
i += 1;
if i < args.len() {
match HealthcheckMode::from_cli_arg(&args[i]) {
Some(mode) => {
cmd.healthcheck_mode = mode;
cmd.healthcheck_mode_invalid = None;
}
None => {
cmd.healthcheck_mode_invalid = Some(args[i].clone());
}
}
} else {
cmd.healthcheck_mode_invalid = Some(String::new());
}
}
s if s.starts_with("--mode=") => {
let raw = s.trim_start_matches("--mode=");
match HealthcheckMode::from_cli_arg(raw) {
Some(mode) => {
cmd.healthcheck_mode = mode;
cmd.healthcheck_mode_invalid = None;
}
None => {
cmd.healthcheck_mode_invalid = Some(raw.to_string());
}
}
}
// PID file option (for stop/reload/status)
"--pid-file" => {
i += 1;
@@ -152,6 +192,20 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
Subcommand::Stop => Some(cmd_stop(&cmd.pid_file)),
Subcommand::Reload => Some(cmd_reload(&cmd.pid_file)),
Subcommand::Status => Some(cmd_status(&cmd.pid_file)),
Subcommand::Healthcheck => {
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
if invalid_mode.is_empty() {
eprintln!("[telemt] Missing value for --mode (supported: liveness, ready)");
} else {
eprintln!(
"[telemt] Invalid --mode value '{invalid_mode}' (supported: liveness, ready)"
);
}
Some(2)
} else {
Some(healthcheck::run(&cmd.config_path, cmd.healthcheck_mode))
}
}
Subcommand::Init => {
if let Some(opts) = cmd.init_opts.clone() {
match run_init(opts) {
@@ -177,6 +231,20 @@ pub fn execute_subcommand(cmd: &ParsedCommand) -> Option<i32> {
eprintln!("[telemt] Subcommand not supported on this platform");
Some(1)
}
Subcommand::Healthcheck => {
if let Some(invalid_mode) = cmd.healthcheck_mode_invalid.as_ref() {
if invalid_mode.is_empty() {
eprintln!("[telemt] Missing value for --mode (supported: liveness, ready)");
} else {
eprintln!(
"[telemt] Invalid --mode value '{invalid_mode}' (supported: liveness, ready)"
);
}
Some(2)
} else {
Some(healthcheck::run(&cmd.config_path, cmd.healthcheck_mode))
}
}
Subcommand::Init => {
if let Some(opts) = cmd.init_opts.clone() {
match run_init(opts) {
@@ -621,6 +689,7 @@ tls_domain = "{domain}"
mask = true
mask_port = 443
fake_cert_len = 2048
serverhello_compact = false
tls_full_cert_ttl_secs = 90
[access]

View File

@@ -21,6 +21,8 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_PER_CORE: u16 = 64;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_PER_CORE: u16 = 64;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_ACTIVE_WRITERS_GLOBAL: u32 = 256;
const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256;
const DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED: bool = false;
const DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED: bool = false;
const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 4096;
const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 768;
const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024;
@@ -100,7 +102,7 @@ pub(crate) fn default_fake_cert_len() -> usize {
}
pub(crate) fn default_tls_front_dir() -> String {
"/etc/telemt/tlsfront".to_string()
"tlsfront".to_string()
}
pub(crate) fn default_replay_check_len() -> usize {
@@ -529,6 +531,14 @@ pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
25
}
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
}
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
}
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
120
}
@@ -558,13 +568,17 @@ pub(crate) fn default_beobachten_flush_secs() -> u64 {
}
pub(crate) fn default_beobachten_file() -> String {
"/etc/telemt/beobachten.txt".to_string()
"beobachten.txt".to_string()
}
pub(crate) fn default_tls_new_session_tickets() -> u8 {
0
}
pub(crate) fn default_serverhello_compact() -> bool {
false
}
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
90
}

View File

@@ -86,6 +86,8 @@ pub struct HotFields {
pub telemetry_user_enabled: bool,
pub telemetry_me_level: MeTelemetryLevel,
pub me_socks_kdf_policy: MeSocksKdfPolicy,
pub me_route_backpressure_enabled: bool,
pub me_route_fairshare_enabled: bool,
pub me_floor_mode: MeFloorMode,
pub me_adaptive_floor_idle_secs: u64,
pub me_adaptive_floor_min_writers_single_endpoint: u8,
@@ -187,6 +189,8 @@ impl HotFields {
telemetry_user_enabled: cfg.general.telemetry.user_enabled,
telemetry_me_level: cfg.general.telemetry.me_level,
me_socks_kdf_policy: cfg.general.me_socks_kdf_policy,
me_route_backpressure_enabled: cfg.general.me_route_backpressure_enabled,
me_route_fairshare_enabled: cfg.general.me_route_fairshare_enabled,
me_floor_mode: cfg.general.me_floor_mode,
me_adaptive_floor_idle_secs: cfg.general.me_adaptive_floor_idle_secs,
me_adaptive_floor_min_writers_single_endpoint: cfg
@@ -529,6 +533,8 @@ fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyConfig {
new.general.me_route_backpressure_high_timeout_ms;
cfg.general.me_route_backpressure_high_watermark_pct =
new.general.me_route_backpressure_high_watermark_pct;
cfg.general.me_route_backpressure_enabled = new.general.me_route_backpressure_enabled;
cfg.general.me_route_fairshare_enabled = new.general.me_route_fairshare_enabled;
cfg.general.me_reader_route_data_wait_ms = new.general.me_reader_route_data_wait_ms;
cfg.general.me_d2c_flush_batch_max_frames = new.general.me_d2c_flush_batch_max_frames;
cfg.general.me_d2c_flush_batch_max_bytes = new.general.me_d2c_flush_batch_max_bytes;
@@ -618,6 +624,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|| old.censorship.server_hello_delay_min_ms != new.censorship.server_hello_delay_min_ms
|| old.censorship.server_hello_delay_max_ms != new.censorship.server_hello_delay_max_ms
|| old.censorship.tls_new_session_tickets != new.censorship.tls_new_session_tickets
|| old.censorship.serverhello_compact != new.censorship.serverhello_compact
|| old.censorship.tls_full_cert_ttl_secs != new.censorship.tls_full_cert_ttl_secs
|| old.censorship.alpn_enforce != new.censorship.alpn_enforce
|| old.censorship.mask_proxy_protocol != new.censorship.mask_proxy_protocol
@@ -1053,6 +1060,8 @@ fn log_changes(
!= new_hot.me_route_backpressure_high_timeout_ms
|| old_hot.me_route_backpressure_high_watermark_pct
!= new_hot.me_route_backpressure_high_watermark_pct
|| old_hot.me_route_backpressure_enabled != new_hot.me_route_backpressure_enabled
|| old_hot.me_route_fairshare_enabled != new_hot.me_route_fairshare_enabled
|| old_hot.me_reader_route_data_wait_ms != new_hot.me_reader_route_data_wait_ms
|| old_hot.me_health_interval_ms_unhealthy != new_hot.me_health_interval_ms_unhealthy
|| old_hot.me_health_interval_ms_healthy != new_hot.me_health_interval_ms_healthy
@@ -1060,10 +1069,12 @@ fn log_changes(
|| old_hot.me_warn_rate_limit_ms != new_hot.me_warn_rate_limit_ms
{
info!(
"config reload: me_route_backpressure: base={}ms high={}ms watermark={}%; me_reader_route_data_wait_ms={}; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms",
"config reload: me_route_backpressure: enabled={} base={}ms high={}ms watermark={}%; me_route_fairshare_enabled={}; me_reader_route_data_wait_ms={}; me_health_interval: unhealthy={}ms healthy={}ms; me_admission_poll={}ms; me_warn_rate_limit={}ms",
new_hot.me_route_backpressure_enabled,
new_hot.me_route_backpressure_base_timeout_ms,
new_hot.me_route_backpressure_high_timeout_ms,
new_hot.me_route_backpressure_high_watermark_pct,
new_hot.me_route_fairshare_enabled,
new_hot.me_reader_route_data_wait_ms,
new_hot.me_health_interval_ms_unhealthy,
new_hot.me_health_interval_ms_healthy,

View File

@@ -22,6 +22,672 @@ const MAX_ME_ROUTE_CHANNEL_CAPACITY: usize = 8_192;
const MAX_ME_C2ME_CHANNEL_CAPACITY: usize = 8_192;
const MIN_MAX_CLIENT_FRAME_BYTES: usize = 4 * 1024;
const MAX_MAX_CLIENT_FRAME_BYTES: usize = 16 * 1024 * 1024;
const MAX_API_REQUEST_BODY_LIMIT_BYTES: usize = 1024 * 1024;
fn is_valid_tls_domain_name(domain: &str) -> bool {
!domain.is_empty()
&& !domain
.chars()
.any(|ch| ch.is_whitespace() || matches!(ch, '/' | '\\'))
}
const TOP_LEVEL_CONFIG_KEYS: &[&str] = &[
"general",
"network",
"server",
"timeouts",
"censorship",
"access",
"upstreams",
"show_link",
"dc_overrides",
"default_dc",
"beobachten",
"beobachten_minutes",
"beobachten_flush_secs",
"beobachten_file",
"include",
];
const GENERAL_CONFIG_KEYS: &[&str] = &[
"data_path",
"quota_state_path",
"config_strict",
"modes",
"prefer_ipv6",
"fast_mode",
"use_middle_proxy",
"proxy_secret_path",
"proxy_secret_url",
"proxy_config_v4_cache_path",
"proxy_config_v4_url",
"proxy_config_v6_cache_path",
"proxy_config_v6_url",
"ad_tag",
"middle_proxy_nat_ip",
"middle_proxy_nat_probe",
"middle_proxy_nat_stun",
"middle_proxy_nat_stun_servers",
"stun_nat_probe_concurrency",
"middle_proxy_pool_size",
"middle_proxy_warm_standby",
"me_init_retry_attempts",
"me2dc_fallback",
"me2dc_fast",
"me_keepalive_enabled",
"me_keepalive_interval_secs",
"me_keepalive_jitter_secs",
"me_keepalive_payload_random",
"rpc_proxy_req_every",
"me_writer_cmd_channel_capacity",
"me_route_channel_capacity",
"me_c2me_channel_capacity",
"me_c2me_send_timeout_ms",
"me_reader_route_data_wait_ms",
"me_d2c_flush_batch_max_frames",
"me_d2c_flush_batch_max_bytes",
"me_d2c_flush_batch_max_delay_us",
"me_d2c_ack_flush_immediate",
"me_quota_soft_overshoot_bytes",
"me_d2c_frame_buf_shrink_threshold_bytes",
"direct_relay_copy_buf_c2s_bytes",
"direct_relay_copy_buf_s2c_bytes",
"crypto_pending_buffer",
"max_client_frame",
"desync_all_full",
"beobachten",
"beobachten_minutes",
"beobachten_flush_secs",
"beobachten_file",
"hardswap",
"me_warmup_stagger_enabled",
"me_warmup_step_delay_ms",
"me_warmup_step_jitter_ms",
"me_reconnect_max_concurrent_per_dc",
"me_reconnect_backoff_base_ms",
"me_reconnect_backoff_cap_ms",
"me_reconnect_fast_retry_count",
"me_single_endpoint_shadow_writers",
"me_single_endpoint_outage_mode_enabled",
"me_single_endpoint_outage_disable_quarantine",
"me_single_endpoint_outage_backoff_min_ms",
"me_single_endpoint_outage_backoff_max_ms",
"me_single_endpoint_shadow_rotate_every_secs",
"me_floor_mode",
"me_adaptive_floor_idle_secs",
"me_adaptive_floor_min_writers_single_endpoint",
"me_adaptive_floor_min_writers_multi_endpoint",
"me_adaptive_floor_recover_grace_secs",
"me_adaptive_floor_writers_per_core_total",
"me_adaptive_floor_cpu_cores_override",
"me_adaptive_floor_max_extra_writers_single_per_core",
"me_adaptive_floor_max_extra_writers_multi_per_core",
"me_adaptive_floor_max_active_writers_per_core",
"me_adaptive_floor_max_warm_writers_per_core",
"me_adaptive_floor_max_active_writers_global",
"me_adaptive_floor_max_warm_writers_global",
"upstream_connect_retry_attempts",
"upstream_connect_retry_backoff_ms",
"upstream_connect_budget_ms",
"tg_connect",
"upstream_unhealthy_fail_threshold",
"upstream_connect_failfast_hard_errors",
"stun_iface_mismatch_ignore",
"unknown_dc_log_path",
"unknown_dc_file_log_enabled",
"log_level",
"disable_colors",
"telemetry",
"me_socks_kdf_policy",
"me_route_backpressure_enabled",
"me_route_fairshare_enabled",
"me_route_backpressure_base_timeout_ms",
"me_route_backpressure_high_timeout_ms",
"me_route_backpressure_high_watermark_pct",
"me_health_interval_ms_unhealthy",
"me_health_interval_ms_healthy",
"me_admission_poll_ms",
"me_warn_rate_limit_ms",
"me_route_no_writer_mode",
"me_route_no_writer_wait_ms",
"me_route_hybrid_max_wait_ms",
"me_route_blocking_send_timeout_ms",
"me_route_inline_recovery_attempts",
"me_route_inline_recovery_wait_ms",
"links",
"fast_mode_min_tls_record",
"update_every",
"me_reinit_every_secs",
"me_hardswap_warmup_delay_min_ms",
"me_hardswap_warmup_delay_max_ms",
"me_hardswap_warmup_extra_passes",
"me_hardswap_warmup_pass_backoff_base_ms",
"me_config_stable_snapshots",
"me_config_apply_cooldown_secs",
"me_snapshot_require_http_2xx",
"me_snapshot_reject_empty_map",
"me_snapshot_min_proxy_for_lines",
"proxy_secret_stable_snapshots",
"proxy_secret_rotate_runtime",
"me_secret_atomic_snapshot",
"proxy_secret_len_max",
"me_pool_drain_ttl_secs",
"me_instadrain",
"me_pool_drain_threshold",
"me_pool_drain_soft_evict_enabled",
"me_pool_drain_soft_evict_grace_secs",
"me_pool_drain_soft_evict_per_writer",
"me_pool_drain_soft_evict_budget_per_core",
"me_pool_drain_soft_evict_cooldown_ms",
"me_bind_stale_mode",
"me_bind_stale_ttl_secs",
"me_pool_min_fresh_ratio",
"me_reinit_drain_timeout_secs",
"proxy_secret_auto_reload_secs",
"proxy_config_auto_reload_secs",
"me_reinit_singleflight",
"me_reinit_trigger_channel",
"me_reinit_coalesce_window_ms",
"me_deterministic_writer_sort",
"me_writer_pick_mode",
"me_writer_pick_sample_size",
"ntp_check",
"ntp_servers",
"auto_degradation_enabled",
"degradation_min_unavailable_dc_groups",
"rst_on_close",
];
const NETWORK_CONFIG_KEYS: &[&str] = &[
"ipv4",
"ipv6",
"prefer",
"multipath",
"stun_use",
"stun_servers",
"stun_tcp_fallback",
"http_ip_detect_urls",
"cache_public_ip_path",
"dns_overrides",
];
const SERVER_CONFIG_KEYS: &[&str] = &[
"port",
"listen_addr_ipv4",
"listen_addr_ipv6",
"listen_unix_sock",
"listen_unix_sock_perm",
"listen_tcp",
"proxy_protocol",
"proxy_protocol_header_timeout_ms",
"proxy_protocol_trusted_cidrs",
"metrics_port",
"metrics_listen",
"metrics_whitelist",
"api",
"admin_api",
"listeners",
"listen_backlog",
"max_connections",
"accept_permit_timeout_ms",
"conntrack_control",
];
const API_CONFIG_KEYS: &[&str] = &[
"enabled",
"listen",
"whitelist",
"gray_action",
"auth_header",
"request_body_limit_bytes",
"minimal_runtime_enabled",
"minimal_runtime_cache_ttl_ms",
"runtime_edge_enabled",
"runtime_edge_cache_ttl_ms",
"runtime_edge_top_n",
"runtime_edge_events_capacity",
"read_only",
];
const CONNTRACK_CONTROL_CONFIG_KEYS: &[&str] = &[
"inline_conntrack_control",
"mode",
"backend",
"profile",
"hybrid_listener_ips",
"pressure_high_watermark_pct",
"pressure_low_watermark_pct",
"delete_budget_per_sec",
];
const LISTENER_CONFIG_KEYS: &[&str] = &[
"ip",
"port",
"announce",
"announce_ip",
"proxy_protocol",
"reuse_allow",
];
const TIMEOUTS_CONFIG_KEYS: &[&str] = &[
"client_first_byte_idle_secs",
"client_handshake",
"relay_idle_policy_v2_enabled",
"relay_client_idle_soft_secs",
"relay_client_idle_hard_secs",
"relay_idle_grace_after_downstream_activity_secs",
"client_keepalive",
"client_ack",
"me_one_retry",
"me_one_timeout_ms",
];
const CENSORSHIP_CONFIG_KEYS: &[&str] = &[
"tls_domain",
"tls_domains",
"unknown_sni_action",
"tls_fetch_scope",
"tls_fetch",
"mask",
"mask_host",
"mask_port",
"mask_unix_sock",
"fake_cert_len",
"tls_emulation",
"tls_front_dir",
"server_hello_delay_min_ms",
"server_hello_delay_max_ms",
"tls_new_session_tickets",
"serverhello_compact",
"tls_full_cert_ttl_secs",
"alpn_enforce",
"mask_proxy_protocol",
"mask_shape_hardening",
"mask_shape_hardening_aggressive_mode",
"mask_shape_bucket_floor_bytes",
"mask_shape_bucket_cap_bytes",
"mask_shape_above_cap_blur",
"mask_shape_above_cap_blur_max_bytes",
"mask_relay_max_bytes",
"mask_relay_timeout_ms",
"mask_relay_idle_timeout_ms",
"mask_classifier_prefetch_timeout_ms",
"mask_timing_normalization_enabled",
"mask_timing_normalization_floor_ms",
"mask_timing_normalization_ceiling_ms",
];
const TLS_FETCH_CONFIG_KEYS: &[&str] = &[
"profiles",
"strict_route",
"attempt_timeout_ms",
"total_budget_ms",
"grease_enabled",
"deterministic",
"profile_cache_ttl_secs",
];
const ACCESS_CONFIG_KEYS: &[&str] = &[
"users",
"user_ad_tags",
"user_max_tcp_conns",
"user_max_tcp_conns_global_each",
"user_expirations",
"user_data_quota",
"user_rate_limits",
"cidr_rate_limits",
"user_max_unique_ips",
"user_max_unique_ips_global_each",
"user_max_unique_ips_mode",
"user_max_unique_ips_window_secs",
"replay_check_len",
"replay_window_secs",
"ignore_time_skew",
];
const RATE_LIMIT_BPS_CONFIG_KEYS: &[&str] = &["up_bps", "down_bps"];
const UPSTREAM_CONFIG_KEYS: &[&str] = &[
"type",
"interface",
"bind_addresses",
"bindtodevice",
"force_bind",
"address",
"user_id",
"username",
"password",
"url",
"weight",
"enabled",
"scopes",
"ipv4",
"ipv6",
];
const PROXY_MODES_CONFIG_KEYS: &[&str] = &["classic", "secure", "tls"];
const TELEMETRY_CONFIG_KEYS: &[&str] = &["core_enabled", "user_enabled", "me_level"];
const LINKS_CONFIG_KEYS: &[&str] = &["show", "public_host", "public_port"];
#[derive(Debug)]
struct UnknownConfigKey {
path: String,
suggestion: Option<String>,
}
fn table_at<'a>(value: &'a toml::Value, path: &[&str]) -> Option<&'a toml::Table> {
let mut current = value;
for segment in path {
current = current.get(*segment)?;
}
current.as_table()
}
fn is_strict_config(parsed_toml: &toml::Value) -> bool {
table_at(parsed_toml, &["general"])
.and_then(|table| table.get("config_strict"))
.and_then(toml::Value::as_bool)
.unwrap_or(false)
}
fn known_config_keys_for_suggestion() -> Vec<&'static str> {
let mut keys = Vec::new();
for group in [
TOP_LEVEL_CONFIG_KEYS,
GENERAL_CONFIG_KEYS,
NETWORK_CONFIG_KEYS,
SERVER_CONFIG_KEYS,
API_CONFIG_KEYS,
CONNTRACK_CONTROL_CONFIG_KEYS,
LISTENER_CONFIG_KEYS,
TIMEOUTS_CONFIG_KEYS,
CENSORSHIP_CONFIG_KEYS,
TLS_FETCH_CONFIG_KEYS,
ACCESS_CONFIG_KEYS,
RATE_LIMIT_BPS_CONFIG_KEYS,
UPSTREAM_CONFIG_KEYS,
PROXY_MODES_CONFIG_KEYS,
TELEMETRY_CONFIG_KEYS,
LINKS_CONFIG_KEYS,
] {
keys.extend_from_slice(group);
}
keys
}
fn levenshtein_distance(a: &str, b: &str) -> usize {
let b_chars: Vec<char> = b.chars().collect();
let mut prev: Vec<usize> = (0..=b_chars.len()).collect();
let mut curr = vec![0usize; b_chars.len() + 1];
for (i, ca) in a.chars().enumerate() {
curr[0] = i + 1;
for (j, cb) in b_chars.iter().enumerate() {
let replace = if ca == *cb { prev[j] } else { prev[j] + 1 };
curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(replace);
}
std::mem::swap(&mut prev, &mut curr);
}
prev[b_chars.len()]
}
fn unknown_key_suggestion(key: &str, known_keys: &[&'static str]) -> Option<String> {
let normalized = key.to_ascii_lowercase();
let mut best: Option<(&str, usize)> = None;
for known in known_keys {
let distance = levenshtein_distance(&normalized, known);
let is_better = match best {
Some((_, best_distance)) => distance < best_distance,
None => true,
};
if distance <= 4 && is_better {
best = Some((known, distance));
}
}
best.map(|(known, _)| known.to_string())
}
fn push_unknown_keys(
unknown: &mut Vec<UnknownConfigKey>,
known_for_suggestion: &[&'static str],
path: &str,
table: &toml::Table,
allowed: &[&str],
) {
for key in table.keys() {
if !allowed.contains(&key.as_str()) {
let full_path = if path.is_empty() {
key.clone()
} else {
format!("{path}.{key}")
};
unknown.push(UnknownConfigKey {
path: full_path,
suggestion: unknown_key_suggestion(key, known_for_suggestion),
});
}
}
}
fn check_known_table(
parsed_toml: &toml::Value,
unknown: &mut Vec<UnknownConfigKey>,
known_for_suggestion: &[&'static str],
path: &[&str],
allowed: &[&str],
) {
if let Some(table) = table_at(parsed_toml, path) {
push_unknown_keys(
unknown,
known_for_suggestion,
&path.join("."),
table,
allowed,
);
}
}
fn check_nested_table_value(
unknown: &mut Vec<UnknownConfigKey>,
known_for_suggestion: &[&'static str],
path: String,
value: &toml::Value,
allowed: &[&str],
) {
if let Some(table) = value.as_table() {
push_unknown_keys(unknown, known_for_suggestion, &path, table, allowed);
}
}
fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<UnknownConfigKey> {
let known_for_suggestion = known_config_keys_for_suggestion();
let mut unknown = Vec::new();
if let Some(root) = parsed_toml.as_table() {
push_unknown_keys(
&mut unknown,
&known_for_suggestion,
"",
root,
TOP_LEVEL_CONFIG_KEYS,
);
}
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general"],
GENERAL_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general", "modes"],
PROXY_MODES_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general", "telemetry"],
TELEMETRY_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general", "links"],
LINKS_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["network"],
NETWORK_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server"],
SERVER_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server", "api"],
API_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server", "admin_api"],
API_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server", "conntrack_control"],
CONNTRACK_CONTROL_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["timeouts"],
TIMEOUTS_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["censorship"],
CENSORSHIP_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["censorship", "tls_fetch"],
TLS_FETCH_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["access"],
ACCESS_CONFIG_KEYS,
);
if let Some(listeners) = table_at(parsed_toml, &["server"])
.and_then(|table| table.get("listeners"))
.and_then(toml::Value::as_array)
{
for (idx, listener) in listeners.iter().enumerate() {
check_nested_table_value(
&mut unknown,
&known_for_suggestion,
format!("server.listeners[{idx}]"),
listener,
LISTENER_CONFIG_KEYS,
);
}
}
if let Some(upstreams) = parsed_toml.get("upstreams").and_then(toml::Value::as_array) {
for (idx, upstream) in upstreams.iter().enumerate() {
check_nested_table_value(
&mut unknown,
&known_for_suggestion,
format!("upstreams[{idx}]"),
upstream,
UPSTREAM_CONFIG_KEYS,
);
}
}
for access_map in ["user_rate_limits", "cidr_rate_limits"] {
if let Some(table) = table_at(parsed_toml, &["access"])
.and_then(|access| access.get(access_map))
.and_then(toml::Value::as_table)
{
for (entry_name, value) in table {
check_nested_table_value(
&mut unknown,
&known_for_suggestion,
format!("access.{access_map}.{entry_name}"),
value,
RATE_LIMIT_BPS_CONFIG_KEYS,
);
}
}
}
unknown
}
fn handle_unknown_config_keys(parsed_toml: &toml::Value) -> Result<()> {
let unknown = collect_unknown_config_keys(parsed_toml);
if unknown.is_empty() {
return Ok(());
}
for item in &unknown {
if let Some(suggestion) = item.suggestion.as_deref() {
warn!(
key = %item.path,
suggestion = %suggestion,
"Unknown config key ignored; did you mean the suggested key?"
);
} else {
warn!(key = %item.path, "Unknown config key ignored");
}
}
if is_strict_config(parsed_toml) {
let mut paths = Vec::with_capacity(unknown.len());
for item in unknown {
if let Some(suggestion) = item.suggestion {
paths.push(format!("{} (did you mean `{}`?)", item.path, suggestion));
} else {
paths.push(item.path);
}
}
return Err(ProxyError::Config(format!(
"unknown config keys are not allowed when general.config_strict=true: {}",
paths.join(", ")
)));
}
Ok(())
}
#[derive(Debug, Clone)]
pub(crate) struct LoadedConfig {
@@ -337,12 +1003,17 @@ impl ProxyConfig {
let parsed_toml: toml::Value =
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
handle_unknown_config_keys(&parsed_toml)?;
let general_table = parsed_toml
.get("general")
.and_then(|value| value.as_table());
let network_table = parsed_toml
.get("network")
.and_then(|value| value.as_table());
let server_table = parsed_toml.get("server").and_then(|value| value.as_table());
let conntrack_control_table = server_table
.and_then(|table| table.get("conntrack_control"))
.and_then(|value| value.as_table());
let update_every_is_explicit = general_table
.map(|table| table.contains_key("update_every"))
.unwrap_or(false);
@@ -372,10 +1043,17 @@ impl ProxyConfig {
let stun_servers_is_explicit = network_table
.map(|table| table.contains_key("stun_servers"))
.unwrap_or(false);
let inline_conntrack_control_is_explicit = conntrack_control_table
.map(|table| table.contains_key("inline_conntrack_control"))
.unwrap_or(false);
let mut config: ProxyConfig = parsed_toml
.try_into()
.map_err(|e| ProxyError::Config(e.to_string()))?;
config
.server
.conntrack_control
.inline_conntrack_control_explicit = inline_conntrack_control_is_explicit;
if !update_every_is_explicit && (legacy_secret_is_explicit || legacy_config_is_explicit) {
config.general.update_every = None;
@@ -629,12 +1307,6 @@ impl ProxyConfig {
));
}
if config.censorship.mask_relay_max_bytes == 0 {
return Err(ProxyError::Config(
"censorship.mask_relay_max_bytes must be > 0".to_string(),
));
}
if config.censorship.mask_relay_max_bytes > 67_108_864 {
return Err(ProxyError::Config(
"censorship.mask_relay_max_bytes must be <= 67108864".to_string(),
@@ -1082,9 +1754,9 @@ impl ProxyConfig {
));
}
if config.general.me_route_blocking_send_timeout_ms > 5000 {
if !(1..=5000).contains(&config.general.me_route_blocking_send_timeout_ms) {
return Err(ProxyError::Config(
"general.me_route_blocking_send_timeout_ms must be within [0, 5000]".to_string(),
"general.me_route_blocking_send_timeout_ms must be within [1, 5000]".to_string(),
));
}
@@ -1106,9 +1778,11 @@ impl ProxyConfig {
));
}
if config.server.api.request_body_limit_bytes == 0 {
if !(1..=MAX_API_REQUEST_BODY_LIMIT_BYTES)
.contains(&config.server.api.request_body_limit_bytes)
{
return Err(ProxyError::Config(
"server.api.request_body_limit_bytes must be > 0".to_string(),
"server.api.request_body_limit_bytes must be within [1, 1048576]".to_string(),
));
}
@@ -1436,13 +2110,22 @@ impl ProxyConfig {
return Err(ProxyError::Config("No modes enabled".to_string()));
}
if self.censorship.tls_domain.contains(' ') || self.censorship.tls_domain.contains('/') {
if !is_valid_tls_domain_name(&self.censorship.tls_domain) {
return Err(ProxyError::Config(format!(
"Invalid tls_domain: '{}'. Must be a valid domain name",
self.censorship.tls_domain
)));
}
for domain in &self.censorship.tls_domains {
if !is_valid_tls_domain_name(domain) {
return Err(ProxyError::Config(format!(
"Invalid tls_domains entry: '{}'. Must be a valid domain name",
domain
)));
}
}
for (user, tag) in &self.access.user_ad_tags {
let zeros = "00000000000000000000000000000000";
if !is_valid_ad_tag(tag) {
@@ -1881,6 +2564,43 @@ mod tests {
);
}
#[test]
fn conntrack_inline_explicit_flag_is_false_when_omitted() {
let cfg = load_config_from_temp_toml(
r#"
[general]
[network]
[server]
[server.conntrack_control]
[access]
"#,
);
assert!(
!cfg.server
.conntrack_control
.inline_conntrack_control_explicit
);
}
#[test]
fn conntrack_inline_explicit_flag_is_true_when_present() {
let cfg = load_config_from_temp_toml(
r#"
[general]
[network]
[server]
[server.conntrack_control]
inline_conntrack_control = true
[access]
"#,
);
assert!(
cfg.server
.conntrack_control
.inline_conntrack_control_explicit
);
}
#[test]
fn unknown_sni_action_parses_and_defaults_to_drop() {
let cfg_default: ProxyConfig = toml::from_str(
@@ -1929,6 +2649,22 @@ mod tests {
cfg_accept.censorship.unknown_sni_action,
UnknownSniAction::Accept
);
let cfg_reject: ProxyConfig = toml::from_str(
r#"
[server]
[general]
[network]
[access]
[censorship]
unknown_sni_action = "reject_handshake"
"#,
)
.unwrap();
assert_eq!(
cfg_reject.censorship.unknown_sni_action,
UnknownSniAction::RejectHandshake
);
}
#[test]
@@ -2544,6 +3280,26 @@ mod tests {
let _ = std::fs::remove_file(path);
}
#[test]
fn me_route_blocking_send_timeout_ms_zero_is_rejected() {
let toml = r#"
[general]
me_route_blocking_send_timeout_ms = 0
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_me_route_blocking_send_timeout_zero_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.me_route_blocking_send_timeout_ms must be within [1, 5000]"));
let _ = std::fs::remove_file(path);
}
#[test]
fn me_route_no_writer_mode_is_parsed() {
let toml = r#"

View File

@@ -238,7 +238,7 @@ mask_shape_above_cap_blur_max_bytes = 8
}
#[test]
fn load_rejects_zero_mask_relay_max_bytes() {
fn load_accepts_zero_mask_relay_max_bytes_as_unlimited() {
let path = write_temp_config(
r#"
[censorship]
@@ -246,12 +246,9 @@ mask_relay_max_bytes = 0
"#,
);
let err = ProxyConfig::load(&path).expect_err("mask_relay_max_bytes must be > 0");
let msg = err.to_string();
assert!(
msg.contains("censorship.mask_relay_max_bytes must be > 0"),
"error must explain non-zero relay cap invariant, got: {msg}"
);
let cfg = ProxyConfig::load(&path)
.expect("mask_relay_max_bytes=0 must be accepted as unlimited relay cap");
assert_eq!(cfg.censorship.mask_relay_max_bytes, 0);
remove_temp_config(&path);
}

View File

@@ -26,6 +26,10 @@ pub enum LogLevel {
Silent,
}
fn default_quota_state_path() -> PathBuf {
PathBuf::from("telemt.limit.json")
}
impl LogLevel {
/// Convert to tracing EnvFilter directive string.
pub fn to_filter_str(&self) -> &'static str {
@@ -375,6 +379,15 @@ pub struct GeneralConfig {
#[serde(default)]
pub data_path: Option<PathBuf>,
/// JSON state file for runtime per-user quota consumption.
#[serde(default = "default_quota_state_path")]
pub quota_state_path: PathBuf,
/// Reject unknown TOML config keys during load.
/// Startup fails fast; hot-reload rejects the new snapshot and keeps the current config.
#[serde(default)]
pub config_strict: bool,
#[serde(default)]
pub modes: ProxyModes,
@@ -392,14 +405,26 @@ pub struct GeneralConfig {
#[serde(default = "default_proxy_secret_path")]
pub proxy_secret_path: Option<String>,
/// Optional custom URL for infrastructure secret (https://core.telegram.org/getProxySecret if absent).
#[serde(default)]
pub proxy_secret_url: Option<String>,
/// Optional path to cache raw getProxyConfig (IPv4) snapshot for startup fallback.
#[serde(default = "default_proxy_config_v4_cache_path")]
pub proxy_config_v4_cache_path: Option<String>,
/// Optional custom URL for getProxyConfig (https://core.telegram.org/getProxyConfig if absent).
#[serde(default)]
pub proxy_config_v4_url: Option<String>,
/// Optional path to cache raw getProxyConfigV6 snapshot for startup fallback.
#[serde(default = "default_proxy_config_v6_cache_path")]
pub proxy_config_v6_cache_path: Option<String>,
/// Optional custom URL for getProxyConfigV6 (https://core.telegram.org/getProxyConfigV6 if absent).
#[serde(default)]
pub proxy_config_v6_url: Option<String>,
/// Global ad_tag (32 hex chars from @MTProxybot). Fallback when user has no per-user tag in access.user_ad_tags.
#[serde(default)]
pub ad_tag: Option<String>,
@@ -518,10 +543,17 @@ pub struct GeneralConfig {
pub me_d2c_frame_buf_shrink_threshold_bytes: usize,
/// Copy buffer size for client->DC direction in direct relay.
///
/// This is also the upper bound for one amortized upload rate-limit burst:
/// upload debt is settled before the next relay read instead of blocking
/// inside the completed read path.
#[serde(default = "default_direct_relay_copy_buf_c2s_bytes")]
pub direct_relay_copy_buf_c2s_bytes: usize,
/// Copy buffer size for DC->client direction in direct relay.
///
/// This bounds one direct download rate-limit grant because writes are
/// clipped to the currently available shaper budget.
#[serde(default = "default_direct_relay_copy_buf_s2c_bytes")]
pub direct_relay_copy_buf_s2c_bytes: usize,
@@ -717,6 +749,14 @@ pub struct GeneralConfig {
#[serde(default)]
pub me_socks_kdf_policy: MeSocksKdfPolicy,
/// Enable route-level ME backpressure controls in reader fairness path.
#[serde(default = "default_me_route_backpressure_enabled")]
pub me_route_backpressure_enabled: bool,
/// Enable worker-local fairshare scheduler for ME reader routing.
#[serde(default = "default_me_route_fairshare_enabled")]
pub me_route_fairshare_enabled: bool,
/// Base backpressure timeout in milliseconds for ME route channel send.
#[serde(default = "default_me_route_backpressure_base_timeout_ms")]
pub me_route_backpressure_base_timeout_ms: u64,
@@ -758,7 +798,7 @@ pub struct GeneralConfig {
pub me_route_hybrid_max_wait_ms: u64,
/// Maximum wait in milliseconds for blocking ME writer channel send fallback.
/// `0` keeps legacy unbounded wait behavior.
/// Must be within [1, 5000].
#[serde(default = "default_me_route_blocking_send_timeout_ms")]
pub me_route_blocking_send_timeout_ms: u64,
@@ -954,14 +994,19 @@ impl Default for GeneralConfig {
fn default() -> Self {
Self {
data_path: None,
quota_state_path: default_quota_state_path(),
config_strict: false,
modes: ProxyModes::default(),
prefer_ipv6: false,
fast_mode: default_true(),
use_middle_proxy: default_true(),
ad_tag: None,
proxy_secret_path: default_proxy_secret_path(),
proxy_secret_url: None,
proxy_config_v4_cache_path: default_proxy_config_v4_cache_path(),
proxy_config_v4_url: None,
proxy_config_v6_cache_path: default_proxy_config_v6_cache_path(),
proxy_config_v6_url: None,
middle_proxy_nat_ip: None,
middle_proxy_nat_probe: default_true(),
middle_proxy_nat_stun: default_middle_proxy_nat_stun(),
@@ -1044,6 +1089,8 @@ impl Default for GeneralConfig {
disable_colors: false,
telemetry: TelemetryConfig::default(),
me_socks_kdf_policy: MeSocksKdfPolicy::Strict,
me_route_backpressure_enabled: default_me_route_backpressure_enabled(),
me_route_fairshare_enabled: default_me_route_fairshare_enabled(),
me_route_backpressure_base_timeout_ms: default_me_route_backpressure_base_timeout_ms(),
me_route_backpressure_high_timeout_ms: default_me_route_backpressure_high_timeout_ms(),
me_route_backpressure_high_watermark_pct:
@@ -1329,6 +1376,10 @@ pub struct ConntrackControlConfig {
#[serde(default = "default_conntrack_control_enabled")]
pub inline_conntrack_control: bool,
/// Tracks whether inline_conntrack_control was explicitly set in config.
#[serde(skip)]
pub inline_conntrack_control_explicit: bool,
/// Conntrack mode for listener ingress traffic.
#[serde(default)]
pub mode: ConntrackMode,
@@ -1363,6 +1414,7 @@ impl Default for ConntrackControlConfig {
fn default() -> Self {
Self {
inline_conntrack_control: default_conntrack_control_enabled(),
inline_conntrack_control_explicit: false,
mode: ConntrackMode::default(),
backend: ConntrackBackend::default(),
profile: ConntrackPressureProfile::default(),
@@ -1551,6 +1603,13 @@ pub enum UnknownSniAction {
Drop,
Mask,
Accept,
/// Reject the TLS handshake by sending a fatal `unrecognized_name` alert
/// (RFC 6066, AlertDescription = 112) before closing the connection.
/// Mimics nginx `ssl_reject_handshake on;` behavior on the default vhost —
/// the wire response indistinguishable from a stock modern web server
/// that simply does not host the requested name.
#[serde(rename = "reject_handshake")]
RejectHandshake,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -1686,9 +1745,16 @@ pub struct AntiCensorshipConfig {
#[serde(default = "default_tls_new_session_tickets")]
pub tls_new_session_tickets: u8,
/// Enable compact ServerHello payload mode.
/// When false, FakeTLS always uses full ServerHello payload behavior.
/// When true, compact certificate payload mode can be used by TTL policy.
#[serde(default = "default_serverhello_compact")]
pub serverhello_compact: bool,
/// TTL in seconds for sending full certificate payload per client IP.
/// First client connection per (SNI domain, client IP) gets full cert payload.
/// Subsequent handshakes within TTL use compact cert metadata payload.
/// Applied only when `serverhello_compact` is enabled.
#[serde(default = "default_tls_full_cert_ttl_secs")]
pub tls_full_cert_ttl_secs: u64,
@@ -1731,6 +1797,7 @@ pub struct AntiCensorshipConfig {
pub mask_shape_above_cap_blur_max_bytes: usize,
/// Maximum bytes relayed per direction on unauthenticated masking fallback paths.
/// Set to 0 to disable byte cap (unlimited within relay/idle timeouts).
#[serde(default = "default_mask_relay_max_bytes")]
pub mask_relay_max_bytes: usize,
@@ -1782,6 +1849,7 @@ impl Default for AntiCensorshipConfig {
server_hello_delay_min_ms: default_server_hello_delay_min_ms(),
server_hello_delay_max_ms: default_server_hello_delay_max_ms(),
tls_new_session_tickets: default_tls_new_session_tickets(),
serverhello_compact: default_serverhello_compact(),
tls_full_cert_ttl_secs: default_tls_full_cert_ttl_secs(),
alpn_enforce: default_alpn_enforce(),
mask_proxy_protocol: 0,
@@ -1830,17 +1898,26 @@ pub struct AccessConfig {
///
/// Each entry supports independent upload (`up_bps`) and download
/// (`down_bps`) ceilings. A value of `0` in one direction means
/// "unlimited" for that direction.
/// "unlimited" for that direction. Limits are amortized: a relay quantum
/// may pass as a bounded burst, and the limiter applies the resulting wait
/// before later traffic in the same direction proceeds.
#[serde(default)]
pub user_rate_limits: HashMap<String, RateLimitBps>,
/// Per-CIDR aggregate transport rate limits in bits-per-second.
///
/// Matching uses longest-prefix-wins semantics. A value of `0` in one
/// direction means "unlimited" for that direction.
/// direction means "unlimited" for that direction. Limits are amortized
/// with the same bounded-burst contract as per-user rate limits.
#[serde(default)]
pub cidr_rate_limits: HashMap<IpNetwork, RateLimitBps>,
/// Per-username client source IP/CIDR deny list. Checked after successful
/// authentication; matching IPs get the same rejection path as invalid auth
/// (handshake fails closed for that connection).
#[serde(default)]
pub user_source_deny: HashMap<String, Vec<IpNetwork>>,
#[serde(default)]
pub user_max_unique_ips: HashMap<String, usize>,
@@ -1876,6 +1953,7 @@ impl Default for AccessConfig {
user_data_quota: HashMap::new(),
user_rate_limits: HashMap::new(),
cidr_rate_limits: HashMap::new(),
user_source_deny: HashMap::new(),
user_max_unique_ips: HashMap::new(),
user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(),
user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(),
@@ -1887,6 +1965,15 @@ impl Default for AccessConfig {
}
}
impl AccessConfig {
/// Returns true if `ip` is contained in any CIDR listed for `username` under `user_source_deny`.
pub fn is_user_source_ip_denied(&self, username: &str, ip: IpAddr) -> bool {
self.user_source_deny
.get(username)
.is_some_and(|nets| nets.iter().any(|n| n.contains(ip)))
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RateLimitBps {
#[serde(default)]

View File

@@ -24,6 +24,13 @@ enum NetfilterBackend {
Iptables,
}
#[derive(Clone, Copy)]
struct ConntrackRuntimeSupport {
netfilter_backend: Option<NetfilterBackend>,
has_cap_net_admin: bool,
has_conntrack_binary: bool,
}
#[derive(Clone, Copy)]
struct PressureSample {
conn_pct: Option<u8>,
@@ -56,11 +63,8 @@ pub(crate) fn spawn_conntrack_controller(
shared: Arc<ProxySharedState>,
) {
if !cfg!(target_os = "linux") {
let enabled = config_rx
.borrow()
.server
.conntrack_control
.inline_conntrack_control;
let cfg = config_rx.borrow();
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
stats.set_conntrack_control_enabled(enabled);
stats.set_conntrack_control_available(false);
stats.set_conntrack_pressure_active(false);
@@ -68,9 +72,14 @@ pub(crate) fn spawn_conntrack_controller(
stats.set_conntrack_rule_apply_ok(false);
shared.disable_conntrack_close_sender();
shared.set_conntrack_pressure_active(false);
if enabled {
if enabled
&& cfg
.server
.conntrack_control
.inline_conntrack_control_explicit
{
warn!(
"conntrack control is configured but unsupported on this OS; disabling runtime worker"
"conntrack control explicitly enabled but unsupported on this OS; disabling runtime worker"
);
}
return;
@@ -92,16 +101,17 @@ async fn run_conntrack_controller(
let mut cfg = config_rx.borrow().clone();
let mut pressure_state = PressureState::new(stats.as_ref());
let mut delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
let mut backend = pick_backend(cfg.server.conntrack_control.backend);
let mut runtime_support = probe_runtime_support(cfg.server.conntrack_control.backend);
let mut effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
apply_runtime_state(
stats.as_ref(),
shared.as_ref(),
&cfg,
backend.is_some(),
runtime_support,
false,
);
reconcile_rules(&cfg, backend, stats.as_ref()).await;
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
loop {
tokio::select! {
@@ -110,17 +120,18 @@ async fn run_conntrack_controller(
break;
}
cfg = config_rx.borrow_and_update().clone();
backend = pick_backend(cfg.server.conntrack_control.backend);
runtime_support = probe_runtime_support(cfg.server.conntrack_control.backend);
effective_enabled = effective_conntrack_enabled(&cfg, runtime_support);
delete_budget_tokens = cfg.server.conntrack_control.delete_budget_per_sec;
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, backend.is_some(), pressure_state.active);
reconcile_rules(&cfg, backend, stats.as_ref()).await;
apply_runtime_state(stats.as_ref(), shared.as_ref(), &cfg, runtime_support, pressure_state.active);
reconcile_rules(&cfg, runtime_support, stats.as_ref()).await;
}
event = close_rx.recv() => {
let Some(event) = event else {
break;
};
stats.set_conntrack_event_queue_depth(close_rx.len() as u64);
if !cfg.server.conntrack_control.inline_conntrack_control {
if !effective_enabled {
continue;
}
if !pressure_state.active {
@@ -156,6 +167,7 @@ async fn run_conntrack_controller(
stats.as_ref(),
shared.as_ref(),
&cfg,
effective_enabled,
&sample,
&mut pressure_state,
);
@@ -175,20 +187,30 @@ fn apply_runtime_state(
stats: &Stats,
shared: &ProxySharedState,
cfg: &ProxyConfig,
backend_available: bool,
runtime_support: ConntrackRuntimeSupport,
pressure_active: bool,
) {
let enabled = cfg.server.conntrack_control.inline_conntrack_control;
let available = enabled && backend_available && has_cap_net_admin();
if enabled && !available {
let available = effective_conntrack_enabled(cfg, runtime_support);
if enabled
&& !available
&& cfg
.server
.conntrack_control
.inline_conntrack_control_explicit
{
warn!(
"conntrack control enabled but unavailable (missing CAP_NET_ADMIN or backend binaries)"
has_cap_net_admin = runtime_support.has_cap_net_admin,
backend_available = runtime_support.netfilter_backend.is_some(),
conntrack_binary_available = runtime_support.has_conntrack_binary,
configured_backend = ?cfg.server.conntrack_control.backend,
"conntrack control explicitly enabled but unavailable; disabling runtime features"
);
}
stats.set_conntrack_control_enabled(enabled);
stats.set_conntrack_control_available(available);
shared.set_conntrack_pressure_active(enabled && pressure_active);
stats.set_conntrack_pressure_active(enabled && pressure_active);
shared.set_conntrack_pressure_active(available && pressure_active);
stats.set_conntrack_pressure_active(available && pressure_active);
}
fn collect_pressure_sample(
@@ -228,10 +250,11 @@ fn update_pressure_state(
stats: &Stats,
shared: &ProxySharedState,
cfg: &ProxyConfig,
effective_enabled: bool,
sample: &PressureSample,
state: &mut PressureState,
) {
if !cfg.server.conntrack_control.inline_conntrack_control {
if !effective_enabled {
if state.active {
state.active = false;
state.low_streak = 0;
@@ -285,22 +308,26 @@ fn update_pressure_state(
state.low_streak = 0;
}
async fn reconcile_rules(cfg: &ProxyConfig, backend: Option<NetfilterBackend>, stats: &Stats) {
async fn reconcile_rules(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
stats: &Stats,
) {
if !cfg.server.conntrack_control.inline_conntrack_control {
clear_notrack_rules_all_backends().await;
stats.set_conntrack_rule_apply_ok(true);
return;
}
if !has_cap_net_admin() {
if !effective_conntrack_enabled(cfg, runtime_support) {
clear_notrack_rules_all_backends().await;
stats.set_conntrack_rule_apply_ok(false);
return;
}
let Some(backend) = backend else {
stats.set_conntrack_rule_apply_ok(false);
return;
};
let backend = runtime_support
.netfilter_backend
.expect("netfilter backend must be available for effective conntrack control");
let apply_result = match backend {
NetfilterBackend::Nftables => apply_nft_rules(cfg).await,
@@ -315,6 +342,24 @@ async fn reconcile_rules(cfg: &ProxyConfig, backend: Option<NetfilterBackend>, s
}
}
fn probe_runtime_support(configured_backend: ConntrackBackend) -> ConntrackRuntimeSupport {
ConntrackRuntimeSupport {
netfilter_backend: pick_backend(configured_backend),
has_cap_net_admin: has_cap_net_admin(),
has_conntrack_binary: command_exists("conntrack"),
}
}
fn effective_conntrack_enabled(
cfg: &ProxyConfig,
runtime_support: ConntrackRuntimeSupport,
) -> bool {
cfg.server.conntrack_control.inline_conntrack_control
&& runtime_support.has_cap_net_admin
&& runtime_support.netfilter_backend.is_some()
&& runtime_support.has_conntrack_binary
}
fn pick_backend(configured: ConntrackBackend) -> Option<NetfilterBackend> {
match configured {
ConntrackBackend::Auto => {
@@ -710,7 +755,7 @@ mod tests {
me_queue_pressure_delta: 0,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &sample, &mut state);
assert!(state.active);
assert!(shared.conntrack_pressure_active());
@@ -731,7 +776,14 @@ mod tests {
accept_timeout_delta: 0,
me_queue_pressure_delta: 0,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &high_sample, &mut state);
update_pressure_state(
&stats,
shared.as_ref(),
&cfg,
true,
&high_sample,
&mut state,
);
assert!(state.active);
let low_sample = PressureSample {
@@ -740,11 +792,11 @@ mod tests {
accept_timeout_delta: 0,
me_queue_pressure_delta: 0,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
assert!(state.active);
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
assert!(state.active);
update_pressure_state(&stats, shared.as_ref(), &cfg, &low_sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, true, &low_sample, &mut state);
assert!(!state.active);
assert!(!shared.conntrack_pressure_active());
@@ -765,7 +817,7 @@ mod tests {
me_queue_pressure_delta: 10,
};
update_pressure_state(&stats, shared.as_ref(), &cfg, &sample, &mut state);
update_pressure_state(&stats, shared.as_ref(), &cfg, false, &sample, &mut state);
assert!(!state.active);
assert!(!shared.conntrack_pressure_active());

View File

@@ -222,6 +222,21 @@ pub enum ProxyError {
#[error("Proxy error: {0}")]
Proxy(String),
#[error("ME connection lost")]
MiddleConnectionLost,
#[error("Session terminated")]
RouteSwitched,
#[error("Traffic budget wait cancelled")]
TrafficBudgetWaitCancelled,
#[error("Traffic budget wait deadline exceeded")]
TrafficBudgetWaitDeadlineExceeded,
#[error("ME client writer cancelled")]
MiddleClientWriterCancelled,
// ============= Config Errors =============
#[error("Config error: {0}")]
Config(String),

211
src/healthcheck.rs Normal file
View File

@@ -0,0 +1,211 @@
use std::io::{Read, Write};
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream};
use std::time::Duration;
use serde_json::Value;
use crate::config::ProxyConfig;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HealthcheckMode {
Liveness,
Ready,
}
impl HealthcheckMode {
pub(crate) fn from_cli_arg(value: &str) -> Option<Self> {
match value {
"liveness" => Some(Self::Liveness),
"ready" => Some(Self::Ready),
_ => None,
}
}
fn request_path(self) -> &'static str {
match self {
Self::Liveness => "/v1/health",
Self::Ready => "/v1/health/ready",
}
}
}
pub(crate) fn run(config_path: &str, mode: HealthcheckMode) -> i32 {
match run_inner(config_path, mode) {
Ok(()) => 0,
Err(error) => {
eprintln!("[telemt] healthcheck failed: {error}");
1
}
}
}
fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
let config =
ProxyConfig::load(config_path).map_err(|error| format!("config load failed: {error}"))?;
let api_cfg = &config.server.api;
if !api_cfg.enabled {
return Ok(());
}
let listen: SocketAddr = api_cfg
.listen
.parse()
.map_err(|_| format!("invalid API listen address: {}", api_cfg.listen))?;
if listen.port() == 0 {
return Err("API listen port is 0".to_string());
}
let target = probe_target(listen);
let mut stream = TcpStream::connect_timeout(&target, Duration::from_secs(2))
.map_err(|error| format!("connect {target} failed: {error}"))?;
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.map_err(|error| format!("set read timeout failed: {error}"))?;
stream
.set_write_timeout(Some(Duration::from_secs(2)))
.map_err(|error| format!("set write timeout failed: {error}"))?;
let request = build_request(target, mode.request_path(), &api_cfg.auth_header);
stream
.write_all(request.as_bytes())
.map_err(|error| format!("request write failed: {error}"))?;
stream
.flush()
.map_err(|error| format!("request flush failed: {error}"))?;
let mut raw_response = Vec::new();
stream
.read_to_end(&mut raw_response)
.map_err(|error| format!("response read failed: {error}"))?;
let response =
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
let (status_code, body) = split_response(&response)?;
if status_code != 200 {
return Err(format!("HTTP status {status_code}"));
}
validate_payload(mode, body)?;
Ok(())
}
fn probe_target(listen: SocketAddr) -> SocketAddr {
match listen {
SocketAddr::V4(addr) => {
let ip = if addr.ip().is_unspecified() {
Ipv4Addr::LOCALHOST
} else {
*addr.ip()
};
SocketAddr::from((ip, addr.port()))
}
SocketAddr::V6(addr) => {
let ip = if addr.ip().is_unspecified() {
Ipv6Addr::LOCALHOST
} else {
*addr.ip()
};
SocketAddr::from((ip, addr.port()))
}
}
}
fn build_request(target: SocketAddr, path: &str, auth_header: &str) -> String {
let mut request = format!(
"GET {path} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\n",
target
);
if !auth_header.is_empty() {
request.push_str("Authorization: ");
request.push_str(auth_header);
request.push_str("\r\n");
}
request.push_str("\r\n");
request
}
fn split_response(response: &str) -> Result<(u16, &str), String> {
let header_end = response
.find("\r\n\r\n")
.ok_or_else(|| "invalid HTTP response headers".to_string())?;
let header = &response[..header_end];
let body = &response[header_end + 4..];
let status_line = header
.lines()
.next()
.ok_or_else(|| "missing HTTP status line".to_string())?;
let status_code = parse_status_code(status_line)?;
Ok((status_code, body))
}
fn parse_status_code(status_line: &str) -> Result<u16, String> {
let mut parts = status_line.split_whitespace();
let version = parts
.next()
.ok_or_else(|| "missing HTTP version".to_string())?;
if !version.starts_with("HTTP/") {
return Err(format!("invalid HTTP status line: {status_line}"));
}
let code = parts
.next()
.ok_or_else(|| "missing HTTP status code".to_string())?;
code.parse::<u16>()
.map_err(|_| format!("invalid HTTP status code: {code}"))
}
fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
let payload: Value =
serde_json::from_str(body).map_err(|_| "response body is not valid JSON".to_string())?;
if payload.get("ok").and_then(Value::as_bool) != Some(true) {
return Err("response JSON has ok=false".to_string());
}
let data = payload
.get("data")
.ok_or_else(|| "response JSON has no data field".to_string())?;
match mode {
HealthcheckMode::Liveness => {
if data.get("status").and_then(Value::as_str) != Some("ok") {
return Err("liveness status is not ok".to_string());
}
}
HealthcheckMode::Ready => {
if data.get("ready").and_then(Value::as_bool) != Some(true) {
return Err("readiness flag is false".to_string());
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{HealthcheckMode, parse_status_code, split_response, validate_payload};
#[test]
fn parse_status_code_reads_http_200() {
let status = parse_status_code("HTTP/1.1 200 OK").expect("must parse status");
assert_eq!(status, 200);
}
#[test]
fn split_response_extracts_status_and_body() {
let response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{\"ok\":true}";
let (status, body) = split_response(response).expect("must split response");
assert_eq!(status, 200);
assert_eq!(body, "{\"ok\":true}");
}
#[test]
fn validate_payload_accepts_liveness_contract() {
let body = "{\"ok\":true,\"data\":{\"status\":\"ok\"}}";
validate_payload(HealthcheckMode::Liveness, body).expect("liveness payload must pass");
}
#[test]
fn validate_payload_rejects_not_ready() {
let body = "{\"ok\":true,\"data\":{\"ready\":false}}";
let result = validate_payload(HealthcheckMode::Ready, body);
assert!(result.is_err());
}
}

View File

@@ -9,30 +9,53 @@ use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::{Mutex as AsyncMutex, RwLock};
use tokio::sync::{Mutex as AsyncMutex, RwLock, RwLockWriteGuard};
use crate::config::UserMaxUniqueIpsMode;
const CLEANUP_DRAIN_BATCH_LIMIT: usize = 1024;
const MAX_ACTIVE_IP_ENTRIES: u64 = 131_072;
const MAX_RECENT_IP_ENTRIES: u64 = 262_144;
/// Tracks active and recent client IPs for per-user admission control.
#[derive(Debug, Clone)]
pub struct UserIpTracker {
active_ips: Arc<RwLock<HashMap<String, HashMap<IpAddr, usize>>>>,
recent_ips: Arc<RwLock<HashMap<String, HashMap<IpAddr, Instant>>>>,
active_entry_count: Arc<AtomicU64>,
recent_entry_count: Arc<AtomicU64>,
active_cap_rejects: Arc<AtomicU64>,
recent_cap_rejects: Arc<AtomicU64>,
cleanup_deferred_releases: Arc<AtomicU64>,
max_ips: Arc<RwLock<HashMap<String, usize>>>,
default_max_ips: Arc<RwLock<usize>>,
limit_mode: Arc<RwLock<UserMaxUniqueIpsMode>>,
limit_window: Arc<RwLock<Duration>>,
last_compact_epoch_secs: Arc<AtomicU64>,
cleanup_queue: Arc<Mutex<Vec<(String, IpAddr)>>>,
cleanup_queue_len: Arc<AtomicU64>,
cleanup_queue: Arc<Mutex<HashMap<(String, IpAddr), usize>>>,
cleanup_drain_lock: Arc<AsyncMutex<()>>,
}
/// Point-in-time memory counters for user/IP limiter state.
#[derive(Debug, Clone, Copy)]
pub struct UserIpTrackerMemoryStats {
/// Number of users with active IP state.
pub active_users: usize,
/// Number of users with recent IP state.
pub recent_users: usize,
/// Number of active `(user, ip)` entries.
pub active_entries: usize,
/// Number of recent-window `(user, ip)` entries.
pub recent_entries: usize,
/// Number of deferred disconnect cleanups waiting to be drained.
pub cleanup_queue_len: usize,
/// Number of new connections rejected by the global active-entry cap.
pub active_cap_rejects: u64,
/// Number of new connections rejected by the global recent-entry cap.
pub recent_cap_rejects: u64,
/// Number of release cleanups deferred through the cleanup queue.
pub cleanup_deferred_releases: u64,
}
impl UserIpTracker {
@@ -40,22 +63,81 @@ impl UserIpTracker {
Self {
active_ips: Arc::new(RwLock::new(HashMap::new())),
recent_ips: Arc::new(RwLock::new(HashMap::new())),
active_entry_count: Arc::new(AtomicU64::new(0)),
recent_entry_count: Arc::new(AtomicU64::new(0)),
active_cap_rejects: Arc::new(AtomicU64::new(0)),
recent_cap_rejects: Arc::new(AtomicU64::new(0)),
cleanup_deferred_releases: Arc::new(AtomicU64::new(0)),
max_ips: Arc::new(RwLock::new(HashMap::new())),
default_max_ips: Arc::new(RwLock::new(0)),
limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)),
limit_window: Arc::new(RwLock::new(Duration::from_secs(30))),
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
cleanup_queue: Arc::new(Mutex::new(Vec::new())),
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
cleanup_queue: Arc::new(Mutex::new(HashMap::new())),
cleanup_drain_lock: Arc::new(AsyncMutex::new(())),
}
}
fn decrement_counter(counter: &AtomicU64, amount: usize) {
if amount == 0 {
return;
}
let amount = amount as u64;
let _ = counter.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| {
Some(current.saturating_sub(amount))
});
}
fn apply_active_cleanup(
active_ips: &mut HashMap<String, HashMap<IpAddr, usize>>,
user: &str,
ip: IpAddr,
pending_count: usize,
) -> usize {
if pending_count == 0 {
return 0;
}
let mut remove_user = false;
let mut removed_active_entries = 0usize;
if let Some(user_ips) = active_ips.get_mut(user) {
if let Some(count) = user_ips.get_mut(&ip) {
if *count > pending_count {
*count -= pending_count;
} else if user_ips.remove(&ip).is_some() {
removed_active_entries = 1;
}
}
remove_user = user_ips.is_empty();
}
if remove_user {
active_ips.remove(user);
}
removed_active_entries
}
/// Queues a deferred active IP cleanup for a later async drain.
pub fn enqueue_cleanup(&self, user: String, ip: IpAddr) {
match self.cleanup_queue.lock() {
Ok(mut queue) => queue.push((user, ip)),
Ok(mut queue) => {
let count = queue.entry((user, ip)).or_insert(0);
if *count == 0 {
self.cleanup_queue_len.fetch_add(1, Ordering::Relaxed);
}
*count = count.saturating_add(1);
self.cleanup_deferred_releases
.fetch_add(1, Ordering::Relaxed);
}
Err(poisoned) => {
let mut queue = poisoned.into_inner();
queue.push((user.clone(), ip));
let count = queue.entry((user.clone(), ip)).or_insert(0);
if *count == 0 {
self.cleanup_queue_len.fetch_add(1, Ordering::Relaxed);
}
*count = count.saturating_add(1);
self.cleanup_deferred_releases
.fetch_add(1, Ordering::Relaxed);
self.cleanup_queue.clear_poison();
tracing::warn!(
"UserIpTracker cleanup_queue lock poisoned; recovered and enqueued IP cleanup for {} ({})",
@@ -75,21 +157,38 @@ impl UserIpTracker {
}
#[cfg(test)]
pub(crate) fn cleanup_queue_mutex_for_tests(&self) -> Arc<Mutex<Vec<(String, IpAddr)>>> {
pub(crate) fn cleanup_queue_mutex_for_tests(
&self,
) -> Arc<Mutex<HashMap<(String, IpAddr), usize>>> {
Arc::clone(&self.cleanup_queue)
}
pub(crate) async fn drain_cleanup_queue(&self) {
// Serialize queue draining and active-IP mutation so check-and-add cannot
// observe stale active entries that are already queued for removal.
let _drain_guard = self.cleanup_drain_lock.lock().await;
if self.cleanup_queue_len.load(Ordering::Relaxed) == 0 {
return;
}
let Ok(_drain_guard) = self.cleanup_drain_lock.try_lock() else {
return;
};
let to_remove = {
match self.cleanup_queue.lock() {
Ok(mut queue) => {
if queue.is_empty() {
return;
}
std::mem::take(&mut *queue)
let mut drained =
HashMap::with_capacity(queue.len().min(CLEANUP_DRAIN_BATCH_LIMIT));
for _ in 0..CLEANUP_DRAIN_BATCH_LIMIT {
let Some(key) = queue.keys().next().cloned() else {
break;
};
if let Some(count) = queue.remove(&key) {
self.cleanup_queue_len.fetch_sub(1, Ordering::Relaxed);
drained.insert(key, count);
}
}
drained
}
Err(poisoned) => {
let mut queue = poisoned.into_inner();
@@ -97,28 +196,34 @@ impl UserIpTracker {
self.cleanup_queue.clear_poison();
return;
}
let drained = std::mem::take(&mut *queue);
let mut drained =
HashMap::with_capacity(queue.len().min(CLEANUP_DRAIN_BATCH_LIMIT));
for _ in 0..CLEANUP_DRAIN_BATCH_LIMIT {
let Some(key) = queue.keys().next().cloned() else {
break;
};
if let Some(count) = queue.remove(&key) {
self.cleanup_queue_len.fetch_sub(1, Ordering::Relaxed);
drained.insert(key, count);
}
}
self.cleanup_queue.clear_poison();
drained
}
}
};
if to_remove.is_empty() {
return;
}
let mut active_ips = self.active_ips.write().await;
for (user, ip) in to_remove {
if let Some(user_ips) = active_ips.get_mut(&user) {
if let Some(count) = user_ips.get_mut(&ip) {
if *count > 1 {
*count -= 1;
} else {
user_ips.remove(&ip);
}
}
if user_ips.is_empty() {
active_ips.remove(&user);
}
}
let mut removed_active_entries = 0usize;
for ((user, ip), pending_count) in to_remove {
removed_active_entries = removed_active_entries.saturating_add(
Self::apply_active_cleanup(&mut active_ips, &user, ip, pending_count),
);
}
Self::decrement_counter(&self.active_entry_count, removed_active_entries);
}
fn now_epoch_secs() -> u64 {
@@ -128,6 +233,24 @@ impl UserIpTracker {
.as_secs()
}
async fn active_and_recent_write(
&self,
) -> (
RwLockWriteGuard<'_, HashMap<String, HashMap<IpAddr, usize>>>,
RwLockWriteGuard<'_, HashMap<String, HashMap<IpAddr, Instant>>>,
) {
loop {
let active_ips = self.active_ips.write().await;
match self.recent_ips.try_write() {
Ok(recent_ips) => return (active_ips, recent_ips),
Err(_) => {
drop(active_ips);
tokio::task::yield_now().await;
}
}
}
}
async fn maybe_compact_empty_users(&self) {
const COMPACT_INTERVAL_SECS: u64 = 60;
let now_epoch_secs = Self::now_epoch_secs();
@@ -148,14 +271,16 @@ impl UserIpTracker {
return;
}
let mut active_ips = self.active_ips.write().await;
let mut recent_ips = self.recent_ips.write().await;
let window = *self.limit_window.read().await;
let now = Instant::now();
let (mut active_ips, mut recent_ips) = self.active_and_recent_write().await;
let mut pruned_recent_entries = 0usize;
for user_recent in recent_ips.values_mut() {
Self::prune_recent(user_recent, now, window);
pruned_recent_entries =
pruned_recent_entries.saturating_add(Self::prune_recent(user_recent, now, window));
}
Self::decrement_counter(&self.recent_entry_count, pruned_recent_entries);
let mut users =
Vec::<String>::with_capacity(active_ips.len().saturating_add(recent_ips.len()));
@@ -182,12 +307,17 @@ impl UserIpTracker {
}
}
pub async fn run_periodic_maintenance(self: Arc<Self>) {
let mut interval = tokio::time::interval(Duration::from_secs(1));
loop {
interval.tick().await;
self.drain_cleanup_queue().await;
self.maybe_compact_empty_users().await;
}
}
pub async fn memory_stats(&self) -> UserIpTrackerMemoryStats {
let cleanup_queue_len = self
.cleanup_queue
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.len();
let cleanup_queue_len = self.cleanup_queue_len.load(Ordering::Relaxed) as usize;
let active_ips = self.active_ips.read().await;
let recent_ips = self.recent_ips.read().await;
let active_entries = active_ips.values().map(HashMap::len).sum();
@@ -199,6 +329,9 @@ impl UserIpTracker {
active_entries,
recent_entries,
cleanup_queue_len,
active_cap_rejects: self.active_cap_rejects.load(Ordering::Relaxed),
recent_cap_rejects: self.recent_cap_rejects.load(Ordering::Relaxed),
cleanup_deferred_releases: self.cleanup_deferred_releases.load(Ordering::Relaxed),
}
}
@@ -229,11 +362,17 @@ impl UserIpTracker {
max_ips.clone_from(limits);
}
fn prune_recent(user_recent: &mut HashMap<IpAddr, Instant>, now: Instant, window: Duration) {
fn prune_recent(
user_recent: &mut HashMap<IpAddr, Instant>,
now: Instant,
window: Duration,
) -> usize {
if user_recent.is_empty() {
return;
return 0;
}
let before = user_recent.len();
user_recent.retain(|_, seen_at| now.duration_since(*seen_at) <= window);
before.saturating_sub(user_recent.len())
}
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
@@ -252,26 +391,40 @@ impl UserIpTracker {
let window = *self.limit_window.read().await;
let now = Instant::now();
let mut active_ips = self.active_ips.write().await;
let (mut active_ips, mut recent_ips) = self.active_and_recent_write().await;
let user_active = active_ips
.entry(username.to_string())
.or_insert_with(HashMap::new);
let mut recent_ips = self.recent_ips.write().await;
let user_recent = recent_ips
.entry(username.to_string())
.or_insert_with(HashMap::new);
Self::prune_recent(user_recent, now, window);
let pruned_recent_entries = Self::prune_recent(user_recent, now, window);
Self::decrement_counter(&self.recent_entry_count, pruned_recent_entries);
let recent_contains_ip = user_recent.contains_key(&ip);
if let Some(count) = user_active.get_mut(&ip) {
if !recent_contains_ip
&& self.recent_entry_count.load(Ordering::Relaxed) >= MAX_RECENT_IP_ENTRIES
{
self.recent_cap_rejects.fetch_add(1, Ordering::Relaxed);
return Err(format!(
"IP tracker recent entry cap reached: entries={}/{}",
self.recent_entry_count.load(Ordering::Relaxed),
MAX_RECENT_IP_ENTRIES
));
}
*count = count.saturating_add(1);
user_recent.insert(ip, now);
if user_recent.insert(ip, now).is_none() {
self.recent_entry_count.fetch_add(1, Ordering::Relaxed);
}
return Ok(());
}
let is_new_ip = !recent_contains_ip;
if let Some(limit) = limit {
let active_limit_reached = user_active.len() >= limit;
let recent_limit_reached = user_recent.len() >= limit;
let recent_limit_reached = user_recent.len() >= limit && is_new_ip;
let deny = match mode {
UserMaxUniqueIpsMode::ActiveWindow => active_limit_reached,
UserMaxUniqueIpsMode::TimeWindow => recent_limit_reached,
@@ -291,30 +444,62 @@ impl UserIpTracker {
}
}
user_active.insert(ip, 1);
user_recent.insert(ip, now);
if self.active_entry_count.load(Ordering::Relaxed) >= MAX_ACTIVE_IP_ENTRIES {
self.active_cap_rejects.fetch_add(1, Ordering::Relaxed);
return Err(format!(
"IP tracker active entry cap reached: entries={}/{}",
self.active_entry_count.load(Ordering::Relaxed),
MAX_ACTIVE_IP_ENTRIES
));
}
if is_new_ip && self.recent_entry_count.load(Ordering::Relaxed) >= MAX_RECENT_IP_ENTRIES {
self.recent_cap_rejects.fetch_add(1, Ordering::Relaxed);
return Err(format!(
"IP tracker recent entry cap reached: entries={}/{}",
self.recent_entry_count.load(Ordering::Relaxed),
MAX_RECENT_IP_ENTRIES
));
}
if user_active.insert(ip, 1).is_none() {
self.active_entry_count.fetch_add(1, Ordering::Relaxed);
}
if user_recent.insert(ip, now).is_none() {
self.recent_entry_count.fetch_add(1, Ordering::Relaxed);
}
Ok(())
}
pub async fn remove_ip(&self, username: &str, ip: IpAddr) {
self.maybe_compact_empty_users().await;
let mut active_ips = self.active_ips.write().await;
let mut removed_active_entries = 0usize;
if let Some(user_ips) = active_ips.get_mut(username) {
if let Some(count) = user_ips.get_mut(&ip) {
if *count > 1 {
*count -= 1;
} else {
user_ips.remove(&ip);
if user_ips.remove(&ip).is_some() {
removed_active_entries = 1;
}
}
}
if user_ips.is_empty() {
active_ips.remove(username);
}
}
Self::decrement_counter(&self.active_entry_count, removed_active_entries);
}
pub async fn get_recent_counts_for_users(&self, users: &[String]) -> HashMap<String, usize> {
self.drain_cleanup_queue().await;
self.get_recent_counts_for_users_snapshot(users).await
}
pub(crate) async fn get_recent_counts_for_users_snapshot(
&self,
users: &[String],
) -> HashMap<String, usize> {
let window = *self.limit_window.read().await;
let now = Instant::now();
let recent_ips = self.recent_ips.read().await;
@@ -389,19 +574,29 @@ impl UserIpTracker {
pub async fn get_stats(&self) -> Vec<(String, usize, usize)> {
self.drain_cleanup_queue().await;
self.get_stats_snapshot().await
}
pub(crate) async fn get_stats_snapshot(&self) -> Vec<(String, usize, usize)> {
let active_ips = self.active_ips.read().await;
let active_counts = active_ips
.iter()
.map(|(username, user_ips)| (username.clone(), user_ips.len()))
.collect::<Vec<_>>();
drop(active_ips);
let max_ips = self.max_ips.read().await;
let default_max_ips = *self.default_max_ips.read().await;
let mut stats = Vec::new();
for (username, user_ips) in active_ips.iter() {
let mut stats = Vec::with_capacity(active_counts.len());
for (username, active_count) in active_counts {
let limit = max_ips
.get(username)
.get(&username)
.copied()
.filter(|limit| *limit > 0)
.or((default_max_ips > 0).then_some(default_max_ips))
.unwrap_or(0);
stats.push((username.clone(), user_ips.len(), limit));
stats.push((username, active_count, limit));
}
stats.sort_by(|a, b| a.0.cmp(&b.0));
@@ -410,20 +605,30 @@ impl UserIpTracker {
pub async fn clear_user_ips(&self, username: &str) {
let mut active_ips = self.active_ips.write().await;
active_ips.remove(username);
let removed_active_entries = active_ips
.remove(username)
.map(|ips| ips.len())
.unwrap_or(0);
drop(active_ips);
Self::decrement_counter(&self.active_entry_count, removed_active_entries);
let mut recent_ips = self.recent_ips.write().await;
recent_ips.remove(username);
let removed_recent_entries = recent_ips
.remove(username)
.map(|ips| ips.len())
.unwrap_or(0);
Self::decrement_counter(&self.recent_entry_count, removed_recent_entries);
}
pub async fn clear_all(&self) {
let mut active_ips = self.active_ips.write().await;
active_ips.clear();
drop(active_ips);
self.active_entry_count.store(0, Ordering::Relaxed);
let mut recent_ips = self.recent_ips.write().await;
recent_ips.clear();
self.recent_entry_count.store(0, Ordering::Relaxed);
}
pub async fn is_ip_active(&self, username: &str, ip: IpAddr) -> bool {
@@ -851,4 +1056,19 @@ mod tests {
.unwrap_or(false);
assert!(!stale_exists);
}
#[tokio::test]
async fn test_time_window_allows_same_ip_reconnect() {
let tracker = UserIpTracker::new();
tracker.set_user_limit("test_user", 1).await;
tracker
.set_limit_policy(UserMaxUniqueIpsMode::TimeWindow, 1)
.await;
let ip1 = test_ipv4(10, 4, 0, 1);
assert!(tracker.check_and_add("test_user", ip1).await.is_ok());
tracker.remove_ip("test_user", ip1).await;
assert!(tracker.check_and_add("test_user", ip1).await.is_ok());
}
}

View File

@@ -17,6 +17,7 @@ pub(crate) async fn configure_admission_gate(
route_runtime: Arc<RouteRuntimeController>,
admission_tx: &watch::Sender<bool>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
me_ready_rx: watch::Receiver<u64>,
) {
if config.general.use_middle_proxy {
if let Some(pool) = me_pool.as_ref() {
@@ -52,6 +53,7 @@ pub(crate) async fn configure_admission_gate(
let admission_tx_gate = admission_tx.clone();
let route_runtime_gate = route_runtime.clone();
let mut config_rx_gate = config_rx.clone();
let mut me_ready_rx_gate = me_ready_rx;
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
tokio::spawn(async move {
let mut gate_open = initial_gate_open;
@@ -74,6 +76,11 @@ pub(crate) async fn configure_admission_gate(
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
continue;
}
changed = me_ready_rx_gate.changed() => {
if changed.is_err() {
break;
}
}
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
}
let ready = pool_for_gate.admission_ready_conditional_cast().await;

View File

@@ -1,6 +1,6 @@
#![allow(clippy::items_after_test_module)]
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::sync::watch;
@@ -17,7 +17,7 @@ use crate::transport::middle_proxy::{
pub(crate) fn resolve_runtime_config_path(
config_path_cli: &str,
startup_cwd: &std::path::Path,
startup_cwd: &Path,
config_path_explicit: bool,
) -> PathBuf {
if config_path_explicit {
@@ -46,6 +46,39 @@ pub(crate) fn resolve_runtime_config_path(
startup_cwd.join("config.toml")
}
pub(crate) fn resolve_runtime_base_dir(
config_path: &Path,
startup_cwd: &Path,
config_path_explicit: bool,
data_path: Option<&Path>,
) -> PathBuf {
if let Some(path) = data_path {
return normalize_runtime_dir(path, startup_cwd);
}
if startup_cwd != Path::new("/") {
return normalize_runtime_dir(startup_cwd, startup_cwd);
}
if config_path_explicit
&& let Some(parent) = config_path.parent()
&& !parent.as_os_str().is_empty()
{
return normalize_runtime_dir(parent, startup_cwd);
}
PathBuf::from("/etc/telemt")
}
fn normalize_runtime_dir(path: &Path, startup_cwd: &Path) -> PathBuf {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
startup_cwd.join(path)
};
absolute.canonicalize().unwrap_or(absolute)
}
/// Parsed CLI arguments.
pub(crate) struct CliArgs {
pub config_path: String,
@@ -231,7 +264,13 @@ fn print_help() {
#[cfg(test)]
mod tests {
use super::resolve_runtime_config_path;
use std::path::{Path, PathBuf};
use super::{
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
resolve_runtime_base_dir, resolve_runtime_config_path,
};
use crate::error::{ProxyError, StreamError};
#[test]
fn resolve_runtime_config_path_anchors_relative_to_startup_cwd() {
@@ -299,6 +338,166 @@ mod tests {
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_prefers_cli_data_path() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let startup_cwd = std::env::temp_dir().join(format!("telemt_runtime_base_cwd_{nonce}"));
let data_path = std::env::temp_dir().join(format!("telemt_runtime_base_data_{nonce}"));
std::fs::create_dir_all(&startup_cwd).unwrap();
std::fs::create_dir_all(&data_path).unwrap();
let resolved = resolve_runtime_base_dir(
&startup_cwd.join("config.toml"),
&startup_cwd,
true,
Some(&data_path),
);
assert_eq!(resolved, data_path.canonicalize().unwrap());
let _ = std::fs::remove_dir(&data_path);
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_uses_working_directory_before_explicit_config_parent() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let startup_cwd = std::env::temp_dir().join(format!("telemt_runtime_base_start_{nonce}"));
let config_dir = std::env::temp_dir().join(format!("telemt_runtime_base_cfg_{nonce}"));
std::fs::create_dir_all(&startup_cwd).unwrap();
std::fs::create_dir_all(&config_dir).unwrap();
let resolved =
resolve_runtime_base_dir(&config_dir.join("telemt.toml"), &startup_cwd, true, None);
assert_eq!(resolved, startup_cwd.canonicalize().unwrap());
let _ = std::fs::remove_dir(&config_dir);
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_uses_explicit_config_parent_from_root() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let config_dir = std::env::temp_dir().join(format!("telemt_runtime_base_root_cfg_{nonce}"));
std::fs::create_dir_all(&config_dir).unwrap();
let resolved =
resolve_runtime_base_dir(&config_dir.join("telemt.toml"), Path::new("/"), true, None);
assert_eq!(resolved, config_dir.canonicalize().unwrap());
let _ = std::fs::remove_dir(&config_dir);
}
#[test]
fn resolve_runtime_base_dir_uses_systemd_working_directory_before_etc() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let startup_cwd = std::env::temp_dir().join(format!("telemt_runtime_base_systemd_{nonce}"));
std::fs::create_dir_all(&startup_cwd).unwrap();
let resolved =
resolve_runtime_base_dir(&startup_cwd.join("config.toml"), &startup_cwd, false, None);
assert_eq!(resolved, startup_cwd.canonicalize().unwrap());
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_falls_back_to_etc_from_root() {
let resolved = resolve_runtime_base_dir(
Path::new("/etc/telemt/config.toml"),
Path::new("/"),
false,
None,
);
assert_eq!(resolved, PathBuf::from("/etc/telemt"));
}
#[test]
fn expected_handshake_eof_matches_connection_reset() {
let err = ProxyError::Io(std::io::Error::from(std::io::ErrorKind::ConnectionReset));
assert!(is_expected_handshake_eof(&err));
}
#[test]
fn expected_handshake_eof_matches_stream_io_unexpected_eof() {
let err = ProxyError::Stream(StreamError::Io(std::io::Error::from(
std::io::ErrorKind::UnexpectedEof,
)));
assert!(is_expected_handshake_eof(&err));
}
#[test]
fn peer_close_description_is_human_readable_for_all_peer_close_kinds() {
let cases = [
(
std::io::ErrorKind::ConnectionReset,
"Peer reset TCP connection (RST)",
),
(
std::io::ErrorKind::ConnectionAborted,
"Peer aborted TCP connection during transport",
),
(
std::io::ErrorKind::BrokenPipe,
"Peer closed write side (broken pipe)",
),
(
std::io::ErrorKind::NotConnected,
"Socket was already closed by peer",
),
];
for (kind, expected) in cases {
let err = ProxyError::Io(std::io::Error::from(kind));
assert_eq!(peer_close_description(&err), Some(expected));
}
}
#[test]
fn handshake_close_description_is_human_readable_for_all_expected_kinds() {
let cases = [
(
ProxyError::Io(std::io::Error::from(std::io::ErrorKind::UnexpectedEof)),
"Peer closed before sending full 64-byte MTProto handshake",
),
(
ProxyError::Io(std::io::Error::from(std::io::ErrorKind::ConnectionReset)),
"Peer reset TCP connection during initial MTProto handshake",
),
(
ProxyError::Io(std::io::Error::from(std::io::ErrorKind::ConnectionAborted)),
"Peer aborted TCP connection during initial MTProto handshake",
),
(
ProxyError::Io(std::io::Error::from(std::io::ErrorKind::BrokenPipe)),
"Peer closed write side before MTProto handshake completed",
),
(
ProxyError::Io(std::io::Error::from(std::io::ErrorKind::NotConnected)),
"Handshake socket was already closed by peer",
),
(
ProxyError::Stream(StreamError::UnexpectedEof),
"Peer closed before sending full 64-byte MTProto handshake",
),
];
for (err, expected) in cases {
assert_eq!(expected_handshake_close_description(&err), Some(expected));
}
}
}
pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) {
@@ -428,7 +627,63 @@ pub(crate) async fn wait_until_admission_open(admission_rx: &mut watch::Receiver
}
pub(crate) fn is_expected_handshake_eof(err: &crate::error::ProxyError) -> bool {
err.to_string().contains("expected 64 bytes, got 0")
expected_handshake_close_description(err).is_some()
}
pub(crate) fn peer_close_description(err: &crate::error::ProxyError) -> Option<&'static str> {
fn from_kind(kind: std::io::ErrorKind) -> Option<&'static str> {
match kind {
std::io::ErrorKind::ConnectionReset => Some("Peer reset TCP connection (RST)"),
std::io::ErrorKind::ConnectionAborted => {
Some("Peer aborted TCP connection during transport")
}
std::io::ErrorKind::BrokenPipe => Some("Peer closed write side (broken pipe)"),
std::io::ErrorKind::NotConnected => Some("Socket was already closed by peer"),
_ => None,
}
}
match err {
crate::error::ProxyError::Io(ioe) => from_kind(ioe.kind()),
crate::error::ProxyError::Stream(crate::error::StreamError::Io(ioe)) => {
from_kind(ioe.kind())
}
_ => None,
}
}
pub(crate) fn expected_handshake_close_description(
err: &crate::error::ProxyError,
) -> Option<&'static str> {
fn from_kind(kind: std::io::ErrorKind) -> Option<&'static str> {
match kind {
std::io::ErrorKind::UnexpectedEof => {
Some("Peer closed before sending full 64-byte MTProto handshake")
}
std::io::ErrorKind::ConnectionReset => {
Some("Peer reset TCP connection during initial MTProto handshake")
}
std::io::ErrorKind::ConnectionAborted => {
Some("Peer aborted TCP connection during initial MTProto handshake")
}
std::io::ErrorKind::BrokenPipe => {
Some("Peer closed write side before MTProto handshake completed")
}
std::io::ErrorKind::NotConnected => Some("Handshake socket was already closed by peer"),
_ => None,
}
}
match err {
crate::error::ProxyError::Io(ioe) => from_kind(ioe.kind()),
crate::error::ProxyError::Stream(crate::error::StreamError::UnexpectedEof) => {
Some("Peer closed before sending full 64-byte MTProto handshake")
}
crate::error::ProxyError::Stream(crate::error::StreamError::Io(ioe)) => {
from_kind(ioe.kind())
}
_ => None,
}
}
pub(crate) async fn load_startup_proxy_config_snapshot(

View File

@@ -13,7 +13,7 @@ use crate::config::{ProxyConfig, RstOnCloseMode};
use crate::crypto::SecureRandom;
use crate::ip_tracker::UserIpTracker;
use crate::proxy::ClientHandler;
use crate::proxy::route_mode::{ROUTE_SWITCH_ERROR_MSG, RouteRuntimeController};
use crate::proxy::route_mode::RouteRuntimeController;
use crate::proxy::shared_state::ProxySharedState;
use crate::startup::{COMPONENT_LISTENERS_BIND, StartupTracker};
use crate::stats::beobachten::BeobachtenStore;
@@ -24,7 +24,10 @@ use crate::transport::middle_proxy::MePool;
use crate::transport::socket::set_linger_zero;
use crate::transport::{ListenOptions, UpstreamManager, create_listener, find_listener_processes};
use super::helpers::{is_expected_handshake_eof, print_proxy_links};
use super::helpers::{
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
print_proxy_links,
};
pub(crate) struct BoundListeners {
pub(crate) listeners: Vec<(TcpListener, bool)>,
@@ -485,45 +488,32 @@ pub(crate) fn spawn_tcp_accept_loops(
Ok(guard) => *guard,
Err(_) => None,
};
let peer_closed = matches!(
&e,
crate::error::ProxyError::Io(ioe)
if matches!(
ioe.kind(),
std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::NotConnected
)
) || matches!(
&e,
crate::error::ProxyError::Stream(
crate::error::StreamError::Io(ioe)
)
if matches!(
ioe.kind(),
std::io::ErrorKind::ConnectionReset
| std::io::ErrorKind::ConnectionAborted
| std::io::ErrorKind::BrokenPipe
| std::io::ErrorKind::NotConnected
)
);
let peer_close_reason = peer_close_description(&e);
let handshake_close_reason =
expected_handshake_close_description(&e);
let me_closed = matches!(
&e,
crate::error::ProxyError::Proxy(msg) if msg == "ME connection lost"
);
let route_switched = matches!(
&e,
crate::error::ProxyError::Proxy(msg) if msg == ROUTE_SWITCH_ERROR_MSG
);
let me_closed =
matches!(&e, crate::error::ProxyError::MiddleConnectionLost);
let route_switched =
matches!(&e, crate::error::ProxyError::RouteSwitched);
match (peer_closed, me_closed) {
(true, _) => {
match (peer_close_reason, me_closed) {
(Some(reason), _) => {
if let Some(real_peer) = real_peer {
debug!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed by client");
debug!(
peer = %peer_addr,
real_peer = %real_peer,
error = %e,
close_reason = reason,
"Connection closed by peer"
);
} else {
debug!(peer = %peer_addr, error = %e, "Connection closed by client");
debug!(
peer = %peer_addr,
error = %e,
close_reason = reason,
"Connection closed by peer"
);
}
}
(_, true) => {
@@ -541,10 +531,23 @@ pub(crate) fn spawn_tcp_accept_loops(
}
}
_ if is_expected_handshake_eof(&e) => {
let reason = handshake_close_reason
.unwrap_or("Peer closed during initial handshake");
if let Some(real_peer) = real_peer {
info!(peer = %peer_addr, real_peer = %real_peer, error = %e, "Connection closed during initial handshake");
info!(
peer = %peer_addr,
real_peer = %real_peer,
error = %e,
close_reason = reason,
"Connection closed during initial handshake"
);
} else {
info!(peer = %peer_addr, error = %e, "Connection closed during initial handshake");
info!(
peer = %peer_addr,
error = %e,
close_reason = reason,
"Connection closed during initial handshake"
);
}
}
_ => {

View File

@@ -3,7 +3,7 @@
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tokio::sync::{RwLock, watch};
use tracing::{error, info, warn};
use crate::config::ProxyConfig;
@@ -29,6 +29,7 @@ pub(crate) async fn initialize_me_pool(
rng: Arc<SecureRandom>,
stats: Arc<Stats>,
api_me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
me_ready_tx: watch::Sender<u64>,
) -> Option<Arc<MePool>> {
if !use_middle_proxy {
return None;
@@ -66,6 +67,7 @@ pub(crate) async fn initialize_me_pool(
match crate::transport::middle_proxy::fetch_proxy_secret_with_upstream(
proxy_secret_path,
config.general.proxy_secret_len_max,
config.general.proxy_secret_url.as_deref(),
Some(upstream_manager.clone()),
)
.await
@@ -126,7 +128,11 @@ pub(crate) async fn initialize_me_pool(
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V4)
.await;
let cfg_v4 = load_startup_proxy_config_snapshot(
"https://core.telegram.org/getProxyConfig",
config
.general
.proxy_config_v4_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfig"),
config.general.proxy_config_v4_cache_path.as_deref(),
me2dc_fallback,
"getProxyConfig",
@@ -158,7 +164,11 @@ pub(crate) async fn initialize_me_pool(
.set_me_status(StartupMeStatus::Initializing, COMPONENT_ME_PROXY_CONFIG_V6)
.await;
let cfg_v6 = load_startup_proxy_config_snapshot(
"https://core.telegram.org/getProxyConfigV6",
config
.general
.proxy_config_v6_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfigV6"),
config.general.proxy_config_v6_cache_path.as_deref(),
me2dc_fallback,
"getProxyConfigV6",
@@ -268,6 +278,8 @@ pub(crate) async fn initialize_me_pool(
config.general.me_socks_kdf_policy,
config.general.me_writer_cmd_channel_capacity,
config.general.me_route_channel_capacity,
config.general.me_route_backpressure_enabled,
config.general.me_route_fairshare_enabled,
config.general.me_route_backpressure_base_timeout_ms,
config.general.me_route_backpressure_high_timeout_ms,
config.general.me_route_backpressure_high_watermark_pct,
@@ -303,6 +315,7 @@ pub(crate) async fn initialize_me_pool(
let pool_bg = pool.clone();
let rng_bg = rng.clone();
let startup_tracker_bg = startup_tracker.clone();
let me_ready_tx_bg = me_ready_tx.clone();
let retry_limit = if me_init_retry_attempts == 0 {
String::from("unlimited")
} else {
@@ -336,6 +349,9 @@ pub(crate) async fn initialize_me_pool(
startup_tracker_bg
.set_me_status(StartupMeStatus::Ready, "ready")
.await;
me_ready_tx_bg.send_modify(|version| {
*version = version.saturating_add(1);
});
info!(
attempt = init_attempt,
"Middle-End pool initialized successfully"
@@ -463,6 +479,9 @@ pub(crate) async fn initialize_me_pool(
startup_tracker
.set_me_status(StartupMeStatus::Ready, "ready")
.await;
me_ready_tx.send_modify(|version| {
*version = version.saturating_add(1);
});
info!(
attempt = init_attempt,
"Middle-End pool initialized successfully"

View File

@@ -47,7 +47,7 @@ use crate::stats::{ReplayChecker, Stats};
use crate::stream::BufferPool;
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::MePool;
use helpers::{parse_cli, resolve_runtime_config_path};
use helpers::{parse_cli, resolve_runtime_base_dir, resolve_runtime_config_path};
#[cfg(unix)]
use crate::daemon::{DaemonOptions, PidFile, drop_privileges};
@@ -112,8 +112,51 @@ async fn run_telemt_core(
std::process::exit(1);
}
};
if let Some(ref data_path) = data_path
&& !data_path.is_absolute()
{
eprintln!(
"[telemt] data_path must be absolute: {}",
data_path.display()
);
std::process::exit(1);
}
let mut config_path =
resolve_runtime_config_path(&config_path_cli, &startup_cwd, config_path_explicit);
let runtime_base_dir = resolve_runtime_base_dir(
&config_path,
&startup_cwd,
config_path_explicit,
data_path.as_deref(),
);
if !runtime_base_dir.exists()
&& let Err(e) = std::fs::create_dir_all(&runtime_base_dir)
{
eprintln!(
"[telemt] Can't create runtime directory {}: {}",
runtime_base_dir.display(),
e
);
std::process::exit(1);
}
if !runtime_base_dir.is_dir() {
eprintln!(
"[telemt] Runtime path exists but is not a directory: {}",
runtime_base_dir.display()
);
std::process::exit(1);
}
if let Err(e) = std::env::set_current_dir(&runtime_base_dir) {
eprintln!(
"[telemt] Can't use runtime directory {}: {}",
runtime_base_dir.display(),
e
);
std::process::exit(1);
}
let mut config = match ProxyConfig::load(&config_path) {
Ok(c) => c,
@@ -156,16 +199,15 @@ async fn run_telemt_core(
);
}
} else {
let system_dir = std::path::Path::new("/etc/telemt");
let system_config_path = system_dir.join("telemt.toml");
let startup_config_path = startup_cwd.join("config.toml");
let runtime_config_path = runtime_base_dir.join("telemt.toml");
let fallback_config_path = runtime_base_dir.join("config.toml");
let mut persisted = false;
if let Some(serialized) = serialized.as_ref() {
match std::fs::create_dir_all(system_dir) {
Ok(()) => match std::fs::write(&system_config_path, serialized) {
match std::fs::create_dir_all(&runtime_base_dir) {
Ok(()) => match std::fs::write(&runtime_config_path, serialized) {
Ok(()) => {
config_path = system_config_path;
config_path = runtime_config_path;
eprintln!(
"[telemt] Created default config at {}",
config_path.display()
@@ -175,7 +217,7 @@ async fn run_telemt_core(
Err(write_error) => {
eprintln!(
"[telemt] Warning: failed to write default config at {}: {}",
system_config_path.display(),
runtime_config_path.display(),
write_error
);
}
@@ -183,16 +225,16 @@ async fn run_telemt_core(
Err(create_error) => {
eprintln!(
"[telemt] Warning: failed to create {}: {}",
system_dir.display(),
runtime_base_dir.display(),
create_error
);
}
}
if !persisted {
match std::fs::write(&startup_config_path, serialized) {
match std::fs::write(&fallback_config_path, serialized) {
Ok(()) => {
config_path = startup_config_path;
config_path = fallback_config_path;
eprintln!(
"[telemt] Created default config at {}",
config_path.display()
@@ -202,7 +244,7 @@ async fn run_telemt_core(
Err(write_error) => {
eprintln!(
"[telemt] Warning: failed to write default config at {}: {}",
startup_config_path.display(),
fallback_config_path.display(),
write_error
);
}
@@ -375,6 +417,8 @@ async fn run_telemt_core(
let stats = Arc::new(Stats::new());
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
let quota_state_path = config.general.quota_state_path.clone();
crate::quota_state::load_quota_state(&quota_state_path, stats.as_ref()).await;
let upstream_manager = Arc::new(UpstreamManager::new(
config.upstreams.clone(),
@@ -454,6 +498,7 @@ async fn run_telemt_core(
let config_rx_api = api_config_rx.clone();
let admission_rx_api = admission_rx.clone();
let config_path_api = config_path.clone();
let quota_state_path_api = quota_state_path.clone();
let startup_tracker_api = startup_tracker.clone();
let detected_ips_rx_api = detected_ips_rx.clone();
tokio::spawn(async move {
@@ -467,6 +512,7 @@ async fn run_telemt_core(
config_rx_api,
admission_rx_api,
config_path_api,
quota_state_path_api,
detected_ips_rx_api,
process_started_at_epoch_secs,
startup_tracker_api,
@@ -618,6 +664,8 @@ async fn run_telemt_core(
.await;
}
let (me_ready_tx, me_ready_rx) = watch::channel(0_u64);
let me_pool: Option<Arc<MePool>> = me_startup::initialize_me_pool(
use_middle_proxy,
&config,
@@ -628,6 +676,7 @@ async fn run_telemt_core(
rng.clone(),
stats.clone(),
api_me_pool.clone(),
me_ready_tx.clone(),
)
.await;
@@ -701,6 +750,7 @@ async fn run_telemt_core(
api_config_tx.clone(),
me_pool.clone(),
shared_state.clone(),
me_ready_tx.clone(),
)
.await;
let config_rx = runtime_watches.config_rx;
@@ -714,6 +764,7 @@ async fn run_telemt_core(
route_runtime.clone(),
&admission_tx,
config_rx.clone(),
me_ready_rx,
)
.await;
let _admission_tx_hold = admission_tx;
@@ -772,6 +823,7 @@ async fn run_telemt_core(
beobachten.clone(),
shared_state.clone(),
ip_tracker.clone(),
tls_cache.clone(),
config_rx.clone(),
)
.await;
@@ -799,7 +851,7 @@ async fn run_telemt_core(
max_connections.clone(),
);
shutdown::wait_for_shutdown(process_started_at, me_pool, stats).await;
shutdown::wait_for_shutdown(process_started_at, me_pool, stats, quota_state_path).await;
Ok(())
}

View File

@@ -21,6 +21,7 @@ use crate::startup::{
use crate::stats::beobachten::BeobachtenStore;
use crate::stats::telemetry::TelemetryPolicy;
use crate::stats::{ReplayChecker, Stats};
use crate::tls_front::TlsFrontCache;
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
@@ -52,6 +53,7 @@ pub(crate) async fn spawn_runtime_tasks(
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
me_pool_for_policy: Option<Arc<MePool>>,
shared_state: Arc<ProxySharedState>,
me_ready_tx: watch::Sender<u64>,
) -> RuntimeWatches {
let um_clone = upstream_manager.clone();
let dc_overrides_for_health = config.dc_overrides.clone();
@@ -71,6 +73,18 @@ pub(crate) async fn spawn_runtime_tasks(
rc_clone.run_periodic_cleanup().await;
});
let stats_maintenance = stats.clone();
tokio::spawn(async move {
stats_maintenance
.run_periodic_user_stats_maintenance()
.await;
});
let ip_tracker_maintenance = ip_tracker.clone();
tokio::spawn(async move {
ip_tracker_maintenance.run_periodic_maintenance().await;
});
let detected_ip_v4: Option<IpAddr> = probe.detected_ipv4.map(IpAddr::V4);
let detected_ip_v6: Option<IpAddr> = probe.detected_ipv6.map(IpAddr::V6);
debug!(
@@ -122,6 +136,8 @@ pub(crate) async fn spawn_runtime_tasks(
if let Some(pool) = &me_pool_for_policy {
pool.update_runtime_transport_policy(
cfg.general.me_socks_kdf_policy,
cfg.general.me_route_backpressure_enabled,
cfg.general.me_route_fairshare_enabled,
cfg.general.me_route_backpressure_base_timeout_ms,
cfg.general.me_route_backpressure_high_timeout_ms,
cfg.general.me_route_backpressure_high_watermark_pct,
@@ -247,12 +263,14 @@ pub(crate) async fn spawn_runtime_tasks(
let pool_clone_sched = pool.clone();
let rng_clone_sched = rng.clone();
let config_rx_clone_sched = config_rx.clone();
let me_ready_tx_sched = me_ready_tx.clone();
tokio::spawn(async move {
crate::transport::middle_proxy::me_reinit_scheduler(
pool_clone_sched,
rng_clone_sched,
config_rx_clone_sched,
reinit_rx,
me_ready_tx_sched,
)
.await;
});
@@ -326,6 +344,7 @@ pub(crate) async fn spawn_metrics_if_configured(
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
) {
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
@@ -361,6 +380,7 @@ pub(crate) async fn spawn_metrics_if_configured(
let shared_state = shared_state.clone();
let config_rx_metrics = config_rx.clone();
let ip_tracker_metrics = ip_tracker.clone();
let tls_cache_metrics = tls_cache.clone();
let whitelist = config.server.metrics_whitelist.clone();
let listen_backlog = config.server.listen_backlog;
tokio::spawn(async move {
@@ -372,6 +392,7 @@ pub(crate) async fn spawn_metrics_if_configured(
beobachten,
shared_state,
ip_tracker_metrics,
tls_cache_metrics,
config_rx_metrics,
whitelist,
)

View File

@@ -8,6 +8,7 @@
//!
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -48,9 +49,17 @@ pub(crate) async fn wait_for_shutdown(
process_started_at: Instant,
me_pool: Option<Arc<MePool>>,
stats: Arc<Stats>,
quota_state_path: PathBuf,
) {
let signal = wait_for_shutdown_signal().await;
perform_shutdown(signal, process_started_at, me_pool, &stats).await;
perform_shutdown(
signal,
process_started_at,
me_pool,
&stats,
quota_state_path,
)
.await;
}
/// Waits for any shutdown signal (SIGINT, SIGTERM, SIGQUIT).
@@ -79,6 +88,7 @@ async fn perform_shutdown(
process_started_at: Instant,
me_pool: Option<Arc<MePool>>,
stats: &Stats,
quota_state_path: PathBuf,
) {
let shutdown_started_at = Instant::now();
info!(signal = %signal, "Received shutdown signal");
@@ -109,6 +119,22 @@ async fn perform_shutdown(
}
}
match crate::quota_state::save_quota_state(&quota_state_path, stats).await {
Ok(()) => {
info!(
path = %quota_state_path.display(),
"Persisted per-user quota state"
);
}
Err(error) => {
warn!(
error = %error,
path = %quota_state_path.display(),
"Failed to persist per-user quota state"
);
}
}
let shutdown_secs = shutdown_started_at.elapsed().as_secs();
info!(
"Shutdown completed successfully in {} {}.",

View File

@@ -10,6 +10,14 @@ use crate::tls_front::TlsFrontCache;
use crate::tls_front::fetcher::TlsFetchStrategy;
use crate::transport::UpstreamManager;
fn tls_fetch_host_for_domain(mask_host: &str, primary_tls_domain: &str, domain: &str) -> String {
if mask_host.eq_ignore_ascii_case(primary_tls_domain) {
domain.to_string()
} else {
mask_host.to_string()
}
}
pub(crate) async fn bootstrap_tls_front(
config: &ProxyConfig,
tls_domains: &[String],
@@ -56,6 +64,7 @@ pub(crate) async fn bootstrap_tls_front(
let cache_initial = cache.clone();
let domains_initial = tls_domains.to_vec();
let host_initial = mask_host.clone();
let primary_initial = config.censorship.tls_domain.clone();
let unix_sock_initial = mask_unix_sock.clone();
let scope_initial = tls_fetch_scope.clone();
let upstream_initial = upstream_manager.clone();
@@ -64,7 +73,8 @@ pub(crate) async fn bootstrap_tls_front(
let mut join = tokio::task::JoinSet::new();
for domain in domains_initial {
let cache_domain = cache_initial.clone();
let host_domain = host_initial.clone();
let host_domain =
tls_fetch_host_for_domain(&host_initial, &primary_initial, &domain);
let unix_sock_domain = unix_sock_initial.clone();
let scope_domain = scope_initial.clone();
let upstream_domain = upstream_initial.clone();
@@ -117,6 +127,7 @@ pub(crate) async fn bootstrap_tls_front(
let cache_refresh = cache.clone();
let domains_refresh = tls_domains.to_vec();
let host_refresh = mask_host.clone();
let primary_refresh = config.censorship.tls_domain.clone();
let unix_sock_refresh = mask_unix_sock.clone();
let scope_refresh = tls_fetch_scope.clone();
let upstream_refresh = upstream_manager.clone();
@@ -130,7 +141,8 @@ pub(crate) async fn bootstrap_tls_front(
let mut join = tokio::task::JoinSet::new();
for domain in domains_refresh.clone() {
let cache_domain = cache_refresh.clone();
let host_domain = host_refresh.clone();
let host_domain =
tls_fetch_host_for_domain(&host_refresh, &primary_refresh, &domain);
let unix_sock_domain = unix_sock_refresh.clone();
let scope_domain = scope_refresh.clone();
let upstream_domain = upstream_refresh.clone();
@@ -186,3 +198,24 @@ pub(crate) async fn bootstrap_tls_front(
tls_cache
}
#[cfg(test)]
mod tests {
use super::tls_fetch_host_for_domain;
#[test]
fn tls_fetch_host_uses_each_domain_when_mask_host_is_primary_default() {
assert_eq!(
tls_fetch_host_for_domain("a.com", "a.com", "b.com"),
"b.com"
);
}
#[test]
fn tls_fetch_host_preserves_explicit_non_primary_mask_host() {
assert_eq!(
tls_fetch_host_for_domain("origin.example", "a.com", "b.com"),
"origin.example"
);
}
}

View File

@@ -8,6 +8,7 @@ mod crypto;
#[cfg(unix)]
mod daemon;
mod error;
mod healthcheck;
mod ip_tracker;
#[cfg(test)]
#[path = "tests/ip_tracker_encapsulation_adversarial_tests.rs"]
@@ -24,6 +25,7 @@ mod metrics;
mod network;
mod protocol;
mod proxy;
mod quota_state;
mod service;
mod startup;
mod stats;

View File

@@ -11,6 +11,8 @@ use hyper::service::service_fn;
use hyper::{Request, Response, StatusCode};
use ipnetwork::IpNetwork;
use tokio::net::TcpListener;
use tokio::sync::Semaphore;
use tokio::time::timeout;
use tracing::{debug, info, warn};
use crate::config::ProxyConfig;
@@ -18,8 +20,18 @@ use crate::ip_tracker::UserIpTracker;
use crate::proxy::shared_state::ProxySharedState;
use crate::stats::Stats;
use crate::stats::beobachten::BeobachtenStore;
use crate::tls_front::TlsFrontCache;
use crate::tls_front::cache;
use crate::tls_front::fetcher;
use crate::transport::{ListenOptions, create_listener};
// Keeps `/metrics` response size bounded when per-user telemetry is enabled.
const USER_LABELED_METRICS_MAX_USERS: usize = 4096;
// Keeps TLS-front per-domain health series bounded for large generated configs.
const TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS: usize = 256;
const METRICS_MAX_CONTROL_CONNECTIONS: usize = 512;
const METRICS_HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
pub async fn serve(
port: u16,
listen: Option<String>,
@@ -28,6 +40,7 @@ pub async fn serve(
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Vec<IpNetwork>,
) {
@@ -52,6 +65,7 @@ pub async fn serve(
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
@@ -64,11 +78,11 @@ pub async fn serve(
return;
}
// Fallback: bind on 0.0.0.0 and [::] using metrics_port.
// Fallback: keep metrics local unless an explicit metrics_listen is configured.
let mut listener_v4 = None;
let mut listener_v6 = None;
let addr_v4 = SocketAddr::from(([0, 0, 0, 0], port));
let addr_v4 = SocketAddr::from(([127, 0, 0, 1], port));
match bind_metrics_listener(addr_v4, false, listen_backlog) {
Ok(listener) => {
info!(
@@ -82,11 +96,11 @@ pub async fn serve(
}
}
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], port));
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 1], port));
match bind_metrics_listener(addr_v6, true, listen_backlog) {
Ok(listener) => {
info!(
"Metrics endpoint: http://[::]:{}/metrics and /beobachten",
"Metrics endpoint: http://[::1]:{}/metrics and /beobachten",
port
);
listener_v6 = Some(listener);
@@ -107,6 +121,7 @@ pub async fn serve(
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
@@ -117,6 +132,7 @@ pub async fn serve(
let beobachten_v6 = beobachten.clone();
let shared_state_v6 = shared_state.clone();
let ip_tracker_v6 = ip_tracker.clone();
let tls_cache_v6 = tls_cache.clone();
let config_rx_v6 = config_rx.clone();
let whitelist_v6 = whitelist.clone();
tokio::spawn(async move {
@@ -126,6 +142,7 @@ pub async fn serve(
beobachten_v6,
shared_state_v6,
ip_tracker_v6,
tls_cache_v6,
config_rx_v6,
whitelist_v6,
)
@@ -137,6 +154,7 @@ pub async fn serve(
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
@@ -166,9 +184,12 @@ async fn serve_listener(
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Arc<Vec<IpNetwork>>,
) {
let connection_permits = Arc::new(Semaphore::new(METRICS_MAX_CONTROL_CONNECTIONS));
loop {
let (stream, peer) = match listener.accept().await {
Ok(v) => v,
@@ -183,17 +204,32 @@ async fn serve_listener(
continue;
}
let connection_permit = match connection_permits.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
debug!(
peer = %peer,
max_connections = METRICS_MAX_CONTROL_CONNECTIONS,
"Dropping metrics connection: control-plane connection budget exhausted"
);
continue;
}
};
let stats = stats.clone();
let beobachten = beobachten.clone();
let shared_state = shared_state.clone();
let ip_tracker = ip_tracker.clone();
let tls_cache = tls_cache.clone();
let config_rx_conn = config_rx.clone();
tokio::spawn(async move {
let _connection_permit = connection_permit;
let svc = service_fn(move |req| {
let stats = stats.clone();
let beobachten = beobachten.clone();
let shared_state = shared_state.clone();
let ip_tracker = ip_tracker.clone();
let tls_cache = tls_cache.clone();
let config = config_rx_conn.borrow().clone();
async move {
handle(
@@ -202,16 +238,29 @@ async fn serve_listener(
&beobachten,
&shared_state,
&ip_tracker,
tls_cache.as_deref(),
&config,
)
.await
}
});
if let Err(e) = http1::Builder::new()
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
.await
match timeout(
METRICS_HTTP_CONNECTION_TIMEOUT,
http1::Builder::new().serve_connection(hyper_util::rt::TokioIo::new(stream), svc),
)
.await
{
debug!(error = %e, "Metrics connection error");
Ok(Ok(())) => {}
Ok(Err(e)) => {
debug!(error = %e, "Metrics connection error");
}
Err(_) => {
debug!(
peer = %peer,
timeout_ms = METRICS_HTTP_CONNECTION_TIMEOUT.as_millis() as u64,
"Metrics connection timed out"
);
}
}
});
}
@@ -223,10 +272,11 @@ async fn handle<B>(
beobachten: &BeobachtenStore,
shared_state: &ProxySharedState,
ip_tracker: &UserIpTracker,
tls_cache: Option<&TlsFrontCache>,
config: &ProxyConfig,
) -> Result<Response<Full<Bytes>>, Infallible> {
if req.uri().path() == "/metrics" {
let body = render_metrics(stats, shared_state, config, ip_tracker).await;
let body = render_metrics(stats, shared_state, config, ip_tracker, tls_cache).await;
let resp = Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/plain; version=0.0.4; charset=utf-8")
@@ -261,11 +311,138 @@ fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> Stri
beobachten.snapshot_text(ttl)
}
fn tls_front_domains(config: &ProxyConfig) -> Vec<String> {
let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
if !config.censorship.tls_domain.is_empty() {
domains.push(config.censorship.tls_domain.clone());
}
for domain in &config.censorship.tls_domains {
if !domain.is_empty() && !domains.contains(domain) {
domains.push(domain.clone());
}
}
domains
}
fn prometheus_label_value(value: &str) -> String {
value.replace('\\', "\\\\").replace('"', "\\\"")
}
async fn render_tls_front_profile_health(
out: &mut String,
config: &ProxyConfig,
tls_cache: Option<&TlsFrontCache>,
) {
use std::fmt::Write;
let domains = tls_front_domains(config);
let (health, suppressed) = match (config.censorship.tls_emulation, tls_cache) {
(true, Some(cache)) => {
cache
.profile_health_snapshot(&domains, TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS)
.await
}
_ => (Vec::new(), domains.len()),
};
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_domains TLS front configured profile domains by export status"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_domains gauge");
let _ = writeln!(
out,
"telemt_tls_front_profile_domains{{status=\"configured\"}} {}",
domains.len()
);
let _ = writeln!(
out,
"telemt_tls_front_profile_domains{{status=\"emitted\"}} {}",
health.len()
);
let _ = writeln!(
out,
"telemt_tls_front_profile_domains{{status=\"suppressed\"}} {}",
suppressed
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_info TLS front profile source and feature flags per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_info gauge");
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_age_seconds Age of cached TLS front profile data per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_age_seconds gauge");
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_app_data_records TLS front cached app-data record count per configured domain"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_front_profile_app_data_records gauge"
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_ticket_records TLS front cached ticket-like tail record count per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_ticket_records gauge");
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_change_cipher_spec_records TLS front cached ChangeCipherSpec record count per configured domain"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge"
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_app_data_bytes TLS front cached total app-data bytes per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_app_data_bytes gauge");
for item in health {
let domain = prometheus_label_value(&item.domain);
let _ = writeln!(
out,
"telemt_tls_front_profile_info{{domain=\"{}\",source=\"{}\",is_default=\"{}\",has_cert_info=\"{}\",has_cert_payload=\"{}\"}} 1",
domain, item.source, item.is_default, item.has_cert_info, item.has_cert_payload
);
let _ = writeln!(
out,
"telemt_tls_front_profile_age_seconds{{domain=\"{}\"}} {}",
domain, item.age_seconds
);
let _ = writeln!(
out,
"telemt_tls_front_profile_app_data_records{{domain=\"{}\"}} {}",
domain, item.app_data_records
);
let _ = writeln!(
out,
"telemt_tls_front_profile_ticket_records{{domain=\"{}\"}} {}",
domain, item.ticket_records
);
let _ = writeln!(
out,
"telemt_tls_front_profile_change_cipher_spec_records{{domain=\"{}\"}} {}",
domain, item.change_cipher_spec_count
);
let _ = writeln!(
out,
"telemt_tls_front_profile_app_data_bytes{{domain=\"{}\"}} {}",
domain, item.total_app_data_len
);
}
}
async fn render_metrics(
stats: &Stats,
shared_state: &ProxySharedState,
config: &ProxyConfig,
ip_tracker: &UserIpTracker,
tls_cache: Option<&TlsFrontCache>,
) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(4096);
@@ -311,6 +488,12 @@ async fn render_metrics(
"telemt_telemetry_user_enabled {}",
if user_enabled { 1 } else { 0 }
);
let _ = writeln!(
out,
"# HELP telemt_stats_user_entries Retained per-user stats entries"
);
let _ = writeln!(out, "# TYPE telemt_stats_user_entries gauge");
let _ = writeln!(out, "telemt_stats_user_entries {}", stats.user_stats_len());
let _ = writeln!(
out,
@@ -366,6 +549,54 @@ async fn render_metrics(
stats.get_buffer_pool_in_use_gauge()
);
let _ = writeln!(
out,
"# HELP telemt_tls_fetch_profile_cache_entries Current adaptive TLS fetch profile-cache entries"
);
let _ = writeln!(out, "# TYPE telemt_tls_fetch_profile_cache_entries gauge");
let _ = writeln!(
out,
"telemt_tls_fetch_profile_cache_entries {}",
fetcher::profile_cache_entries_for_metrics()
);
let _ = writeln!(
out,
"# HELP telemt_tls_fetch_profile_cache_cap_drops_total Profile-cache winner inserts skipped because the cache cap was reached"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"
);
let _ = writeln!(
out,
"telemt_tls_fetch_profile_cache_cap_drops_total {}",
fetcher::profile_cache_cap_drops_for_metrics()
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_full_cert_budget_ips Current IP entries tracked by TLS full-cert budget"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_full_cert_budget_ips gauge");
let _ = writeln!(
out,
"telemt_tls_front_full_cert_budget_ips {}",
cache::full_cert_sent_ips_for_metrics()
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_full_cert_budget_cap_drops_total New IPs denied full-cert budget tracking because the cap was reached"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter"
);
let _ = writeln!(
out,
"telemt_tls_front_full_cert_budget_cap_drops_total {}",
cache::full_cert_sent_cap_drops_for_metrics()
);
render_tls_front_profile_health(&mut out, config, tls_cache).await;
let _ = writeln!(
out,
"# HELP telemt_connections_total Total accepted connections"
@@ -396,6 +627,21 @@ async fn render_metrics(
}
);
let _ = writeln!(
out,
"# HELP telemt_connections_bad_by_class_total Bad/rejected connections by class"
);
let _ = writeln!(out, "# TYPE telemt_connections_bad_by_class_total counter");
if core_enabled {
for (class, total) in stats.get_connects_bad_class_counts() {
let _ = writeln!(
out,
"telemt_connections_bad_by_class_total{{class=\"{}\"}} {}",
class, total
);
}
}
let _ = writeln!(
out,
"# HELP telemt_handshake_timeouts_total Handshake timeouts"
@@ -411,6 +657,24 @@ async fn render_metrics(
}
);
let _ = writeln!(
out,
"# HELP telemt_handshake_failures_by_class_total Handshake failures by class"
);
let _ = writeln!(
out,
"# TYPE telemt_handshake_failures_by_class_total counter"
);
if core_enabled {
for (class, total) in stats.get_handshake_failure_class_counts() {
let _ = writeln!(
out,
"telemt_handshake_failures_by_class_total{{class=\"{}\"}} {}",
class, total
);
}
}
let _ = writeln!(
out,
"# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation"
@@ -462,6 +726,63 @@ async fn render_metrics(
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_refund_bytes_total Reserved quota bytes returned before commit"
);
let _ = writeln!(out, "# TYPE telemt_quota_refund_bytes_total counter");
let _ = writeln!(
out,
"telemt_quota_refund_bytes_total {}",
if core_enabled {
stats.get_quota_refund_bytes_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_contention_total Quota reservation CAS contention events"
);
let _ = writeln!(out, "# TYPE telemt_quota_contention_total counter");
let _ = writeln!(
out,
"telemt_quota_contention_total {}",
if core_enabled {
stats.get_quota_contention_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_contention_timeout_total Quota reservations that hit the bounded contention budget"
);
let _ = writeln!(out, "# TYPE telemt_quota_contention_timeout_total counter");
let _ = writeln!(
out,
"telemt_quota_contention_timeout_total {}",
if core_enabled {
stats.get_quota_contention_timeout_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_acquire_cancelled_total Quota acquisitions cancelled before reservation completed"
);
let _ = writeln!(out, "# TYPE telemt_quota_acquire_cancelled_total counter");
let _ = writeln!(
out,
"telemt_quota_acquire_cancelled_total {}",
if core_enabled {
stats.get_quota_acquire_cancelled_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_conntrack_control_state Runtime conntrack control state flags"
@@ -576,6 +897,29 @@ async fn render_metrics(
);
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
let _ = writeln!(
out,
"# HELP telemt_rate_limiter_burst_bound_bytes Configured upper bound for one direct relay rate-limit burst"
);
let _ = writeln!(out, "# TYPE telemt_rate_limiter_burst_bound_bytes gauge");
let _ = writeln!(
out,
"telemt_rate_limiter_burst_bound_bytes{{direction=\"up\"}} {}",
if core_enabled {
config.general.direct_relay_copy_buf_c2s_bytes
} else {
0
}
);
let _ = writeln!(
out,
"telemt_rate_limiter_burst_bound_bytes{{direction=\"down\"}} {}",
if core_enabled {
config.general.direct_relay_copy_buf_s2c_bytes
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
@@ -1678,6 +2022,85 @@ async fn render_metrics(
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_child_join_timeout_total Middle relay child tasks that did not join before cleanup deadline"
);
let _ = writeln!(out, "# TYPE telemt_me_child_join_timeout_total counter");
let _ = writeln!(
out,
"telemt_me_child_join_timeout_total {}",
if core_enabled {
stats.get_me_child_join_timeout_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_child_abort_total Middle relay child tasks aborted after bounded cleanup timeout"
);
let _ = writeln!(out, "# TYPE telemt_me_child_abort_total counter");
let _ = writeln!(
out,
"telemt_me_child_abort_total {}",
if core_enabled {
stats.get_me_child_abort_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_flow_wait_events_total Flow wait events by reason, direction, and outcome"
);
let _ = writeln!(out, "# TYPE telemt_flow_wait_events_total counter");
let _ = writeln!(
out,
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"waited\"}} {}",
if core_enabled {
stats.get_flow_wait_middle_rate_limit_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"cancelled\"}} {}",
if core_enabled {
stats.get_flow_wait_middle_rate_limit_cancelled_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_flow_wait_ms_total Flow wait time in milliseconds by reason and direction"
);
let _ = writeln!(out, "# TYPE telemt_flow_wait_ms_total counter");
let _ = writeln!(
out,
"telemt_flow_wait_ms_total{{reason=\"middle_rate_limit\",direction=\"down\"}} {}",
if core_enabled {
stats.get_flow_wait_middle_rate_limit_ms_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_session_drop_fallback_total Session reservations cleaned by Drop instead of explicit async release"
);
let _ = writeln!(out, "# TYPE telemt_session_drop_fallback_total counter");
let _ = writeln!(
out,
"telemt_session_drop_fallback_total {}",
if core_enabled {
stats.get_session_drop_fallback_total()
} else {
0
}
);
let _ = writeln!(
out,
@@ -3019,17 +3442,6 @@ async fn render_metrics(
0
}
);
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_suppressed {}",
if user_enabled { 0 } else { 1 }
);
let ip_memory = ip_tracker.memory_stats().await;
let _ = writeln!(
out,
@@ -3071,11 +3483,46 @@ async fn render_metrics(
"telemt_ip_tracker_cleanup_queue_len {}",
ip_memory.cleanup_queue_len
);
let _ = writeln!(
out,
"# HELP telemt_ip_tracker_cleanup_total Release cleanups deferred through the cleanup queue"
);
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cleanup_total counter");
let _ = writeln!(
out,
"telemt_ip_tracker_cleanup_total{{path=\"deferred\"}} {}",
ip_memory.cleanup_deferred_releases
);
let _ = writeln!(
out,
"# HELP telemt_ip_tracker_cap_rejects_total New connection rejects caused by global IP tracker caps"
);
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cap_rejects_total counter");
let _ = writeln!(
out,
"telemt_ip_tracker_cap_rejects_total{{scope=\"active\"}} {}",
ip_memory.active_cap_rejects
);
let _ = writeln!(
out,
"telemt_ip_tracker_cap_rejects_total{{scope=\"recent\"}} {}",
ip_memory.recent_cap_rejects
);
let mut user_stats_emitted = 0usize;
let mut user_stats_suppressed = 0usize;
let mut unique_ip_emitted = 0usize;
let mut unique_ip_suppressed = 0usize;
if user_enabled {
for entry in stats.iter_user_stats() {
if user_stats_emitted >= USER_LABELED_METRICS_MAX_USERS {
user_stats_suppressed = user_stats_suppressed.saturating_add(1);
continue;
}
let user = entry.key();
let s = entry.value();
user_stats_emitted = user_stats_emitted.saturating_add(1);
let _ = writeln!(
out,
"telemt_user_connections_total{{user=\"{}\"}} {}",
@@ -3117,7 +3564,7 @@ async fn render_metrics(
);
}
let ip_stats = ip_tracker.get_stats().await;
let ip_stats = ip_tracker.get_stats_snapshot().await;
let ip_counts: HashMap<String, usize> = ip_stats
.into_iter()
.map(|(user, count, _)| (user, count))
@@ -3129,7 +3576,7 @@ async fn render_metrics(
unique_users.extend(ip_counts.keys().cloned());
let unique_users_vec: Vec<String> = unique_users.iter().cloned().collect();
let recent_counts = ip_tracker
.get_recent_counts_for_users(&unique_users_vec)
.get_recent_counts_for_users_snapshot(&unique_users_vec)
.await;
let _ = writeln!(
@@ -3154,6 +3601,11 @@ async fn render_metrics(
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_utilization gauge");
for user in unique_users {
if unique_ip_emitted >= USER_LABELED_METRICS_MAX_USERS {
unique_ip_suppressed = unique_ip_suppressed.saturating_add(1);
continue;
}
unique_ip_emitted = unique_ip_emitted.saturating_add(1);
let current = ip_counts.get(&user).copied().unwrap_or(0);
let limit = config
.access
@@ -3193,6 +3645,46 @@ async fn render_metrics(
}
}
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_suppressed {}",
if user_enabled && user_stats_suppressed == 0 && unique_ip_suppressed == 0 {
0
} else {
1
}
);
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_users User-labeled metric users by export status"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_users gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"emitted\"}} {}",
user_stats_emitted
);
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"suppressed\"}} {}",
user_stats_suppressed
);
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"emitted\"}} {}",
unique_ip_emitted
);
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"suppressed\"}} {}",
unique_ip_suppressed
);
out
}
@@ -3201,6 +3693,11 @@ mod tests {
use super::*;
use http_body_util::BodyExt;
use std::net::IpAddr;
use std::time::SystemTime;
use crate::tls_front::types::{
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
};
#[tokio::test]
async fn test_render_metrics_format() {
@@ -3215,8 +3712,9 @@ mod tests {
stats.increment_connects_all();
stats.increment_connects_all();
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
stats.increment_handshake_timeouts();
stats.increment_handshake_failure_class("timeout");
shared_state
.handshake
.auth_expensive_checks_total
@@ -3268,7 +3766,7 @@ mod tests {
.await
.unwrap();
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker).await;
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker, None).await;
assert!(output.contains(&format!(
"telemt_build_info{{version=\"{}\"}} 1",
@@ -3276,7 +3774,11 @@ mod tests {
)));
assert!(output.contains("telemt_connections_total 2"));
assert!(output.contains("telemt_connections_bad_total 1"));
assert!(output.contains(
"telemt_connections_bad_by_class_total{class=\"tls_handshake_bad_client\"} 1"
));
assert!(output.contains("telemt_handshake_timeouts_total 1"));
assert!(output.contains("telemt_handshake_failures_by_class_total{class=\"timeout\"} 1"));
assert!(output.contains("telemt_auth_expensive_checks_total 9"));
assert!(output.contains("telemt_auth_budget_exhausted_total 2"));
assert!(output.contains("telemt_upstream_connect_attempt_total 2"));
@@ -3330,13 +3832,91 @@ mod tests {
assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0"));
}
#[tokio::test]
async fn test_render_tls_front_profile_health() {
let stats = Stats::new();
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let mut config = ProxyConfig::default();
config.censorship.tls_domain = "primary.example".to_string();
config.censorship.tls_domains = vec!["fallback.example".to_string()];
let cache = TlsFrontCache::new(
&[
"primary.example".to_string(),
"fallback.example".to_string(),
],
1024,
"tlsfront-profile-health-test",
);
cache
.set(
"primary.example",
CachedTlsData {
server_hello_template: ParsedServerHello {
version: [0x03, 0x03],
random: [0u8; 32],
session_id: Vec::new(),
cipher_suite: [0x13, 0x01],
compression: 0,
extensions: Vec::new(),
},
cert_info: None,
cert_payload: Some(TlsCertPayload {
cert_chain_der: vec![vec![0x30, 0x01]],
certificate_message: vec![0x0b, 0x00, 0x00, 0x00],
}),
app_data_records_sizes: vec![1024, 512],
total_app_data_len: 1536,
behavior_profile: TlsBehaviorProfile {
change_cipher_spec_count: 1,
app_data_record_sizes: vec![1024, 512],
ticket_record_sizes: vec![69],
source: TlsProfileSource::Merged,
},
fetched_at: SystemTime::now(),
domain: "primary.example".to_string(),
},
)
.await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, Some(&cache)).await;
assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2"));
assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2"));
assert!(output.contains("telemt_tls_front_profile_domains{status=\"suppressed\"} 0"));
assert!(
output.contains("telemt_tls_front_profile_info{domain=\"primary.example\",source=\"merged\",is_default=\"false\",has_cert_info=\"false\",has_cert_payload=\"true\"} 1")
);
assert!(
output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1")
);
assert!(
output.contains(
"telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2"
)
);
assert!(
output
.contains("telemt_tls_front_profile_ticket_records{domain=\"primary.example\"} 1")
);
assert!(output.contains(
"telemt_tls_front_profile_change_cipher_spec_records{domain=\"primary.example\"} 1"
));
assert!(
output.contains(
"telemt_tls_front_profile_app_data_bytes{domain=\"primary.example\"} 1536"
)
);
}
#[tokio::test]
async fn test_render_empty_stats() {
let stats = Stats::new();
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
assert!(output.contains("telemt_connections_total 0"));
assert!(output.contains("telemt_connections_bad_total 0"));
assert!(output.contains("telemt_handshake_timeouts_total 0"));
@@ -3360,7 +3940,7 @@ mod tests {
let mut config = ProxyConfig::default();
config.access.user_max_unique_ips_global_each = 2;
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
@@ -3372,11 +3952,13 @@ mod tests {
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
assert!(output.contains("# TYPE telemt_connections_total counter"));
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
assert!(output.contains("# TYPE telemt_connections_bad_by_class_total counter"));
assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter"));
assert!(output.contains("# TYPE telemt_handshake_failures_by_class_total counter"));
assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter"));
assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter"));
assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter"));
@@ -3406,9 +3988,28 @@ mod tests {
assert!(output.contains("# TYPE telemt_user_unique_ips_recent_window gauge"));
assert!(output.contains("# TYPE telemt_user_unique_ips_limit gauge"));
assert!(output.contains("# TYPE telemt_user_unique_ips_utilization gauge"));
assert!(output.contains("# TYPE telemt_stats_user_entries gauge"));
assert!(output.contains("# TYPE telemt_telemetry_user_series_users gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_users gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_entries gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_queue_len gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_total counter"));
assert!(output.contains("# TYPE telemt_ip_tracker_cap_rejects_total counter"));
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_entries gauge"));
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"));
assert!(output.contains("# TYPE telemt_tls_front_full_cert_budget_ips gauge"));
assert!(
output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter")
);
assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge"));
assert!(
output.contains("# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge")
);
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_bytes gauge"));
}
#[tokio::test]
@@ -3429,6 +4030,7 @@ mod tests {
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
@@ -3463,6 +4065,7 @@ mod tests {
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
@@ -3480,6 +4083,7 @@ mod tests {
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await

View File

@@ -1383,6 +1383,8 @@ fn emulated_server_hello_never_places_alpn_in_server_hello_extensions() {
&session_id,
&cached,
false,
true,
ClientHelloTlsVersion::Tls13,
&rng,
Some(b"h2".to_vec()),
0,
@@ -1624,6 +1626,34 @@ fn test_extract_alpn_multiple() {
assert_eq!(alpn_str, vec!["h2", "spdy", "h3"]);
}
#[test]
fn detect_client_hello_tls_version_prefers_supported_versions_tls13() {
let supported_versions = vec![4, 0x03, 0x04, 0x03, 0x03];
let ch = build_client_hello_with_exts(vec![(0x002b, supported_versions)], "example.com");
assert_eq!(
detect_client_hello_tls_version(&ch),
Some(ClientHelloTlsVersion::Tls13)
);
}
#[test]
fn detect_client_hello_tls_version_falls_back_to_legacy_tls12() {
let ch = build_client_hello_with_exts(Vec::new(), "example.com");
assert_eq!(
detect_client_hello_tls_version(&ch),
Some(ClientHelloTlsVersion::Tls12)
);
}
#[test]
fn detect_client_hello_tls_version_rejects_malformed_supported_versions() {
// list_len=3 is invalid because version vector must contain u16 pairs.
let malformed_supported_versions = vec![3, 0x03, 0x04, 0x03];
let ch =
build_client_hello_with_exts(vec![(0x002b, malformed_supported_versions)], "example.com");
assert!(detect_client_hello_tls_version(&ch).is_none());
}
#[test]
fn extract_sni_rejects_zero_length_host_name() {
let mut sni_ext = Vec::new();

View File

@@ -811,6 +811,122 @@ pub fn extract_alpn_from_client_hello(handshake: &[u8]) -> Vec<Vec<u8>> {
out
}
/// ClientHello TLS generation inferred from handshake fields.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ClientHelloTlsVersion {
Tls12,
Tls13,
}
/// Detect TLS generation from a ClientHello.
///
/// The parser prefers `supported_versions` (0x002b) when present and falls back
/// to `legacy_version` for compatibility with TLS 1.2 style hellos.
pub fn detect_client_hello_tls_version(handshake: &[u8]) -> Option<ClientHelloTlsVersion> {
if handshake.len() < 5 || handshake[0] != TLS_RECORD_HANDSHAKE {
return None;
}
let record_len = u16::from_be_bytes([handshake[3], handshake[4]]) as usize;
if handshake.len() < 5 + record_len {
return None;
}
let mut pos = 5; // after record header
if handshake.get(pos) != Some(&0x01) {
return None; // not ClientHello
}
pos += 1; // message type
if pos + 3 > handshake.len() {
return None;
}
let handshake_len = ((handshake[pos] as usize) << 16)
| ((handshake[pos + 1] as usize) << 8)
| handshake[pos + 2] as usize;
pos += 3; // handshake length bytes
if pos + handshake_len > 5 + record_len {
return None;
}
if pos + 2 + 32 > handshake.len() {
return None;
}
let legacy_version = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]);
pos += 2 + 32; // version + random
let session_id_len = *handshake.get(pos)? as usize;
pos += 1 + session_id_len;
if pos + 2 > handshake.len() {
return None;
}
let cipher_len = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]) as usize;
pos += 2 + cipher_len;
if pos >= handshake.len() {
return None;
}
let comp_len = *handshake.get(pos)? as usize;
pos += 1 + comp_len;
if pos + 2 > handshake.len() {
return None;
}
let ext_len = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]) as usize;
pos += 2;
let ext_end = pos + ext_len;
if ext_end > handshake.len() {
return None;
}
while pos + 4 <= ext_end {
let etype = u16::from_be_bytes([handshake[pos], handshake[pos + 1]]);
let elen = u16::from_be_bytes([handshake[pos + 2], handshake[pos + 3]]) as usize;
pos += 4;
if pos + elen > ext_end {
return None;
}
if etype == extension_type::SUPPORTED_VERSIONS {
if elen < 1 {
return None;
}
let list_len = handshake[pos] as usize;
if list_len == 0 || list_len % 2 != 0 || 1 + list_len > elen {
return None;
}
let mut has_tls12 = false;
let mut ver_pos = pos + 1;
let ver_end = ver_pos + list_len;
while ver_pos + 1 < ver_end {
let version = u16::from_be_bytes([handshake[ver_pos], handshake[ver_pos + 1]]);
if version == 0x0304 {
return Some(ClientHelloTlsVersion::Tls13);
}
if version == 0x0303 || version == 0x0302 || version == 0x0301 {
has_tls12 = true;
}
ver_pos += 2;
}
if has_tls12 {
return Some(ClientHelloTlsVersion::Tls12);
}
return None;
}
pos += elen;
}
if legacy_version >= 0x0303 {
Some(ClientHelloTlsVersion::Tls12)
} else {
None
}
}
/// Check if bytes look like a TLS ClientHello
pub fn is_tls_handshake(first_bytes: &[u8]) -> bool {
if first_bytes.len() < 3 {

View File

@@ -31,38 +31,63 @@ struct UserConnectionReservation {
ip_tracker: Arc<UserIpTracker>,
user: String,
ip: IpAddr,
active: bool,
tracks_ip: bool,
state: SessionReservationState,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum SessionReservationState {
Active,
Released,
}
impl UserConnectionReservation {
fn new(stats: Arc<Stats>, ip_tracker: Arc<UserIpTracker>, user: String, ip: IpAddr) -> Self {
fn new(
stats: Arc<Stats>,
ip_tracker: Arc<UserIpTracker>,
user: String,
ip: IpAddr,
tracks_ip: bool,
) -> Self {
Self {
stats,
ip_tracker,
user,
ip,
active: true,
tracks_ip,
state: SessionReservationState::Active,
}
}
fn mark_released(&mut self) -> bool {
if self.state != SessionReservationState::Active {
return false;
}
self.state = SessionReservationState::Released;
true
}
async fn release(mut self) {
if !self.active {
if !self.mark_released() {
return;
}
self.ip_tracker.remove_ip(&self.user, self.ip).await;
self.active = false;
if self.tracks_ip {
self.ip_tracker.remove_ip(&self.user, self.ip).await;
}
self.stats.decrement_user_curr_connects(&self.user);
}
}
impl Drop for UserConnectionReservation {
fn drop(&mut self) {
if !self.active {
if !self.mark_released() {
return;
}
self.active = false;
self.stats.increment_session_drop_fallback_total();
self.stats.decrement_user_curr_connects(&self.user);
self.ip_tracker.enqueue_cleanup(self.user.clone(), self.ip);
if self.tracks_ip {
self.ip_tracker.enqueue_cleanup(self.user.clone(), self.ip);
}
}
}
@@ -324,17 +349,38 @@ fn record_beobachten_class(
beobachten.record(class, peer_ip, beobachten_ttl(config));
}
fn classify_expected_64_got_0(kind: std::io::ErrorKind) -> Option<&'static str> {
match kind {
std::io::ErrorKind::UnexpectedEof => Some("expected_64_got_0_unexpected_eof"),
std::io::ErrorKind::ConnectionReset => Some("expected_64_got_0_connection_reset"),
std::io::ErrorKind::ConnectionAborted => Some("expected_64_got_0_connection_aborted"),
std::io::ErrorKind::BrokenPipe => Some("expected_64_got_0_broken_pipe"),
std::io::ErrorKind::NotConnected => Some("expected_64_got_0_not_connected"),
_ => None,
}
}
fn classify_handshake_failure_class(error: &ProxyError) -> &'static str {
match error {
ProxyError::Io(err) => classify_expected_64_got_0(err.kind()).unwrap_or("other"),
ProxyError::Stream(StreamError::UnexpectedEof) => "expected_64_got_0_unexpected_eof",
ProxyError::Stream(StreamError::Io(err)) => {
classify_expected_64_got_0(err.kind()).unwrap_or("other")
}
_ => "other",
}
}
fn record_handshake_failure_class(
beobachten: &BeobachtenStore,
config: &ProxyConfig,
peer_ip: IpAddr,
error: &ProxyError,
) {
let class = match error {
ProxyError::Io(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
"expected_64_got_0"
}
ProxyError::Stream(StreamError::UnexpectedEof) => "expected_64_got_0",
// Keep beobachten buckets stable while detailed per-kind classification
// is tracked in API counters.
let class = match classify_handshake_failure_class(error) {
value if value.starts_with("expected_64_got_0_") => "expected_64_got_0",
_ => "other",
};
record_beobachten_class(beobachten, config, peer_ip, class);
@@ -343,7 +389,7 @@ fn record_handshake_failure_class(
#[inline]
fn increment_bad_on_unknown_tls_sni(stats: &Stats, error: &ProxyError) {
if matches!(error, ProxyError::UnknownTlsSni) {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("unknown_tls_sni");
}
}
@@ -433,6 +479,17 @@ where
let mut local_addr = synthetic_local_addr(config.server.port);
if proxy_protocol_enabled {
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs) {
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
warn!(
peer = %peer,
trusted = ?config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
}
let proxy_header_timeout =
Duration::from_millis(config.server.proxy_protocol_header_timeout_ms.max(1));
match timeout(
@@ -442,17 +499,6 @@ where
.await
{
Ok(Ok(info)) => {
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs)
{
stats.increment_connects_bad();
warn!(
peer = %peer,
trusted = ?config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
}
debug!(
peer = %peer,
client = %info.src_addr,
@@ -465,13 +511,13 @@ where
}
}
Ok(Err(e)) => {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("proxy_protocol_invalid_header");
warn!(peer = %peer, error = %e, "Invalid PROXY protocol header");
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
return Err(e);
}
Err(_) => {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("proxy_protocol_header_timeout");
warn!(peer = %peer, timeout_ms = proxy_header_timeout.as_millis(), "PROXY protocol header timeout");
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
@@ -561,7 +607,7 @@ where
// third-party clients or future Telegram versions.
if !tls_clienthello_len_in_bounds(tls_len) {
debug!(peer = %real_peer, tls_len = tls_len, max_tls_len = MAX_TLS_PLAINTEXT_SIZE, "TLS handshake length out of bounds");
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_clienthello_len_out_of_bounds");
maybe_apply_mask_reject_delay(&config).await;
let (reader, writer) = tokio::io::split(stream);
return Ok(masking_outcome(
@@ -581,7 +627,7 @@ where
Ok(n) => n,
Err(e) => {
debug!(peer = %real_peer, error = %e, tls_len = tls_len, "TLS ClientHello body read failed; engaging masking fallback");
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_clienthello_read_error");
maybe_apply_mask_reject_delay(&config).await;
let initial_len = 5;
let (reader, writer) = tokio::io::split(stream);
@@ -599,7 +645,7 @@ where
if body_read < tls_len {
debug!(peer = %real_peer, got = body_read, expected = tls_len, "Truncated in-range TLS ClientHello; engaging masking fallback");
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_clienthello_truncated");
maybe_apply_mask_reject_delay(&config).await;
let initial_len = 5 + body_read;
let (reader, writer) = tokio::io::split(stream);
@@ -623,7 +669,7 @@ where
).await {
HandshakeResult::Success(result) => result,
HandshakeResult::BadClient { reader, writer } => {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
return Ok(masking_outcome(
reader,
writer,
@@ -663,7 +709,7 @@ where
wrap_tls_application_record(&pending_plaintext)
};
let reader = tokio::io::AsyncReadExt::chain(std::io::Cursor::new(pending_record), reader);
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_mtproto_bad_client");
debug!(
peer = %peer,
"Authenticated TLS session failed MTProto validation; engaging masking fallback"
@@ -693,7 +739,7 @@ where
} else {
if !config.general.modes.classic && !config.general.modes.secure {
debug!(peer = %real_peer, "Non-TLS modes disabled");
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("direct_modes_disabled");
maybe_apply_mask_reject_delay(&config).await;
let (reader, writer) = tokio::io::split(stream);
return Ok(masking_outcome(
@@ -720,7 +766,7 @@ where
).await {
HandshakeResult::Success(result) => result,
HandshakeResult::BadClient { reader, writer } => {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("direct_mtproto_bad_client");
return Ok(masking_outcome(
reader,
writer,
@@ -757,6 +803,7 @@ where
Ok(Ok(outcome)) => outcome,
Ok(Err(e)) => {
debug!(peer = %peer, error = %e, "Handshake failed");
stats_for_timeout.increment_handshake_failure_class(classify_handshake_failure_class(&e));
record_handshake_failure_class(
&beobachten_for_timeout,
&config_for_timeout,
@@ -767,6 +814,7 @@ where
}
Err(_) => {
stats_for_timeout.increment_handshake_timeouts();
stats_for_timeout.increment_handshake_failure_class("timeout");
debug!(peer = %peer, "Handshake timeout");
record_beobachten_class(
&beobachten_for_timeout,
@@ -943,6 +991,21 @@ impl RunningClientHandler {
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
if self.proxy_protocol_enabled {
if !is_trusted_proxy_source(
self.peer.ip(),
&self.config.server.proxy_protocol_trusted_cidrs,
) {
self.stats
.increment_connects_bad_with_class("proxy_protocol_untrusted");
warn!(
peer = %self.peer,
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(&self.beobachten, &self.config, self.peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
}
let proxy_header_timeout =
Duration::from_millis(self.config.server.proxy_protocol_header_timeout_ms.max(1));
match timeout(
@@ -952,24 +1015,6 @@ impl RunningClientHandler {
.await
{
Ok(Ok(info)) => {
if !is_trusted_proxy_source(
self.peer.ip(),
&self.config.server.proxy_protocol_trusted_cidrs,
) {
self.stats.increment_connects_bad();
warn!(
peer = %self.peer,
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(
&self.beobachten,
&self.config,
self.peer.ip(),
"other",
);
return Err(ProxyError::InvalidProxyProtocol);
}
debug!(
peer = %self.peer,
client = %info.src_addr,
@@ -986,7 +1031,8 @@ impl RunningClientHandler {
}
}
Ok(Err(e)) => {
self.stats.increment_connects_bad();
self.stats
.increment_connects_bad_with_class("proxy_protocol_invalid_header");
warn!(peer = %self.peer, error = %e, "Invalid PROXY protocol header");
record_beobachten_class(
&self.beobachten,
@@ -997,7 +1043,8 @@ impl RunningClientHandler {
return Err(e);
}
Err(_) => {
self.stats.increment_connects_bad();
self.stats
.increment_connects_bad_with_class("proxy_protocol_header_timeout");
warn!(
peer = %self.peer,
timeout_ms = proxy_header_timeout.as_millis(),
@@ -1095,6 +1142,7 @@ impl RunningClientHandler {
Ok(Ok(outcome)) => outcome,
Ok(Err(e)) => {
debug!(peer = %peer_for_log, error = %e, "Handshake failed");
stats.increment_handshake_failure_class(classify_handshake_failure_class(&e));
record_handshake_failure_class(
&beobachten_for_timeout,
&config_for_timeout,
@@ -1105,6 +1153,7 @@ impl RunningClientHandler {
}
Err(_) => {
stats.increment_handshake_timeouts();
stats.increment_handshake_failure_class("timeout");
debug!(peer = %peer_for_log, "Handshake timeout");
record_beobachten_class(
&beobachten_for_timeout,
@@ -1140,7 +1189,8 @@ impl RunningClientHandler {
// third-party clients or future Telegram versions.
if !tls_clienthello_len_in_bounds(tls_len) {
debug!(peer = %peer, tls_len = tls_len, max_tls_len = MAX_TLS_PLAINTEXT_SIZE, "TLS handshake length out of bounds");
self.stats.increment_connects_bad();
self.stats
.increment_connects_bad_with_class("tls_clienthello_len_out_of_bounds");
maybe_apply_mask_reject_delay(&self.config).await;
let (reader, writer) = self.stream.into_split();
return Ok(masking_outcome(
@@ -1160,7 +1210,8 @@ impl RunningClientHandler {
Ok(n) => n,
Err(e) => {
debug!(peer = %peer, error = %e, tls_len = tls_len, "TLS ClientHello body read failed; engaging masking fallback");
self.stats.increment_connects_bad();
self.stats
.increment_connects_bad_with_class("tls_clienthello_read_error");
maybe_apply_mask_reject_delay(&self.config).await;
let (reader, writer) = self.stream.into_split();
return Ok(masking_outcome(
@@ -1177,7 +1228,8 @@ impl RunningClientHandler {
if body_read < tls_len {
debug!(peer = %peer, got = body_read, expected = tls_len, "Truncated in-range TLS ClientHello; engaging masking fallback");
self.stats.increment_connects_bad();
self.stats
.increment_connects_bad_with_class("tls_clienthello_truncated");
maybe_apply_mask_reject_delay(&self.config).await;
let initial_len = 5 + body_read;
let (reader, writer) = self.stream.into_split();
@@ -1214,7 +1266,7 @@ impl RunningClientHandler {
{
HandshakeResult::Success(result) => result,
HandshakeResult::BadClient { reader, writer } => {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
return Ok(masking_outcome(
reader,
writer,
@@ -1264,7 +1316,7 @@ impl RunningClientHandler {
};
let reader =
tokio::io::AsyncReadExt::chain(std::io::Cursor::new(pending_record), reader);
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_mtproto_bad_client");
debug!(
peer = %peer,
"Authenticated TLS session failed MTProto validation; engaging masking fallback"
@@ -1311,7 +1363,8 @@ impl RunningClientHandler {
if !self.config.general.modes.classic && !self.config.general.modes.secure {
debug!(peer = %peer, "Non-TLS modes disabled");
self.stats.increment_connects_bad();
self.stats
.increment_connects_bad_with_class("direct_modes_disabled");
maybe_apply_mask_reject_delay(&self.config).await;
let (reader, writer) = self.stream.into_split();
return Ok(masking_outcome(
@@ -1351,7 +1404,7 @@ impl RunningClientHandler {
{
HandshakeResult::Success(result) => result,
HandshakeResult::BadClient { reader, writer } => {
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("direct_mtproto_bad_client");
return Ok(masking_outcome(
reader,
writer,
@@ -1387,8 +1440,8 @@ impl RunningClientHandler {
/// Main dispatch after successful handshake.
/// Two modes:
/// - Direct: TCP relay to TG DC (existing behavior)
/// - Middle Proxy: RPC multiplex through ME pool (new — supports CDN DCs)
/// - Direct: TCP relay to TG DC (existing behavior)
/// - Middle Proxy: RPC multiplex through ME pool (supports CDN DCs)
#[cfg(test)]
async fn handle_authenticated_static<R, W>(
client_reader: CryptoReader<R>,
@@ -1589,6 +1642,7 @@ impl RunningClientHandler {
ip_tracker,
user.to_string(),
peer_addr.ip(),
true,
))
}
@@ -1634,7 +1688,6 @@ impl RunningClientHandler {
match ip_tracker.check_and_add(user, peer_addr.ip()).await {
Ok(()) => {
ip_tracker.remove_ip(user, peer_addr.ip()).await;
stats.decrement_user_curr_connects(user);
}
Err(reason) => {
stats.decrement_user_curr_connects(user);
@@ -1650,6 +1703,7 @@ impl RunningClientHandler {
}
}
stats.decrement_user_curr_connects(user);
Ok(())
}
}

View File

@@ -18,8 +18,7 @@ use crate::error::{ProxyError, Result};
use crate::protocol::constants::*;
use crate::proxy::handshake::{HandshakeSuccess, encrypt_tg_nonce_with_ciphers, generate_tg_nonce};
use crate::proxy::route_mode::{
ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state,
cutover_stagger_delay,
RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay,
};
use crate::proxy::shared_state::{
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
@@ -360,7 +359,7 @@ where
"Cutover affected direct session, closing client connection"
);
tokio::time::sleep(delay).await;
break Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
break Err(ProxyError::RouteSwitched);
}
tokio::select! {
result = &mut relay_result => {

View File

@@ -55,6 +55,7 @@ const STICKY_HINT_MAX_ENTRIES: usize = 65_536;
const CANDIDATE_HINT_TRACK_CAP: usize = 64;
const OVERLOAD_CANDIDATE_BUDGET_HINTED: usize = 16;
const OVERLOAD_CANDIDATE_BUDGET_UNHINTED: usize = 8;
const EXPENSIVE_INVALID_SCAN_SATURATION_THRESHOLD: usize = 64;
const RECENT_USER_RING_SCAN_LIMIT: usize = 32;
type HmacSha256 = Hmac<Sha256>;
@@ -551,6 +552,19 @@ fn auth_probe_note_saturation_in(shared: &ProxySharedState, now: Instant) {
}
}
fn auth_probe_note_expensive_invalid_scan_in(
shared: &ProxySharedState,
now: Instant,
validation_checks: usize,
overload: bool,
) {
if overload || validation_checks < EXPENSIVE_INVALID_SCAN_SATURATION_THRESHOLD {
return;
}
auth_probe_note_saturation_in(shared, now);
}
fn auth_probe_record_failure_in(shared: &ProxySharedState, peer_ip: IpAddr, now: Instant) {
let peer_ip = normalize_auth_probe_ip(peer_ip);
let state = &shared.handshake.auth_probe;
@@ -1119,6 +1133,10 @@ where
} else {
None
};
// Fail-closed to TLS 1.3 semantics when ClientHello version is ambiguous:
// this avoids leaking certificate payload on malformed probes.
let client_tls_version = tls::detect_client_hello_tls_version(handshake)
.unwrap_or(tls::ClientHelloTlsVersion::Tls13);
if client_sni.is_some() && matched_tls_domain.is_none() && preferred_user_hint.is_none() {
let sni = client_sni.as_deref().unwrap_or_default();
@@ -1132,9 +1150,20 @@ where
"TLS handshake accepted by unknown SNI policy"
);
}
action @ (UnknownSniAction::Drop | UnknownSniAction::Mask) => {
action @ (UnknownSniAction::Drop
| UnknownSniAction::Mask
| UnknownSniAction::RejectHandshake) => {
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
// For Drop/Mask we apply the synthetic ServerHello delay so
// the fail-closed path is timing-indistinguishable from the
// success path. For RejectHandshake we deliberately skip the
// delay: a stock modern nginx with `ssl_reject_handshake on;`
// responds with the alert essentially immediately, so
// injecting 8-24ms here would itself become a distinguisher
// against the public baseline we are trying to blend into.
if !matches!(action, UnknownSniAction::RejectHandshake) {
maybe_apply_server_hello_delay(config).await;
}
let log_now = Instant::now();
if should_emit_unknown_sni_warn_in(shared, log_now) {
warn!(
@@ -1153,8 +1182,33 @@ where
"TLS handshake rejected by unknown SNI policy"
);
}
if matches!(action, UnknownSniAction::RejectHandshake) {
// TLS alert record layer:
// 0x15 ContentType.alert
// 0x03 0x03 legacy_record_version = TLS 1.2
// (matches what modern nginx emits in
// the first server -> client record,
// per RFC 8446 5.1 guidance)
// 0x00 0x02 length = 2
// Alert payload:
// 0x02 AlertLevel.fatal
// 0x70 AlertDescription.unrecognized_name (112, RFC 6066)
const TLS_ALERT_UNRECOGNIZED_NAME: [u8; 7] =
[0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x70];
if let Err(e) = writer.write_all(&TLS_ALERT_UNRECOGNIZED_NAME).await {
debug!(
peer = %peer,
error = %e,
"Failed to write unrecognized_name TLS alert"
);
} else {
let _ = writer.flush().await;
}
}
return match action {
UnknownSniAction::Drop => HandshakeResult::Error(ProxyError::UnknownTlsSni),
UnknownSniAction::Drop | UnknownSniAction::RejectHandshake => {
HandshakeResult::Error(ProxyError::UnknownTlsSni)
}
UnknownSniAction::Mask => HandshakeResult::BadClient { reader, writer },
UnknownSniAction::Accept => unreachable!(),
};
@@ -1338,7 +1392,14 @@ where
}
if !matched {
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
let failure_now = Instant::now();
auth_probe_note_expensive_invalid_scan_in(
shared,
failure_now,
validation_checks,
overload,
);
auth_probe_record_failure_in(shared, peer.ip(), failure_now);
maybe_apply_server_hello_delay(config).await;
debug!(
peer = %peer,
@@ -1389,6 +1450,20 @@ where
validated_secret.copy_from_slice(secret);
}
if config
.access
.is_user_source_ip_denied(validated_user.as_str(), peer.ip())
{
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
warn!(
peer = %peer,
user = %validated_user,
"TLS handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
);
return HandshakeResult::BadClient { reader, writer };
}
// Reject known replay digests before expensive cache/domain/ALPN policy work.
let digest_half = &validation_digest[..tls::TLS_DIGEST_HALF_LEN];
if replay_checker.check_tls_digest(digest_half) {
@@ -1403,12 +1478,18 @@ where
let selected_domain =
matched_tls_domain.unwrap_or(config.censorship.tls_domain.as_str());
let cached_entry = cache.get(selected_domain).await;
let use_full_cert_payload = cache
.take_full_cert_budget_for_ip(
peer.ip(),
Duration::from_secs(config.censorship.tls_full_cert_ttl_secs),
)
.await;
let use_full_cert_payload = if config.censorship.serverhello_compact
&& matches!(client_tls_version, tls::ClientHelloTlsVersion::Tls12)
{
cache
.take_full_cert_budget_for_ip(
peer.ip(),
Duration::from_secs(config.censorship.tls_full_cert_ttl_secs),
)
.await
} else {
true
};
Some((cached_entry, use_full_cert_payload))
} else {
None
@@ -1429,6 +1510,8 @@ where
validation_session_id_slice,
&cached_entry,
use_full_cert_payload,
config.censorship.serverhello_compact,
client_tls_version,
rng,
selected_alpn.clone(),
config.censorship.tls_new_session_tickets,
@@ -1705,7 +1788,14 @@ where
}
if !matched {
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
let failure_now = Instant::now();
auth_probe_note_expensive_invalid_scan_in(
shared,
failure_now,
validation_checks,
overload,
);
auth_probe_record_failure_in(shared, peer.ip(), failure_now);
maybe_apply_server_hello_delay(config).await;
debug!(
peer = %peer,
@@ -1719,6 +1809,20 @@ where
let validation = matched_validation.expect("validation must exist when matched");
if config
.access
.is_user_source_ip_denied(matched_user.as_str(), peer.ip())
{
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
warn!(
peer = %peer,
user = %matched_user,
"MTProto handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
);
return HandshakeResult::BadClient { reader, writer };
}
// Apply replay tracking only after successful authentication.
//
// This ordering prevents an attacker from producing invalid handshakes that
@@ -1797,6 +1901,20 @@ where
.auth_expensive_checks_total
.fetch_add(validation_checks as u64, Ordering::Relaxed);
if config
.access
.is_user_source_ip_denied(user.as_str(), peer.ip())
{
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
warn!(
peer = %peer,
user = %user,
"MTProto handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
);
return HandshakeResult::BadClient { reader, writer };
}
// Apply replay tracking only after successful authentication.
//
// This ordering prevents an attacker from producing invalid handshakes that

View File

@@ -2,6 +2,7 @@
use crate::config::ProxyConfig;
use crate::network::dns_overrides::resolve_socket_addr;
use crate::protocol::tls;
use crate::stats::beobachten::BeobachtenStore;
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
#[cfg(unix)]
@@ -60,21 +61,18 @@ where
let mut buf = Box::new([0u8; MASK_BUFFER_SIZE]);
let mut total = 0usize;
let mut ended_by_eof = false;
if byte_cap == 0 {
return CopyOutcome {
total,
ended_by_eof,
};
}
let unlimited = byte_cap == 0;
loop {
let remaining_budget = byte_cap.saturating_sub(total);
if remaining_budget == 0 {
break;
}
let read_len = remaining_budget.min(MASK_BUFFER_SIZE);
let read_len = if unlimited {
MASK_BUFFER_SIZE
} else {
let remaining_budget = byte_cap.saturating_sub(total);
if remaining_budget == 0 {
break;
}
remaining_budget.min(MASK_BUFFER_SIZE)
};
let read_res = timeout(idle_timeout, reader.read(&mut buf[..read_len])).await;
let n = match read_res {
Ok(Ok(n)) => n,
@@ -331,6 +329,89 @@ async fn wait_mask_outcome_budget(started: Instant, config: &ProxyConfig) {
}
}
#[cfg(test)]
mod tls_domain_mask_host_tests {
use super::{mask_host_for_initial_data, matching_tls_domain_for_sni};
use crate::config::ProxyConfig;
fn client_hello_with_sni(sni_host: &str) -> Vec<u8> {
let mut body = Vec::new();
body.extend_from_slice(&[0x03, 0x03]);
body.extend_from_slice(&[0u8; 32]);
body.push(32);
body.extend_from_slice(&[0x42u8; 32]);
body.extend_from_slice(&2u16.to_be_bytes());
body.extend_from_slice(&[0x13, 0x01]);
body.push(1);
body.push(0);
let host_bytes = sni_host.as_bytes();
let mut sni_payload = Vec::new();
sni_payload.extend_from_slice(&((host_bytes.len() + 3) as u16).to_be_bytes());
sni_payload.push(0);
sni_payload.extend_from_slice(&(host_bytes.len() as u16).to_be_bytes());
sni_payload.extend_from_slice(host_bytes);
let mut extensions = Vec::new();
extensions.extend_from_slice(&0x0000u16.to_be_bytes());
extensions.extend_from_slice(&(sni_payload.len() as u16).to_be_bytes());
extensions.extend_from_slice(&sni_payload);
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
body.extend_from_slice(&extensions);
let mut handshake = Vec::new();
handshake.push(0x01);
let body_len = (body.len() as u32).to_be_bytes();
handshake.extend_from_slice(&body_len[1..4]);
handshake.extend_from_slice(&body);
let mut record = Vec::new();
record.push(0x16);
record.extend_from_slice(&[0x03, 0x01]);
record.extend_from_slice(&(handshake.len() as u16).to_be_bytes());
record.extend_from_slice(&handshake);
record
}
fn config_with_tls_domains() -> ProxyConfig {
let mut config = ProxyConfig::default();
config.censorship.tls_domain = "a.com".to_string();
config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()];
config.censorship.mask_host = Some("a.com".to_string());
config
}
#[test]
fn matching_tls_domain_accepts_primary_and_extra_domains_case_insensitively() {
let config = config_with_tls_domains();
assert_eq!(matching_tls_domain_for_sni(&config, "A.COM"), Some("a.com"));
assert_eq!(matching_tls_domain_for_sni(&config, "B.COM"), Some("b.com"));
assert_eq!(matching_tls_domain_for_sni(&config, "unknown.com"), None);
}
#[test]
fn mask_host_preserves_explicit_non_primary_origin() {
let mut config = config_with_tls_domains();
config.censorship.mask_host = Some("origin.example".to_string());
let initial_data = client_hello_with_sni("b.com");
assert_eq!(
mask_host_for_initial_data(&config, &initial_data),
"origin.example"
);
}
#[test]
fn mask_host_uses_matching_tls_domain_when_mask_host_is_primary_default() {
let config = config_with_tls_domains();
let initial_data = client_hello_with_sni("b.com");
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com");
}
}
/// Detect client type based on initial data
fn detect_client_type(data: &[u8]) -> &'static str {
// Check for HTTP request
@@ -363,6 +444,37 @@ fn parse_mask_host_ip_literal(host: &str) -> Option<IpAddr> {
host.parse::<IpAddr>().ok()
}
fn matching_tls_domain_for_sni<'a>(config: &'a ProxyConfig, sni: &str) -> Option<&'a str> {
if config.censorship.tls_domain.eq_ignore_ascii_case(sni) {
return Some(config.censorship.tls_domain.as_str());
}
for domain in &config.censorship.tls_domains {
if domain.eq_ignore_ascii_case(sni) {
return Some(domain.as_str());
}
}
None
}
fn mask_host_for_initial_data<'a>(config: &'a ProxyConfig, initial_data: &[u8]) -> &'a str {
let configured_mask_host = config
.censorship
.mask_host
.as_deref()
.unwrap_or(&config.censorship.tls_domain);
if !configured_mask_host.eq_ignore_ascii_case(&config.censorship.tls_domain) {
return configured_mask_host;
}
tls::extract_sni_from_client_hello(initial_data)
.as_deref()
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
.unwrap_or(configured_mask_host)
}
fn canonical_ip(ip: IpAddr) -> IpAddr {
match ip {
IpAddr::V6(v6) => v6
@@ -737,11 +849,7 @@ pub async fn handle_bad_client<R, W>(
return;
}
let mask_host = config
.censorship
.mask_host
.as_deref()
.unwrap_or(&config.censorship.tls_domain);
let mask_host = mask_host_for_initial_data(config, initial_data);
let mask_port = config.censorship.mask_port;
// Fail closed when fallback points at our own listener endpoint.
@@ -930,21 +1038,21 @@ async fn consume_client_data<R: AsyncRead + Unpin>(
byte_cap: usize,
idle_timeout: Duration,
) {
if byte_cap == 0 {
return;
}
// Keep drain path fail-closed under slow-loris stalls.
let mut buf = Box::new([0u8; MASK_BUFFER_SIZE]);
let mut total = 0usize;
let unlimited = byte_cap == 0;
loop {
let remaining_budget = byte_cap.saturating_sub(total);
if remaining_budget == 0 {
break;
}
let read_len = remaining_budget.min(MASK_BUFFER_SIZE);
let read_len = if unlimited {
MASK_BUFFER_SIZE
} else {
let remaining_budget = byte_cap.saturating_sub(total);
if remaining_budget == 0 {
break;
}
remaining_budget.min(MASK_BUFFER_SIZE)
};
let n = match timeout(idle_timeout, reader.read(&mut buf[..read_len])).await {
Ok(Ok(n)) => n,
Ok(Err(_)) | Err(_) => break,
@@ -955,7 +1063,7 @@ async fn consume_client_data<R: AsyncRead + Unpin>(
}
total = total.saturating_add(n);
if total >= byte_cap {
if !unlimited && total >= byte_cap {
break;
}
}

View File

@@ -12,8 +12,9 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::sync::{mpsc, oneshot, watch};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch};
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, trace, warn};
use crate::config::{ConntrackPressureProfile, ProxyConfig};
@@ -22,8 +23,7 @@ use crate::error::{ProxyError, Result};
use crate::protocol::constants::{secure_padding_len, *};
use crate::proxy::handshake::HandshakeSuccess;
use crate::proxy::route_mode::{
ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state,
cutover_stagger_delay,
RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay,
};
use crate::proxy::shared_state::{
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
@@ -36,7 +36,11 @@ use crate::stream::{BufferPool, CryptoReader, CryptoWriter, PooledBuffer};
use crate::transport::middle_proxy::{MePool, MeResponse, proto_flags_for_tag};
enum C2MeCommand {
Data { payload: PooledBuffer, flags: u32 },
Data {
payload: PooledBuffer,
flags: u32,
_permit: OwnedSemaphorePermit,
},
Close,
}
@@ -47,6 +51,8 @@ const DESYNC_ERROR_CLASS: &str = "frame_too_large_crypto_desync";
const C2ME_CHANNEL_CAPACITY_FALLBACK: usize = 128;
const C2ME_SOFT_PRESSURE_MIN_FREE_SLOTS: usize = 64;
const C2ME_SENDER_FAIRNESS_BUDGET: usize = 32;
const C2ME_QUEUED_BYTE_PERMIT_UNIT: usize = 16 * 1024;
const C2ME_QUEUED_PERMITS_PER_SLOT: usize = 4;
const RELAY_IDLE_IO_POLL_MAX: Duration = Duration::from_secs(1);
const TINY_FRAME_DEBT_PER_TINY: u32 = 8;
const TINY_FRAME_DEBT_LIMIT: u32 = 512;
@@ -59,6 +65,15 @@ const ME_D2C_SINGLE_WRITE_COALESCE_MAX_BYTES: usize = 128 * 1024;
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
const QUOTA_RESERVE_BACKOFF_MIN_MS: u64 = 1;
const QUOTA_RESERVE_BACKOFF_MAX_MS: u64 = 16;
const QUOTA_RESERVE_MAX_BACKOFF_ROUNDS: usize = 16;
const ME_CHILD_JOIN_TIMEOUT: Duration = Duration::from_secs(2);
enum MiddleQuotaReserveError {
LimitExceeded,
Contended,
Cancelled,
DeadlineExceeded,
}
#[derive(Default)]
pub(crate) struct DesyncDedupRotationState {
@@ -571,6 +586,43 @@ fn should_yield_c2me_sender(sent_since_yield: usize, has_backlog: bool) -> bool
has_backlog && sent_since_yield >= C2ME_SENDER_FAIRNESS_BUDGET
}
fn c2me_payload_permits(payload_len: usize) -> u32 {
payload_len
.max(1)
.div_ceil(C2ME_QUEUED_BYTE_PERMIT_UNIT)
.min(u32::MAX as usize) as u32
}
fn c2me_queued_permit_budget(channel_capacity: usize, frame_limit: usize) -> usize {
channel_capacity
.saturating_mul(C2ME_QUEUED_PERMITS_PER_SLOT)
.max(c2me_payload_permits(frame_limit) as usize)
.max(1)
}
async fn acquire_c2me_payload_permit(
semaphore: &Arc<Semaphore>,
payload_len: usize,
send_timeout: Option<Duration>,
stats: &Stats,
) -> Result<OwnedSemaphorePermit> {
let permits = c2me_payload_permits(payload_len);
let acquire = semaphore.clone().acquire_many_owned(permits);
match send_timeout {
Some(send_timeout) => match timeout(send_timeout, acquire).await {
Ok(Ok(permit)) => Ok(permit),
Ok(Err(_)) => Err(ProxyError::Proxy("ME sender byte budget closed".into())),
Err(_) => {
stats.increment_me_c2me_send_timeout_total();
Err(ProxyError::Proxy("ME sender byte budget timeout".into()))
}
},
None => acquire
.await
.map_err(|_| ProxyError::Proxy("ME sender byte budget closed".into())),
}
}
fn quota_soft_cap(limit: u64, overshoot: u64) -> u64 {
limit.saturating_add(overshoot)
}
@@ -579,21 +631,43 @@ async fn reserve_user_quota_with_yield(
user_stats: &UserStats,
bytes: u64,
limit: u64,
) -> std::result::Result<u64, QuotaReserveError> {
stats: &Stats,
cancel: &CancellationToken,
deadline: Option<Instant>,
) -> std::result::Result<u64, MiddleQuotaReserveError> {
let mut backoff_ms = QUOTA_RESERVE_BACKOFF_MIN_MS;
let mut backoff_rounds = 0usize;
loop {
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match user_stats.quota_try_reserve(bytes, limit) {
Ok(total) => return Ok(total),
Err(QuotaReserveError::LimitExceeded) => {
return Err(QuotaReserveError::LimitExceeded);
return Err(MiddleQuotaReserveError::LimitExceeded);
}
Err(QuotaReserveError::Contended) => {
stats.increment_quota_contention_total();
std::hint::spin_loop();
}
Err(QuotaReserveError::Contended) => std::hint::spin_loop(),
}
}
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
stats.increment_quota_contention_timeout_total();
return Err(MiddleQuotaReserveError::DeadlineExceeded);
}
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {}
_ = cancel.cancelled() => {
stats.increment_quota_acquire_cancelled_total();
return Err(MiddleQuotaReserveError::Cancelled);
}
}
backoff_rounds = backoff_rounds.saturating_add(1);
if backoff_rounds >= QUOTA_RESERVE_MAX_BACKOFF_ROUNDS {
stats.increment_quota_contention_timeout_total();
return Err(MiddleQuotaReserveError::Contended);
}
backoff_ms = backoff_ms
.saturating_mul(2)
.min(QUOTA_RESERVE_BACKOFF_MAX_MS);
@@ -604,12 +678,13 @@ async fn wait_for_traffic_budget(
lease: Option<&Arc<TrafficLease>>,
direction: RateDirection,
bytes: u64,
) {
deadline: Option<Instant>,
) -> Result<()> {
if bytes == 0 {
return;
return Ok(());
}
let Some(lease) = lease else {
return;
return Ok(());
};
let mut remaining = bytes;
@@ -621,6 +696,9 @@ async fn wait_for_traffic_budget(
}
let wait_started_at = Instant::now();
if deadline.is_some_and(|deadline| wait_started_at >= deadline) {
return Err(ProxyError::TrafficBudgetWaitDeadlineExceeded);
}
tokio::time::sleep(next_refill_delay()).await;
let wait_ms = wait_started_at
.elapsed()
@@ -633,6 +711,59 @@ async fn wait_for_traffic_budget(
wait_ms,
);
}
Ok(())
}
async fn wait_for_traffic_budget_or_cancel(
lease: Option<&Arc<TrafficLease>>,
direction: RateDirection,
bytes: u64,
cancel: &CancellationToken,
stats: &Stats,
deadline: Option<Instant>,
) -> Result<()> {
if bytes == 0 {
return Ok(());
}
let Some(lease) = lease else {
return Ok(());
};
let mut remaining = bytes;
while remaining > 0 {
let consume = lease.try_consume(direction, remaining);
if consume.granted > 0 {
remaining = remaining.saturating_sub(consume.granted);
continue;
}
let wait_started_at = Instant::now();
if deadline.is_some_and(|deadline| wait_started_at >= deadline) {
stats.increment_flow_wait_middle_rate_limit_cancelled_total();
return Err(ProxyError::TrafficBudgetWaitDeadlineExceeded);
}
tokio::select! {
_ = tokio::time::sleep(next_refill_delay()) => {}
_ = cancel.cancelled() => {
stats.increment_flow_wait_middle_rate_limit_cancelled_total();
return Err(ProxyError::TrafficBudgetWaitCancelled);
}
}
let wait_ms = wait_started_at
.elapsed()
.as_millis()
.min(u128::from(u64::MAX)) as u64;
lease.observe_wait_ms(
direction,
consume.blocked_user,
consume.blocked_cidr,
wait_ms,
);
stats.observe_flow_wait_middle_rate_limit_ms(wait_ms);
}
Ok(())
}
fn classify_me_d2c_flush_reason(
@@ -1071,7 +1202,7 @@ where
tokio::time::sleep(delay).await;
let _ = me_pool.send_close(conn_id).await;
me_pool.registry().unregister(conn_id).await;
return Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
return Err(ProxyError::RouteSwitched);
}
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
@@ -1122,13 +1253,19 @@ where
0 => None,
timeout_ms => Some(Duration::from_millis(timeout_ms)),
};
let c2me_byte_budget = c2me_queued_permit_budget(c2me_channel_capacity, frame_limit);
let c2me_byte_semaphore = Arc::new(Semaphore::new(c2me_byte_budget));
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
let me_pool_c2me = me_pool.clone();
let c2me_sender = tokio::spawn(async move {
let mut c2me_sender = tokio::spawn(async move {
let mut sent_since_yield = 0usize;
while let Some(cmd) = c2me_rx.recv().await {
match cmd {
C2MeCommand::Data { payload, flags } => {
C2MeCommand::Data {
payload,
flags,
_permit,
} => {
me_pool_c2me
.send_proxy_req(
conn_id,
@@ -1156,16 +1293,18 @@ where
});
let (stop_tx, mut stop_rx) = oneshot::channel::<()>();
let flow_cancel = CancellationToken::new();
let mut me_rx_task = me_rx;
let stats_clone = stats.clone();
let rng_clone = rng.clone();
let user_clone = user.clone();
let quota_user_stats_me_writer = quota_user_stats.clone();
let traffic_lease_me_writer = traffic_lease.clone();
let flow_cancel_me_writer = flow_cancel.clone();
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
let bytes_me2c_clone = bytes_me2c.clone();
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
let me_writer = tokio::spawn(async move {
let mut me_writer = tokio::spawn(async move {
let mut writer = crypto_writer;
let mut frame_buf = Vec::with_capacity(16 * 1024);
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
@@ -1185,7 +1324,7 @@ where
let Some(first) = msg else {
debug!(conn_id, "ME channel closed");
shrink_session_vec(&mut frame_buf, shrink_threshold);
return Err(ProxyError::Proxy("ME connection lost".into()));
return Err(ProxyError::MiddleConnectionLost);
};
let mut batch_frames = 0usize;
@@ -1207,6 +1346,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1227,7 +1367,7 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1268,6 +1408,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1289,7 +1430,8 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ =
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1332,6 +1474,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1356,7 +1499,11 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ = flush_client_or_cancel(
&mut writer,
&flow_cancel_me_writer,
)
.await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1398,6 +1545,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1422,7 +1570,11 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ = flush_client_or_cancel(
&mut writer,
&flow_cancel_me_writer,
)
.await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1446,7 +1598,7 @@ where
Ok(None) => {
debug!(conn_id, "ME channel closed");
shrink_session_vec(&mut frame_buf, shrink_threshold);
return Err(ProxyError::Proxy("ME connection lost".into()));
return Err(ProxyError::MiddleConnectionLost);
}
Err(_) => {
max_delay_fired = true;
@@ -1468,7 +1620,7 @@ where
} else {
None
};
writer.flush().await.map_err(ProxyError::Io)?;
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1561,7 +1713,7 @@ where
stats.as_ref(),
)
.await;
main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
main_result = Err(ProxyError::RouteSwitched);
break;
}
@@ -1592,26 +1744,50 @@ where
traffic_lease.as_ref(),
RateDirection::Up,
payload.len() as u64,
None,
)
.await;
.await?;
forensics.bytes_c2me = forensics
.bytes_c2me
.saturating_add(payload.len() as u64);
if let (Some(limit), Some(user_stats)) =
(quota_limit, quota_user_stats.as_deref())
{
if reserve_user_quota_with_yield(
match reserve_user_quota_with_yield(
user_stats,
payload.len() as u64,
limit,
stats.as_ref(),
&flow_cancel,
None,
)
.await
.is_err()
{
main_result = Err(ProxyError::DataQuotaExceeded {
user: user.clone(),
});
break;
Ok(_) => {}
Err(MiddleQuotaReserveError::LimitExceeded) => {
main_result = Err(ProxyError::DataQuotaExceeded {
user: user.clone(),
});
break;
}
Err(MiddleQuotaReserveError::Contended) => {
main_result = Err(ProxyError::Proxy(
"ME C->ME quota reservation contended".into(),
));
break;
}
Err(MiddleQuotaReserveError::Cancelled) => {
main_result = Err(ProxyError::Proxy(
"ME C->ME quota reservation cancelled".into(),
));
break;
}
Err(MiddleQuotaReserveError::DeadlineExceeded) => {
main_result = Err(ProxyError::Proxy(
"ME C->ME quota reservation deadline exceeded".into(),
));
break;
}
}
stats.add_user_octets_from_handle(user_stats, payload.len() as u64);
} else {
@@ -1624,11 +1800,29 @@ where
if payload.len() >= 8 && payload[..8].iter().all(|b| *b == 0) {
flags |= RPC_FLAG_NOT_ENCRYPTED;
}
let payload_permit = match acquire_c2me_payload_permit(
&c2me_byte_semaphore,
payload.len(),
c2me_send_timeout,
stats.as_ref(),
)
.await
{
Ok(permit) => permit,
Err(e) => {
main_result = Err(e);
break;
}
};
// Keep client read loop lightweight: route heavy ME send path via a dedicated task.
if enqueue_c2me_command_in(
shared.as_ref(),
&c2me_tx,
C2MeCommand::Data { payload, flags },
C2MeCommand::Data {
payload,
flags,
_permit: payload_permit,
},
c2me_send_timeout,
stats.as_ref(),
)
@@ -1662,22 +1856,34 @@ where
}
drop(c2me_tx);
let c2me_result = c2me_sender
.await
.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME sender join error: {e}"))));
let c2me_result = match timeout(ME_CHILD_JOIN_TIMEOUT, &mut c2me_sender).await {
Ok(joined) => {
joined.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME sender join error: {e}"))))
}
Err(_) => {
stats.increment_me_child_join_timeout_total();
stats.increment_me_child_abort_total();
c2me_sender.abort();
Err(ProxyError::Proxy("ME sender join timeout".into()))
}
};
flow_cancel.cancel();
let _ = stop_tx.send(());
let mut writer_result = me_writer
.await
.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME writer join error: {e}"))));
let mut writer_result = match timeout(ME_CHILD_JOIN_TIMEOUT, &mut me_writer).await {
Ok(joined) => {
joined.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME writer join error: {e}"))))
}
Err(_) => {
stats.increment_me_child_join_timeout_total();
stats.increment_me_child_abort_total();
me_writer.abort();
Err(ProxyError::Proxy("ME writer join timeout".into()))
}
};
// When client closes, but ME channel stopped as unregistered - it isnt error
if client_closed
&& matches!(
writer_result,
Err(ProxyError::Proxy(ref msg)) if msg == "ME connection lost"
)
{
if client_closed && matches!(writer_result, Err(ProxyError::MiddleConnectionLost)) {
writer_result = Ok(());
}
@@ -2201,6 +2407,7 @@ enum MeWriterResponseOutcome {
Close,
}
#[cfg(test)]
async fn process_me_writer_response<W>(
response: MeResponse,
client_writer: &mut CryptoWriter<W>,
@@ -2232,6 +2439,7 @@ where
quota_limit,
quota_soft_overshoot_bytes,
None,
&CancellationToken::new(),
bytes_me2c,
conn_id,
ack_flush_immediate,
@@ -2252,6 +2460,7 @@ async fn process_me_writer_response_with_traffic_lease<W>(
quota_limit: Option<u64>,
quota_soft_overshoot_bytes: u64,
traffic_lease: Option<&Arc<TrafficLease>>,
cancel: &CancellationToken,
bytes_me2c: &AtomicU64,
conn_id: u64,
ack_flush_immediate: bool,
@@ -2261,7 +2470,7 @@ where
W: AsyncWrite + Unpin + Send + 'static,
{
match response {
MeResponse::Data { flags, data } => {
MeResponse::Data { flags, data, .. } => {
if batched {
trace!(conn_id, bytes = data.len(), flags, "ME->C data (batched)");
} else {
@@ -2270,31 +2479,65 @@ where
let data_len = data.len() as u64;
if let (Some(limit), Some(user_stats)) = (quota_limit, quota_user_stats) {
let soft_limit = quota_soft_cap(limit, quota_soft_overshoot_bytes);
if reserve_user_quota_with_yield(user_stats, data_len, soft_limit)
.await
.is_err()
match reserve_user_quota_with_yield(
user_stats, data_len, soft_limit, stats, cancel, None,
)
.await
{
stats.increment_me_d2c_quota_reject_total(MeD2cQuotaRejectStage::PreWrite);
return Err(ProxyError::DataQuotaExceeded {
user: user.to_string(),
});
Ok(_) => {}
Err(MiddleQuotaReserveError::LimitExceeded) => {
stats.increment_me_d2c_quota_reject_total(MeD2cQuotaRejectStage::PreWrite);
return Err(ProxyError::DataQuotaExceeded {
user: user.to_string(),
});
}
Err(MiddleQuotaReserveError::Contended) => {
return Err(ProxyError::Proxy(
"ME D->C quota reservation contended".into(),
));
}
Err(MiddleQuotaReserveError::Cancelled) => {
return Err(ProxyError::Proxy(
"ME D->C quota reservation cancelled".into(),
));
}
Err(MiddleQuotaReserveError::DeadlineExceeded) => {
return Err(ProxyError::Proxy(
"ME D->C quota reservation deadline exceeded".into(),
));
}
}
}
wait_for_traffic_budget(traffic_lease, RateDirection::Down, data_len).await;
wait_for_traffic_budget_or_cancel(
traffic_lease,
RateDirection::Down,
data_len,
cancel,
stats,
None,
)
.await?;
let write_mode =
match write_client_payload(client_writer, proto_tag, flags, &data, rng, frame_buf)
.await
{
Ok(mode) => mode,
Err(err) => {
if quota_limit.is_some() {
stats.add_quota_write_fail_bytes_total(data_len);
stats.increment_quota_write_fail_events_total();
}
return Err(err);
let write_mode = match write_client_payload(
client_writer,
proto_tag,
flags,
&data,
rng,
frame_buf,
cancel,
)
.await
{
Ok(mode) => mode,
Err(err) => {
if quota_limit.is_some() {
stats.add_quota_write_fail_bytes_total(data_len);
stats.increment_quota_write_fail_events_total();
}
};
return Err(err);
}
};
bytes_me2c.fetch_add(data_len, Ordering::Relaxed);
if let Some(user_stats) = quota_user_stats {
@@ -2318,8 +2561,16 @@ where
} else {
trace!(conn_id, confirm, "ME->C quickack");
}
wait_for_traffic_budget(traffic_lease, RateDirection::Down, 4).await;
write_client_ack(client_writer, proto_tag, confirm).await?;
wait_for_traffic_budget_or_cancel(
traffic_lease,
RateDirection::Down,
4,
cancel,
stats,
None,
)
.await?;
write_client_ack(client_writer, proto_tag, confirm, cancel).await?;
stats.increment_me_d2c_ack_frames_total();
Ok(MeWriterResponseOutcome::Continue {
@@ -2371,6 +2622,7 @@ async fn write_client_payload<W>(
data: &[u8],
rng: &SecureRandom,
frame_buf: &mut Vec<u8>,
cancel: &CancellationToken,
) -> Result<MeD2cWriteMode>
where
W: AsyncWrite + Unpin + Send + 'static,
@@ -2398,21 +2650,12 @@ where
frame_buf.reserve(wire_len);
frame_buf.push(first);
frame_buf.extend_from_slice(data);
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
MeD2cWriteMode::Coalesced
} else {
let header = [first];
client_writer
.write_all(&header)
.await
.map_err(ProxyError::Io)?;
client_writer
.write_all(data)
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, &header, cancel).await?;
write_all_client_or_cancel(client_writer, data, cancel).await?;
MeD2cWriteMode::Split
}
} else if len_words < (1 << 24) {
@@ -2427,21 +2670,12 @@ where
frame_buf.reserve(wire_len);
frame_buf.extend_from_slice(&[first, lw[0], lw[1], lw[2]]);
frame_buf.extend_from_slice(data);
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
MeD2cWriteMode::Coalesced
} else {
let header = [first, lw[0], lw[1], lw[2]];
client_writer
.write_all(&header)
.await
.map_err(ProxyError::Io)?;
client_writer
.write_all(data)
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, &header, cancel).await?;
write_all_client_or_cancel(client_writer, data, cancel).await?;
MeD2cWriteMode::Split
}
} else {
@@ -2476,21 +2710,12 @@ where
frame_buf.resize(start + padding_len, 0);
rng.fill(&mut frame_buf[start..]);
}
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
MeD2cWriteMode::Coalesced
} else {
let header = len_val.to_le_bytes();
client_writer
.write_all(&header)
.await
.map_err(ProxyError::Io)?;
client_writer
.write_all(data)
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, &header, cancel).await?;
write_all_client_or_cancel(client_writer, data, cancel).await?;
if padding_len > 0 {
frame_buf.clear();
if frame_buf.capacity() < padding_len {
@@ -2498,10 +2723,7 @@ where
}
frame_buf.resize(padding_len, 0);
rng.fill(frame_buf.as_mut_slice());
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
}
MeD2cWriteMode::Split
}
@@ -2515,6 +2737,7 @@ async fn write_client_ack<W>(
client_writer: &mut CryptoWriter<W>,
proto_tag: ProtoTag,
confirm: u32,
cancel: &CancellationToken,
) -> Result<()>
where
W: AsyncWrite + Unpin + Send + 'static,
@@ -2524,10 +2747,34 @@ where
} else {
confirm.to_le_bytes()
};
client_writer
.write_all(&bytes)
.await
.map_err(ProxyError::Io)
write_all_client_or_cancel(client_writer, &bytes, cancel).await
}
async fn write_all_client_or_cancel<W>(
client_writer: &mut CryptoWriter<W>,
bytes: &[u8],
cancel: &CancellationToken,
) -> Result<()>
where
W: AsyncWrite + Unpin + Send + 'static,
{
tokio::select! {
result = client_writer.write_all(bytes) => result.map_err(ProxyError::Io),
_ = cancel.cancelled() => Err(ProxyError::MiddleClientWriterCancelled),
}
}
async fn flush_client_or_cancel<W>(
client_writer: &mut CryptoWriter<W>,
cancel: &CancellationToken,
) -> Result<()>
where
W: AsyncWrite + Unpin + Send + 'static,
{
tokio::select! {
result = client_writer.flush() => result.map_err(ProxyError::Io),
_ = cancel.cancelled() => Err(ProxyError::MiddleClientWriterCancelled),
}
}
#[cfg(test)]

View File

@@ -215,6 +215,7 @@ struct StatsIo<S> {
c2s_rate_debt_bytes: u64,
c2s_wait: RateWaitState,
s2c_wait: RateWaitState,
quota_wait: RateWaitState,
quota_limit: Option<u64>,
quota_exceeded: Arc<AtomicBool>,
quota_bytes_since_check: u64,
@@ -230,6 +231,7 @@ struct RateWaitState {
}
impl<S> StatsIo<S> {
#[cfg(test)]
fn new(
inner: S,
counters: Arc<SharedCounters>,
@@ -274,6 +276,7 @@ impl<S> StatsIo<S> {
c2s_rate_debt_bytes: 0,
c2s_wait: RateWaitState::default(),
s2c_wait: RateWaitState::default(),
quota_wait: RateWaitState::default(),
quota_limit,
quota_exceeded,
quota_bytes_since_check: 0,
@@ -352,6 +355,11 @@ impl<S> StatsIo<S> {
Poll::Ready(())
}
fn arm_quota_wait(&mut self, cx: &mut Context<'_>) -> Poll<()> {
Self::arm_wait(&mut self.quota_wait, false, false);
Self::poll_wait(&mut self.quota_wait, cx, None, RateDirection::Up)
}
}
#[derive(Debug)]
@@ -429,8 +437,13 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
if this.settle_c2s_rate_debt(cx).is_pending() {
return Poll::Pending;
}
if buf.remaining() == 0 {
return Pin::new(&mut this.inner).poll_read(cx, buf);
}
let mut remaining_before = None;
let mut reserved_read_bytes = 0u64;
let mut read_limit = buf.remaining();
if let Some(limit) = this.quota_limit {
let used_before = this.user_stats.quota_used();
let remaining = limit.saturating_sub(used_before);
@@ -439,50 +452,79 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
return Poll::Ready(Err(quota_io_error()));
}
remaining_before = Some(remaining);
read_limit = read_limit.min(remaining as usize);
if read_limit == 0 {
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
let desired = read_limit as u64;
let mut reserve_rounds = 0usize;
while reserved_read_bytes == 0 {
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match this.user_stats.quota_try_reserve(desired, limit) {
Ok(_) => {
reserved_read_bytes = desired;
break;
}
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
Err(crate::stats::QuotaReserveError::Contended) => {
this.stats.increment_quota_contention_total();
}
}
}
if reserved_read_bytes == 0 {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.stats.increment_quota_contention_timeout_total();
if this.arm_quota_wait(cx).is_pending() {
return Poll::Pending;
}
reserve_rounds = 0;
}
}
}
}
let before = buf.filled().len();
let limited_read = read_limit < buf.remaining();
let read_result = if limited_read {
let mut limited_buf = ReadBuf::new(buf.initialize_unfilled_to(read_limit));
match Pin::new(&mut this.inner).poll_read(cx, &mut limited_buf) {
Poll::Ready(Ok(())) => {
let n = limited_buf.filled().len();
buf.advance(n);
Poll::Ready(Ok(n))
}
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Pending => Poll::Pending,
}
} else {
let before = buf.filled().len();
match Pin::new(&mut this.inner).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
let n = buf.filled().len() - before;
Poll::Ready(Ok(n))
}
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Pending => Poll::Pending,
}
};
match Pin::new(&mut this.inner).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
let n = buf.filled().len() - before;
match read_result {
Poll::Ready(Ok(n)) => {
if reserved_read_bytes > n as u64 {
let refund_bytes = reserved_read_bytes - n as u64;
refund_reserved_quota_bytes(this.user_stats.as_ref(), refund_bytes);
this.stats.add_quota_refund_bytes_total(refund_bytes);
}
if n > 0 {
let n_to_charge = n as u64;
if let (Some(limit), Some(remaining)) = (this.quota_limit, remaining_before) {
let mut reserved_total = None;
let mut reserve_rounds = 0usize;
while reserved_total.is_none() {
let mut saw_contention = false;
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match this.user_stats.quota_try_reserve(n_to_charge, limit) {
Ok(total) => {
reserved_total = Some(total);
break;
}
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
this.quota_exceeded.store(true, Ordering::Release);
buf.set_filled(before);
return Poll::Ready(Err(quota_io_error()));
}
Err(crate::stats::QuotaReserveError::Contended) => {
saw_contention = true;
}
}
}
if reserved_total.is_none() {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.quota_exceeded.store(true, Ordering::Release);
buf.set_filled(before);
return Poll::Ready(Err(quota_io_error()));
}
if saw_contention {
std::thread::yield_now();
}
}
}
if let Some(remaining) = remaining_before {
if should_immediate_quota_check(remaining, n_to_charge) {
this.quota_bytes_since_check = 0;
} else {
@@ -493,10 +535,11 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
this.quota_bytes_since_check = 0;
}
}
if reserved_total.unwrap_or(0) >= limit {
this.quota_exceeded.store(true, Ordering::Release);
}
}
if let Some(limit) = this.quota_limit
&& this.user_stats.quota_used() >= limit
{
this.quota_exceeded.store(true, Ordering::Release);
}
// C→S: client sent data
@@ -520,7 +563,20 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
}
Poll::Ready(Ok(()))
}
other => other,
Poll::Pending => {
if reserved_read_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_read_bytes);
this.stats.add_quota_refund_bytes_total(reserved_read_bytes);
}
Poll::Pending
}
Poll::Ready(Err(err)) => {
if reserved_read_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_read_bytes);
this.stats.add_quota_refund_bytes_total(reserved_read_bytes);
}
Poll::Ready(Err(err))
}
}
}
}
@@ -602,6 +658,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
break;
}
Err(crate::stats::QuotaReserveError::Contended) => {
this.stats.increment_quota_contention_total();
saw_contention = true;
}
}
@@ -610,14 +667,14 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
if reserved_bytes == 0 {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.stats.increment_quota_contention_timeout_total();
if let Some(lease) = this.traffic_lease.as_ref() {
lease.refund(RateDirection::Down, shaper_reserved_bytes);
}
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
if saw_contention {
std::thread::yield_now();
let _ = this.arm_quota_wait(cx);
return Poll::Pending;
} else if saw_contention {
std::hint::spin_loop();
}
}
}
@@ -638,10 +695,9 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
match Pin::new(&mut this.inner).poll_write(cx, write_buf) {
Poll::Ready(Ok(n)) => {
if reserved_bytes > n as u64 {
refund_reserved_quota_bytes(
this.user_stats.as_ref(),
reserved_bytes - n as u64,
);
let refund_bytes = reserved_bytes - n as u64;
refund_reserved_quota_bytes(this.user_stats.as_ref(), refund_bytes);
this.stats.add_quota_refund_bytes_total(refund_bytes);
}
if shaper_reserved_bytes > n as u64
&& let Some(lease) = this.traffic_lease.as_ref()
@@ -692,6 +748,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
Poll::Ready(Err(err)) => {
if reserved_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
this.stats.add_quota_refund_bytes_total(reserved_bytes);
}
if shaper_reserved_bytes > 0
&& let Some(lease) = this.traffic_lease.as_ref()
@@ -703,6 +760,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
Poll::Pending => {
if reserved_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
this.stats.add_quota_refund_bytes_total(reserved_bytes);
}
if shaper_reserved_bytes > 0
&& let Some(lease) = this.traffic_lease.as_ref()

View File

@@ -4,8 +4,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::watch;
pub(crate) const ROUTE_SWITCH_ERROR_MSG: &str = "Session terminated";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub(crate) enum RelayRouteMode {

View File

@@ -282,7 +282,7 @@ async fn user_connection_reservation_drop_enqueues_cleanup_synchronously() {
assert_eq!(stats.get_user_curr_connects(&user), 1);
let reservation =
UserConnectionReservation::new(stats.clone(), ip_tracker.clone(), user.clone(), ip);
UserConnectionReservation::new(stats.clone(), ip_tracker.clone(), user.clone(), ip, true);
// Drop the reservation synchronously without any tokio::spawn/await yielding!
drop(reservation);
@@ -320,6 +320,7 @@ async fn relay_task_abort_releases_user_gate_and_ip_reservation() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 8).await;
let mut cfg = ProxyConfig::default();
cfg.access.user_max_tcp_conns.insert(user.to_string(), 8);
@@ -437,6 +438,7 @@ async fn relay_cutover_releases_user_gate_and_ip_reservation() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 8).await;
let mut cfg = ProxyConfig::default();
cfg.access.user_max_tcp_conns.insert(user.to_string(), 8);
@@ -659,7 +661,7 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s
assert!(
matches!(relay_result, Err(ProxyError::DataQuotaExceeded { .. }))
|| matches!(relay_result, Err(ProxyError::Proxy(ref msg)) if msg == crate::proxy::route_mode::ROUTE_SWITCH_ERROR_MSG),
|| matches!(relay_result, Err(ProxyError::RouteSwitched)),
"overlap race must fail closed via quota enforcement or generic cutover termination"
);
@@ -958,6 +960,36 @@ async fn reservation_limit_failure_does_not_leak_curr_connects_counter() {
assert_eq!(ip_tracker.get_active_ip_count(user).await, 0);
}
#[tokio::test]
async fn unlimited_unique_ip_user_is_still_visible_in_active_ip_tracker() {
let user = "active-ip-observed-user";
let config = crate::config::ProxyConfig::default();
let stats = Arc::new(crate::stats::Stats::new());
let ip_tracker = Arc::new(crate::ip_tracker::UserIpTracker::new());
let peer = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 51, 200, 17)), 50017);
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
user,
&config,
stats.clone(),
peer,
ip_tracker.clone(),
)
.await
.expect("reservation without unique-IP limit must succeed");
assert_eq!(stats.get_user_curr_connects(user), 1);
assert_eq!(
ip_tracker.get_active_ip_count(user).await,
1,
"active IP observability must not depend on unique-IP limit enforcement"
);
reservation.release().await;
assert_eq!(stats.get_user_curr_connects(user), 0);
assert_eq!(ip_tracker.get_active_ip_count(user).await, 0);
}
#[tokio::test]
async fn short_tls_probe_is_masked_through_client_pipeline() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2493,6 +2525,46 @@ fn unexpected_eof_is_classified_without_string_matching() {
);
}
#[test]
fn connection_reset_is_classified_as_expected_handshake_close() {
let beobachten = BeobachtenStore::new();
let mut config = ProxyConfig::default();
config.general.beobachten = true;
config.general.beobachten_minutes = 1;
let reset = ProxyError::Io(std::io::Error::from(std::io::ErrorKind::ConnectionReset));
let peer_ip: IpAddr = "198.51.100.202".parse().unwrap();
record_handshake_failure_class(&beobachten, &config, peer_ip, &reset);
let snapshot = beobachten.snapshot_text(Duration::from_secs(60));
assert!(
snapshot.contains("[expected_64_got_0]"),
"ConnectionReset must be classified as expected handshake close"
);
}
#[test]
fn stream_io_unexpected_eof_is_classified_without_string_matching() {
let beobachten = BeobachtenStore::new();
let mut config = ProxyConfig::default();
config.general.beobachten = true;
config.general.beobachten_minutes = 1;
let eof = ProxyError::Stream(StreamError::Io(std::io::Error::from(
std::io::ErrorKind::UnexpectedEof,
)));
let peer_ip: IpAddr = "198.51.100.203".parse().unwrap();
record_handshake_failure_class(&beobachten, &config, peer_ip, &eof);
let snapshot = beobachten.snapshot_text(Duration::from_secs(60));
assert!(
snapshot.contains("[expected_64_got_0]"),
"StreamError::Io(UnexpectedEof) must be classified as expected handshake close"
);
}
#[test]
fn non_eof_error_is_classified_as_other() {
let beobachten = BeobachtenStore::new();
@@ -2839,6 +2911,7 @@ async fn explicit_reservation_release_cleans_user_and_ip_immediately() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 4).await;
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
user,
@@ -2877,6 +2950,7 @@ async fn explicit_reservation_release_does_not_double_decrement_on_drop() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 4).await;
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
user,
@@ -2907,6 +2981,7 @@ async fn drop_fallback_eventually_cleans_user_and_ip_reservation() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 1).await;
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
user,
@@ -2989,6 +3064,7 @@ async fn release_abort_storm_does_not_leak_user_or_ip_reservations() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, ATTEMPTS + 16).await;
for idx in 0..ATTEMPTS {
let peer = SocketAddr::new(
@@ -3039,6 +3115,7 @@ async fn release_abort_loop_preserves_immediate_same_ip_reacquire() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 1).await;
for _ in 0..ITERATIONS {
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
@@ -3097,6 +3174,7 @@ async fn adversarial_mixed_release_drop_abort_wave_converges_to_zero() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, RESERVATIONS + 8).await;
let mut reservations = Vec::with_capacity(RESERVATIONS);
for idx in 0..RESERVATIONS {
@@ -3177,6 +3255,8 @@ async fn parallel_users_abort_release_isolation_preserves_independent_cleanup()
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user_a, 64).await;
ip_tracker.set_user_limit(user_b, 64).await;
let mut tasks = tokio::task::JoinSet::new();
for idx in 0..64usize {
@@ -3238,6 +3318,7 @@ async fn concurrent_release_storm_leaves_zero_user_and_ip_footprint() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, RESERVATIONS + 8).await;
let mut reservations = Vec::with_capacity(RESERVATIONS);
for idx in 0..RESERVATIONS {
@@ -3292,6 +3373,7 @@ async fn relay_connect_error_releases_user_and_ip_before_return() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 8).await;
let mut config = ProxyConfig::default();
config.access.user_max_tcp_conns.insert(user.to_string(), 1);
@@ -3387,6 +3469,7 @@ async fn mixed_release_and_drop_same_ip_preserves_counter_correctness() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 1).await;
let reservation_a = RunningClientHandler::acquire_user_connection_reservation_static(
user,
@@ -3447,6 +3530,7 @@ async fn drop_one_of_two_same_ip_reservations_keeps_ip_active() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 1).await;
let reservation_a = RunningClientHandler::acquire_user_connection_reservation_static(
user,
@@ -3656,6 +3740,7 @@ async fn cross_thread_drop_uses_captured_runtime_for_ip_cleanup() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 8).await;
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
user,
@@ -3700,6 +3785,7 @@ async fn immediate_reacquire_after_cross_thread_drop_succeeds() {
let stats = Arc::new(Stats::new());
let ip_tracker = Arc::new(UserIpTracker::new());
ip_tracker.set_user_limit(user, 1).await;
let reservation = RunningClientHandler::acquire_user_connection_reservation_static(
user,

View File

@@ -637,6 +637,22 @@ fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
"telemt-unknown-dc-parent-swap-{}",
std::process::id()
));
if let Ok(meta) = fs::symlink_metadata(&parent) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&parent).expect("stale parent-swap path must be removable");
} else {
fs::remove_dir_all(&parent).expect("stale parent-swap directory must be removable");
}
}
let moved = parent.with_extension("bak");
if let Ok(meta) = fs::symlink_metadata(&moved) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&moved).expect("stale parent-swap backup path must be removable");
} else {
fs::remove_dir_all(&moved)
.expect("stale parent-swap backup directory must be removable");
}
}
fs::create_dir_all(&parent).expect("parent-swap test parent must be creatable");
let rel_candidate = format!(
@@ -646,8 +662,6 @@ fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
.expect("candidate must sanitize before parent swap");
let moved = parent.with_extension("bak");
let _ = fs::remove_dir_all(&moved);
fs::rename(&parent, &moved).expect("parent must be movable for swap simulation");
symlink("/tmp", &parent).expect("symlink replacement for parent must be creatable");
@@ -669,6 +683,13 @@ fn adversarial_check_then_symlink_flip_is_blocked_by_nofollow_open() {
"telemt-unknown-dc-check-open-race-{}",
std::process::id()
));
if let Ok(meta) = fs::symlink_metadata(&parent) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&parent).expect("stale check-open-race path must be removable");
} else {
fs::remove_dir_all(&parent).expect("stale check-open-race parent must be removable");
}
}
fs::create_dir_all(&parent).expect("check-open-race parent must be creatable");
let target = parent.join("unknown-dc.log");
@@ -713,6 +734,24 @@ fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
"telemt-unknown-dc-parent-swap-openat-{}",
std::process::id()
));
if let Ok(meta) = fs::symlink_metadata(&base) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&base).expect("stale parent-swap-openat path must be removable");
} else {
fs::remove_dir_all(&base)
.expect("stale parent-swap-openat directory must be removable");
}
}
let moved = base.with_extension("bak");
if let Ok(meta) = fs::symlink_metadata(&moved) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&moved)
.expect("stale parent-swap-openat backup path must be removable");
} else {
fs::remove_dir_all(&moved)
.expect("stale parent-swap-openat backup directory must be removable");
}
}
fs::create_dir_all(&base).expect("parent-swap-openat base must be creatable");
let rel_candidate = format!(
@@ -736,8 +775,6 @@ fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
let outside_target = outside_parent.join("unknown-dc.log");
let _ = fs::remove_file(&outside_target);
let moved = base.with_extension("bak");
let _ = fs::remove_dir_all(&moved);
fs::rename(&base, &moved).expect("base parent must be movable for swap simulation");
symlink(&outside_parent, &base).expect("base parent symlink replacement must be creatable");
@@ -1482,10 +1519,7 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() {
"cutover should terminate direct relay session"
);
assert!(
matches!(
relay_result,
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
),
matches!(relay_result, Err(ProxyError::RouteSwitched)),
"client-visible cutover error must stay generic and avoid route-internal metadata"
);
@@ -1622,10 +1656,7 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea
.expect("direct relay task must not panic");
assert!(
matches!(
relay_result,
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
),
matches!(relay_result, Err(ProxyError::RouteSwitched)),
"storm-cutover termination must remain generic for all direct sessions"
);
}
@@ -1928,10 +1959,7 @@ async fn adversarial_direct_relay_cutover_integrity() {
.expect("Session must not panic");
assert!(
matches!(
result,
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
),
matches!(result, Err(ProxyError::RouteSwitched)),
"Session must terminate with route switch error on cutover"
);
}

View File

@@ -1007,6 +1007,55 @@ async fn tls_unknown_sni_mask_policy_falls_back_to_bad_client() {
assert!(matches!(result, HandshakeResult::BadClient { .. }));
}
#[tokio::test]
async fn tls_unknown_sni_reject_handshake_policy_emits_unrecognized_name_alert() {
use tokio::io::{AsyncReadExt, duplex};
let secret = [0x4Au8; 16];
let mut config = test_config_with_secret_hex("4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a4a");
config.censorship.unknown_sni_action = UnknownSniAction::RejectHandshake;
let replay_checker = ReplayChecker::new(128, Duration::from_secs(60));
let rng = SecureRandom::new();
let peer: SocketAddr = "198.51.100.192:44326".parse().unwrap();
let handshake =
make_valid_tls_client_hello_with_sni_and_alpn(&secret, 0, "unknown.example", &[b"h2"]);
// Wire up a duplex so we can inspect what the server writes towards the
// client. We own the "peer side" half to read from it.
let (server_side, mut peer_side) = duplex(1024);
let (server_read, server_write) = tokio::io::split(server_side);
let result = handle_tls_handshake(
&handshake,
server_read,
server_write,
peer,
&config,
&replay_checker,
&rng,
None,
)
.await;
assert!(matches!(
result,
HandshakeResult::Error(ProxyError::UnknownTlsSni)
));
// Drain what the server wrote. We expect exactly one TLS alert record:
// 0x15 0x03 0x03 0x00 0x02 0x02 0x70
// (ContentType.alert, TLS 1.2, length=2, fatal, unrecognized_name)
drop(result); // drops the server-side writer so peer_side sees EOF
let mut buf = Vec::new();
peer_side.read_to_end(&mut buf).await.unwrap();
assert_eq!(
buf,
[0x15, 0x03, 0x03, 0x00, 0x02, 0x02, 0x70],
"reject_handshake must emit a fatal unrecognized_name TLS alert"
);
}
#[tokio::test]
async fn tls_unknown_sni_accept_policy_continues_auth_path() {
let secret = [0x4Bu8; 16];
@@ -1203,6 +1252,97 @@ async fn tls_overload_budget_limits_candidate_scan_depth() {
);
}
#[tokio::test]
async fn tls_expensive_invalid_scan_activates_saturation_budget() {
let mut config = ProxyConfig::default();
config.access.users.clear();
config.access.ignore_time_skew = true;
for idx in 0..80u8 {
config.access.users.insert(
format!("user-{idx}"),
format!("{:032x}", u128::from(idx) + 1),
);
}
config.rebuild_runtime_user_auth().unwrap();
let replay_checker = ReplayChecker::new(128, Duration::from_secs(60));
let rng = SecureRandom::new();
let shared = ProxySharedState::new();
let attacker_secret = [0xEFu8; 16];
let handshake = make_valid_tls_handshake(&attacker_secret, 0);
let first_peer: SocketAddr = "198.51.100.214:44326".parse().unwrap();
let first = handle_tls_handshake_with_shared(
&handshake,
tokio::io::empty(),
tokio::io::sink(),
first_peer,
&config,
&replay_checker,
&rng,
None,
shared.as_ref(),
)
.await;
assert!(matches!(first, HandshakeResult::BadClient { .. }));
assert!(
auth_probe_saturation_state_for_testing_in_shared(shared.as_ref())
.lock()
.unwrap()
.is_some(),
"expensive invalid scan must activate global saturation"
);
assert_eq!(
shared
.handshake
.auth_expensive_checks_total
.load(Ordering::Relaxed),
80,
"first invalid probe preserves full first-hit compatibility before enabling saturation"
);
{
let mut saturation = auth_probe_saturation_state_for_testing_in_shared(shared.as_ref())
.lock()
.unwrap();
let state = saturation.as_mut().expect("saturation must be present");
state.blocked_until = Instant::now() + Duration::from_millis(200);
}
let second_peer: SocketAddr = "198.51.100.215:44326".parse().unwrap();
let second = handle_tls_handshake_with_shared(
&handshake,
tokio::io::empty(),
tokio::io::sink(),
second_peer,
&config,
&replay_checker,
&rng,
None,
shared.as_ref(),
)
.await;
assert!(matches!(second, HandshakeResult::BadClient { .. }));
assert_eq!(
shared
.handshake
.auth_budget_exhausted_total
.load(Ordering::Relaxed),
1,
"second invalid probe must be capped by overload budget"
);
assert_eq!(
shared
.handshake
.auth_expensive_checks_total
.load(Ordering::Relaxed),
80 + OVERLOAD_CANDIDATE_BUDGET_UNHINTED as u64,
"saturation budget must bound follow-up invalid scans"
);
}
#[tokio::test]
async fn mtproto_runtime_snapshot_prefers_preferred_user_hint() {
let mut config = ProxyConfig::default();

View File

@@ -58,11 +58,22 @@ async fn consume_stall_stress_finishes_within_idle_budget() {
}
#[tokio::test]
async fn consume_zero_cap_returns_immediately() {
async fn consume_zero_cap_is_idle_bounded_on_stall() {
let started = Instant::now();
consume_client_data(tokio::io::empty(), 0, MASK_RELAY_IDLE_TIMEOUT).await;
tokio::time::timeout(
MASK_RELAY_TIMEOUT,
consume_client_data(OneByteThenStall { sent: false }, 0, MASK_RELAY_IDLE_TIMEOUT),
)
.await
.expect("zero-cap consume path must remain bounded by timeout guards");
let elapsed = started.elapsed();
assert!(
started.elapsed() < MASK_RELAY_IDLE_TIMEOUT,
"zero byte cap must return immediately"
elapsed >= (MASK_RELAY_IDLE_TIMEOUT / 2),
"zero cap must not short-circuit before idle timeout path, got {elapsed:?}"
);
assert!(
elapsed < MASK_RELAY_TIMEOUT,
"zero-cap consume path must complete before relay timeout, got {elapsed:?}"
);
}

View File

@@ -148,9 +148,10 @@ async fn positive_copy_with_production_cap_stops_exactly_at_budget() {
}
#[tokio::test]
async fn negative_consume_with_zero_cap_performs_no_reads() {
let read_calls = Arc::new(AtomicUsize::new(0));
let reader = FinitePatternReader::new(1024, 64, Arc::clone(&read_calls));
async fn consume_with_zero_cap_drains_until_eof() {
let payload = 256 * 1024;
let total_read = Arc::new(AtomicUsize::new(0));
let reader = BudgetProbeReader::new(payload, Arc::clone(&total_read));
consume_client_data_with_timeout_and_cap(
reader,
@@ -161,9 +162,27 @@ async fn negative_consume_with_zero_cap_performs_no_reads() {
.await;
assert_eq!(
read_calls.load(Ordering::Relaxed),
0,
"zero cap must return before reading attacker-controlled bytes"
total_read.load(Ordering::Relaxed),
payload,
"zero cap must disable byte budget and drain finite payload to EOF"
);
}
#[tokio::test]
async fn copy_with_zero_cap_drains_until_eof() {
let read_calls = Arc::new(AtomicUsize::new(0));
let payload = 73 * 1024;
let mut reader = FinitePatternReader::new(payload, 3072, read_calls);
let mut writer = CountingWriter::default();
let outcome =
copy_with_idle_timeout(&mut reader, &mut writer, 0, true, MASK_RELAY_IDLE_TIMEOUT).await;
assert_eq!(outcome.total, payload);
assert_eq!(writer.written, payload);
assert!(
outcome.ended_by_eof,
"zero cap must not terminate relay early on byte budget"
);
}

View File

@@ -13,6 +13,8 @@ struct CountedWriter {
fail_writes: bool,
}
struct StalledWriter;
impl CountedWriter {
fn new(write_calls: Arc<AtomicUsize>, fail_writes: bool) -> Self {
Self {
@@ -49,12 +51,36 @@ impl AsyncWrite for CountedWriter {
}
}
impl AsyncWrite for StalledWriter {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
_buf: &[u8],
) -> Poll<io::Result<usize>> {
Poll::Pending
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Pending
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Pending
}
}
fn make_crypto_writer(inner: CountedWriter) -> CryptoWriter<CountedWriter> {
let key = [0u8; 32];
let iv = 0u128;
CryptoWriter::new(inner, AesCtr::new(&key, iv), 8 * 1024)
}
fn make_stalled_crypto_writer() -> CryptoWriter<StalledWriter> {
let key = [0u8; 32];
let iv = 0u128;
CryptoWriter::new(StalledWriter, AesCtr::new(&key, iv), 8 * 1024)
}
#[tokio::test]
async fn me_writer_write_fail_keeps_reserved_quota_and_tracks_fail_metrics() {
let stats = Stats::new();
@@ -70,6 +96,7 @@ async fn me_writer_write_fail_keeps_reserved_quota_and_tracks_fail_metrics() {
MeResponse::Data {
flags: 0,
data: payload.clone(),
route_permit: None,
},
&mut writer,
ProtoTag::Intermediate,
@@ -139,6 +166,7 @@ async fn me_writer_pre_write_quota_reject_happens_before_writer_poll() {
MeResponse::Data {
flags: 0,
data: Bytes::from_static(&[0xAA, 0xBB, 0xCC]),
route_permit: None,
},
&mut writer,
ProtoTag::Intermediate,
@@ -187,3 +215,53 @@ async fn me_writer_pre_write_quota_reject_happens_before_writer_poll() {
);
assert_eq!(bytes_me2c.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn me_writer_data_write_obeys_flow_cancellation() {
let stats = Stats::new();
let user = "middle-me-writer-cancel-user";
let mut writer = make_stalled_crypto_writer();
let mut frame_buf = Vec::new();
let bytes_me2c = AtomicU64::new(0);
let cancel = CancellationToken::new();
cancel.cancel();
let result = process_me_writer_response_with_traffic_lease(
MeResponse::Data {
flags: 0,
data: Bytes::from_static(&[0x31, 0x32, 0x33, 0x34]),
route_permit: None,
},
&mut writer,
ProtoTag::Intermediate,
&SecureRandom::new(),
&mut frame_buf,
&stats,
user,
None,
None,
0,
None,
&cancel,
&bytes_me2c,
13,
true,
false,
)
.await;
assert!(
matches!(result, Err(ProxyError::MiddleClientWriterCancelled)),
"cancelled middle writer must return a bounded cancellation error"
);
assert_eq!(
bytes_me2c.load(Ordering::Relaxed),
0,
"cancelled write must not advance committed ME->C bytes"
);
assert_eq!(
stats.get_user_total_octets(user),
0,
"cancelled write must not advance user output telemetry"
);
}

View File

@@ -12,6 +12,12 @@ fn make_pooled_payload(data: &[u8]) -> PooledBuffer {
payload
}
fn make_c2me_permit() -> tokio::sync::OwnedSemaphorePermit {
Arc::new(tokio::sync::Semaphore::new(1))
.try_acquire_many_owned(1)
.expect("test permit must be available")
}
#[test]
#[ignore = "Tracking for M-04: Verify should_emit_full_desync returns true on first occurrence and false on duplicate within window"]
fn should_emit_full_desync_filters_duplicates() {
@@ -107,6 +113,7 @@ async fn c2me_channel_full_path_yields_then_sends() {
tx.send(C2MeCommand::Data {
payload: make_pooled_payload(&[0xAA]),
flags: 1,
_permit: make_c2me_permit(),
})
.await
.expect("priming queue with one frame must succeed");
@@ -119,6 +126,7 @@ async fn c2me_channel_full_path_yields_then_sends() {
C2MeCommand::Data {
payload: make_pooled_payload(&[0xBB, 0xCC]),
flags: 2,
_permit: make_c2me_permit(),
},
None,
&stats,
@@ -138,7 +146,7 @@ async fn c2me_channel_full_path_yields_then_sends() {
.expect("receiver should observe primed frame")
.expect("first queued command must exist");
match first {
C2MeCommand::Data { payload, flags } => {
C2MeCommand::Data { payload, flags, .. } => {
assert_eq!(payload.as_ref(), &[0xAA]);
assert_eq!(flags, 1);
}
@@ -155,7 +163,7 @@ async fn c2me_channel_full_path_yields_then_sends() {
.expect("receiver should observe backpressure-resumed frame")
.expect("second queued command must exist");
match second {
C2MeCommand::Data { payload, flags } => {
C2MeCommand::Data { payload, flags, .. } => {
assert_eq!(payload.as_ref(), &[0xBB, 0xCC]);
assert_eq!(flags, 2);
}

View File

@@ -4,10 +4,67 @@ use std::io;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use tokio::io::{AsyncWrite, AsyncWriteExt};
use std::task::{Context, Poll, Wake};
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
use tokio::time::Instant;
enum ReadStep {
Data(Vec<u8>),
Pending,
Eof,
Error,
}
struct ScriptedReader {
scripted_reads: Arc<Mutex<VecDeque<ReadStep>>>,
read_calls: Arc<AtomicUsize>,
}
impl ScriptedReader {
fn new(script: Vec<ReadStep>, read_calls: Arc<AtomicUsize>) -> Self {
Self {
scripted_reads: Arc::new(Mutex::new(script.into())),
read_calls,
}
}
}
impl AsyncRead for ScriptedReader {
fn poll_read(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
let this = self.get_mut();
this.read_calls.fetch_add(1, Ordering::Relaxed);
let step = this
.scripted_reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.pop_front()
.unwrap_or(ReadStep::Eof);
match step {
ReadStep::Data(data) => {
let n = data.len().min(buf.remaining());
buf.put_slice(&data[..n]);
Poll::Ready(Ok(()))
}
ReadStep::Pending => Poll::Pending,
ReadStep::Eof => Poll::Ready(Ok(())),
ReadStep::Error => Poll::Ready(Err(io::Error::new(
io::ErrorKind::BrokenPipe,
"forced read failure",
))),
}
}
}
struct NoopWake;
impl Wake for NoopWake {
fn wake(self: Arc<Self>) {}
}
struct ScriptedWriter {
scripted_writes: Arc<Mutex<VecDeque<usize>>>,
write_calls: Arc<AtomicUsize>,
@@ -80,6 +137,127 @@ fn make_stats_io_with_script(
(io, stats, write_calls, quota_exceeded)
}
fn make_stats_io_with_read_script(
user: &str,
quota_limit: u64,
precharged_quota: u64,
script: Vec<ReadStep>,
) -> (
StatsIo<ScriptedReader>,
Arc<Stats>,
Arc<AtomicUsize>,
Arc<AtomicBool>,
) {
let stats = Arc::new(Stats::new());
if precharged_quota > 0 {
let user_stats = stats.get_or_create_user_stats_handle(user);
stats.quota_charge_post_write(user_stats.as_ref(), precharged_quota);
}
let read_calls = Arc::new(AtomicUsize::new(0));
let quota_exceeded = Arc::new(AtomicBool::new(false));
let io = StatsIo::new(
ScriptedReader::new(script, read_calls.clone()),
Arc::new(SharedCounters::new()),
stats.clone(),
user.to_string(),
Some(quota_limit),
quota_exceeded.clone(),
Instant::now(),
);
(io, stats, read_calls, quota_exceeded)
}
fn poll_read_once<R: AsyncRead + Unpin>(
io: &mut StatsIo<R>,
storage: &mut [u8],
) -> Poll<io::Result<usize>> {
let waker = Arc::new(NoopWake).into();
let mut cx = Context::from_waker(&waker);
let mut read_buf = ReadBuf::new(storage);
let before = read_buf.filled().len();
match Pin::new(io).poll_read(&mut cx, &mut read_buf) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(read_buf.filled().len() - before)),
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
Poll::Pending => Poll::Pending,
}
}
#[test]
fn direct_c2s_quota_refunds_unused_on_short_read() {
let user = "direct-c2s-short-read-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Data(vec![0x11; 5])]);
let mut storage = [0u8; 16];
let n = match poll_read_once(&mut io, &mut storage) {
Poll::Ready(Ok(n)) => n,
other => panic!("short read must complete, got {other:?}"),
};
assert_eq!(n, 5);
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 5);
assert_eq!(stats.get_quota_refund_bytes_total(), 11);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[test]
fn direct_c2s_quota_refunds_full_reservation_on_pending() {
let user = "direct-c2s-pending-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Pending]);
let mut storage = [0u8; 16];
assert!(matches!(
poll_read_once(&mut io, &mut storage),
Poll::Pending
));
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 0);
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[test]
fn direct_c2s_quota_refunds_full_reservation_on_eof() {
let user = "direct-c2s-eof-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Eof]);
let mut storage = [0u8; 16];
let n = match poll_read_once(&mut io, &mut storage) {
Poll::Ready(Ok(n)) => n,
other => panic!("EOF read must complete with zero bytes, got {other:?}"),
};
assert_eq!(n, 0);
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 0);
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[test]
fn direct_c2s_quota_refunds_full_reservation_on_error() {
let user = "direct-c2s-error-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Error]);
let mut storage = [0u8; 16];
let error = match poll_read_once(&mut io, &mut storage) {
Poll::Ready(Err(error)) => error,
other => panic!("error read must return error, got {other:?}"),
};
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 0);
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[tokio::test]
async fn direct_partial_write_charges_only_committed_bytes_without_double_charge() {
let user = "direct-partial-charge-user";

114
src/quota_state.rs Normal file
View File

@@ -0,0 +1,114 @@
use std::collections::BTreeMap;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tracing::{info, warn};
use crate::stats::{Stats, UserQuotaSnapshot};
#[derive(Debug, Default, Serialize, Deserialize)]
pub(crate) struct QuotaStateFile {
pub(crate) last_reset_epoch_secs: u64,
pub(crate) users: BTreeMap<String, QuotaUserState>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub(crate) struct QuotaUserState {
pub(crate) used_bytes: u64,
pub(crate) last_reset_epoch_secs: u64,
}
fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
pub(crate) async fn load_quota_state(path: &Path, stats: &Stats) {
let bytes = match tokio::fs::read(path).await {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return,
Err(error) => {
warn!(
error = %error,
path = %path.display(),
"Failed to read quota state file"
);
return;
}
};
let state = match serde_json::from_slice::<QuotaStateFile>(&bytes) {
Ok(state) => state,
Err(error) => {
warn!(
error = %error,
path = %path.display(),
"Failed to parse quota state file"
);
return;
}
};
let loaded_users = state.users.len();
for (user, quota) in state.users {
stats.load_user_quota_state(&user, quota.used_bytes, quota.last_reset_epoch_secs);
}
info!(
path = %path.display(),
loaded_users,
"Loaded per-user quota state"
);
}
pub(crate) async fn save_quota_state(path: &Path, stats: &Stats) -> std::io::Result<()> {
let mut users = BTreeMap::new();
let mut last_reset_epoch_secs = 0;
for (user, quota) in stats.user_quota_snapshot() {
last_reset_epoch_secs = last_reset_epoch_secs.max(quota.last_reset_epoch_secs);
users.insert(user, quota_user_state(quota));
}
let state = QuotaStateFile {
last_reset_epoch_secs,
users,
};
write_state_file(path, &state).await
}
pub(crate) async fn reset_user_quota(
path: &Path,
stats: &Stats,
user: &str,
) -> std::io::Result<UserQuotaSnapshot> {
let snapshot = stats.reset_user_quota(user);
save_quota_state(path, stats).await?;
Ok(snapshot)
}
async fn write_state_file(path: &Path, state: &QuotaStateFile) -> std::io::Result<()> {
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
tokio::fs::create_dir_all(parent).await?;
}
let tmp_path = path.with_extension(format!("tmp.{}", now_epoch_secs()));
let payload = serde_json::to_vec_pretty(state)?;
let mut file = tokio::fs::File::create(&tmp_path).await?;
file.write_all(&payload).await?;
file.write_all(b"\n").await?;
file.sync_all().await?;
drop(file);
tokio::fs::rename(&tmp_path, path).await
}
fn quota_user_state(quota: UserQuotaSnapshot) -> QuotaUserState {
QuotaUserState {
used_bytes: quota.used_bytes,
last_reset_epoch_secs: quota.last_reset_epoch_secs,
}
}

View File

@@ -7,6 +7,7 @@ use std::time::{Duration, Instant};
use parking_lot::Mutex;
const CLEANUP_INTERVAL: Duration = Duration::from_secs(30);
const MAX_BEOBACHTEN_ENTRIES: usize = 65_536;
#[derive(Default)]
struct BeobachtenInner {
@@ -48,12 +49,23 @@ impl BeobachtenStore {
Self::cleanup_if_needed(&mut guard, now, ttl);
let key = (class.to_string(), ip);
let entry = guard.entries.entry(key).or_insert(BeobachtenEntry {
tries: 0,
last_seen: now,
});
entry.tries = entry.tries.saturating_add(1);
entry.last_seen = now;
if let Some(entry) = guard.entries.get_mut(&key) {
entry.tries = entry.tries.saturating_add(1);
entry.last_seen = now;
return;
}
if guard.entries.len() >= MAX_BEOBACHTEN_ENTRIES {
return;
}
guard.entries.insert(
key,
BeobachtenEntry {
tries: 1,
last_seen: now,
},
);
}
pub fn snapshot_text(&self, ttl: Duration) -> String {
@@ -62,16 +74,21 @@ impl BeobachtenStore {
}
let now = Instant::now();
let mut guard = self.inner.lock();
Self::cleanup(&mut guard, now, ttl);
guard.last_cleanup = Some(now);
let entries = {
let mut guard = self.inner.lock();
Self::cleanup(&mut guard, now, ttl);
guard.last_cleanup = Some(now);
guard
.entries
.iter()
.map(|((class, ip), entry)| (class.clone(), *ip, entry.tries))
.collect::<Vec<_>>()
};
let mut grouped = BTreeMap::<String, Vec<(IpAddr, u64)>>::new();
for ((class, ip), entry) in &guard.entries {
grouped
.entry(class.clone())
.or_default()
.push((*ip, entry.tries));
for (class, ip, tries) in entries {
grouped.entry(class).or_default().push((ip, tries));
}
if grouped.is_empty() {

View File

@@ -8,8 +8,8 @@ pub mod telemetry;
use dashmap::DashMap;
use lru::LruCache;
use parking_lot::Mutex;
use std::collections::VecDeque;
use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, VecDeque};
use std::hash::{Hash, Hasher};
use std::num::NonZeroUsize;
use std::sync::Arc;
@@ -88,6 +88,8 @@ impl Drop for RouteConnectionLease {
pub struct Stats {
connects_all: AtomicU64,
connects_bad: AtomicU64,
connects_bad_classes: DashMap<&'static str, AtomicU64>,
handshake_failure_classes: DashMap<&'static str, AtomicU64>,
current_connections_direct: AtomicU64,
current_connections_me: AtomicU64,
handshake_timeouts: AtomicU64,
@@ -272,11 +274,22 @@ pub struct Stats {
me_inline_recovery_total: AtomicU64,
ip_reservation_rollback_tcp_limit_total: AtomicU64,
ip_reservation_rollback_quota_limit_total: AtomicU64,
quota_refund_bytes_total: AtomicU64,
quota_contention_total: AtomicU64,
quota_contention_timeout_total: AtomicU64,
quota_acquire_cancelled_total: AtomicU64,
quota_write_fail_bytes_total: AtomicU64,
quota_write_fail_events_total: AtomicU64,
me_child_join_timeout_total: AtomicU64,
me_child_abort_total: AtomicU64,
flow_wait_middle_rate_limit_total: AtomicU64,
flow_wait_middle_rate_limit_cancelled_total: AtomicU64,
flow_wait_middle_rate_limit_ms_total: AtomicU64,
session_drop_fallback_total: AtomicU64,
telemetry_core_enabled: AtomicBool,
telemetry_user_enabled: AtomicBool,
telemetry_me_level: AtomicU8,
cached_epoch_secs: AtomicU64,
user_stats: DashMap<String, Arc<UserStats>>,
user_stats_last_cleanup_epoch_secs: AtomicU64,
start_time: parking_lot::RwLock<Option<Instant>>,
@@ -295,9 +308,16 @@ pub struct UserStats {
/// This counter is the single source of truth for quota enforcement and
/// intentionally tracks attempted traffic, not guaranteed delivery.
pub quota_used: AtomicU64,
pub quota_last_reset_epoch_secs: AtomicU64,
pub last_seen_epoch_secs: AtomicU64,
}
#[derive(Debug, Clone)]
pub struct UserQuotaSnapshot {
pub used_bytes: u64,
pub last_reset_epoch_secs: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuotaReserveError {
LimitExceeded,
@@ -339,6 +359,7 @@ impl Stats {
pub fn new() -> Self {
let stats = Self::default();
stats.apply_telemetry_policy(TelemetryPolicy::default());
stats.refresh_cached_epoch_secs();
*stats.start_time.write() = Some(Instant::now());
stats
}
@@ -388,33 +409,55 @@ impl Stats {
.as_secs()
}
fn touch_user_stats(stats: &UserStats) {
fn refresh_cached_epoch_secs(&self) -> u64 {
let now_epoch_secs = Self::now_epoch_secs();
self.cached_epoch_secs
.store(now_epoch_secs, Ordering::Relaxed);
now_epoch_secs
}
fn cached_epoch_secs(&self) -> u64 {
let cached = self.cached_epoch_secs.load(Ordering::Relaxed);
if cached != 0 {
return cached;
}
self.refresh_cached_epoch_secs()
}
fn touch_user_stats(&self, stats: &UserStats) {
stats
.last_seen_epoch_secs
.store(Self::now_epoch_secs(), Ordering::Relaxed);
.store(self.cached_epoch_secs(), Ordering::Relaxed);
}
pub(crate) fn get_or_create_user_stats_handle(&self, user: &str) -> Arc<UserStats> {
self.maybe_cleanup_user_stats();
if let Some(existing) = self.user_stats.get(user) {
let handle = Arc::clone(existing.value());
Self::touch_user_stats(handle.as_ref());
self.touch_user_stats(handle.as_ref());
return handle;
}
let entry = self.user_stats.entry(user.to_string()).or_default();
if entry.last_seen_epoch_secs.load(Ordering::Relaxed) == 0 {
Self::touch_user_stats(entry.value().as_ref());
self.touch_user_stats(entry.value().as_ref());
}
Arc::clone(entry.value())
}
pub(crate) async fn run_periodic_user_stats_maintenance(self: Arc<Self>) {
let mut interval = tokio::time::interval(Duration::from_secs(60));
loop {
interval.tick().await;
self.maybe_cleanup_user_stats();
}
}
#[inline]
pub(crate) fn add_user_octets_from_handle(&self, user_stats: &UserStats, bytes: u64) {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats
.octets_from_client
.fetch_add(bytes, Ordering::Relaxed);
@@ -425,7 +468,7 @@ impl Stats {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats
.octets_to_client
.fetch_add(bytes, Ordering::Relaxed);
@@ -436,7 +479,7 @@ impl Stats {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
}
@@ -445,7 +488,7 @@ impl Stats {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
}
@@ -455,7 +498,7 @@ impl Stats {
/// mixing reserve and post-charge on a single I/O event.
#[inline]
pub(crate) fn quota_charge_post_write(&self, user_stats: &UserStats, bytes: u64) -> u64 {
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats
.quota_used
.fetch_add(bytes, Ordering::Relaxed)
@@ -466,7 +509,7 @@ impl Stats {
const USER_STATS_CLEANUP_INTERVAL_SECS: u64 = 60;
const USER_STATS_IDLE_TTL_SECS: u64 = 24 * 60 * 60;
let now_epoch_secs = Self::now_epoch_secs();
let now_epoch_secs = self.refresh_cached_epoch_secs();
let last_cleanup_epoch_secs = self
.user_stats_last_cleanup_epoch_secs
.load(Ordering::Relaxed);
@@ -518,10 +561,32 @@ impl Stats {
self.connects_all.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_connects_bad(&self) {
if self.telemetry_core_enabled() {
self.connects_bad.fetch_add(1, Ordering::Relaxed);
pub fn increment_connects_bad_with_class(&self, class: &'static str) {
if !self.telemetry_core_enabled() {
return;
}
self.connects_bad.fetch_add(1, Ordering::Relaxed);
let entry = self
.connects_bad_classes
.entry(class)
.or_insert_with(|| AtomicU64::new(0));
entry.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_connects_bad(&self) {
self.increment_connects_bad_with_class("other");
}
pub fn increment_handshake_failure_class(&self, class: &'static str) {
if !self.telemetry_core_enabled() {
return;
}
let entry = self
.handshake_failure_classes
.entry(class)
.or_insert_with(|| AtomicU64::new(0));
entry.fetch_add(1, Ordering::Relaxed);
}
pub fn increment_current_connections_direct(&self) {
self.current_connections_direct
@@ -1406,6 +1471,29 @@ impl Stats {
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn add_quota_refund_bytes_total(&self, bytes: u64) {
if self.telemetry_core_enabled() {
self.quota_refund_bytes_total
.fetch_add(bytes, Ordering::Relaxed);
}
}
pub fn increment_quota_contention_total(&self) {
if self.telemetry_core_enabled() {
self.quota_contention_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_quota_contention_timeout_total(&self) {
if self.telemetry_core_enabled() {
self.quota_contention_timeout_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_quota_acquire_cancelled_total(&self) {
if self.telemetry_core_enabled() {
self.quota_acquire_cancelled_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn add_quota_write_fail_bytes_total(&self, bytes: u64) {
if self.telemetry_core_enabled() {
self.quota_write_fail_bytes_total
@@ -1418,6 +1506,37 @@ impl Stats {
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_child_join_timeout_total(&self) {
if self.telemetry_core_enabled() {
self.me_child_join_timeout_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_child_abort_total(&self) {
if self.telemetry_core_enabled() {
self.me_child_abort_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn observe_flow_wait_middle_rate_limit_ms(&self, wait_ms: u64) {
if self.telemetry_core_enabled() {
self.flow_wait_middle_rate_limit_total
.fetch_add(1, Ordering::Relaxed);
self.flow_wait_middle_rate_limit_ms_total
.fetch_add(wait_ms, Ordering::Relaxed);
}
}
pub fn increment_flow_wait_middle_rate_limit_cancelled_total(&self) {
if self.telemetry_core_enabled() {
self.flow_wait_middle_rate_limit_cancelled_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_session_drop_fallback_total(&self) {
if self.telemetry_core_enabled() {
self.session_drop_fallback_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_endpoint_quarantine_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_endpoint_quarantine_total
@@ -1640,6 +1759,37 @@ impl Stats {
pub fn get_connects_bad(&self) -> u64 {
self.connects_bad.load(Ordering::Relaxed)
}
pub fn get_connects_bad_class_counts(&self) -> Vec<(String, u64)> {
let mut out: Vec<(String, u64)> = self
.connects_bad_classes
.iter()
.map(|entry| {
(
entry.key().to_string(),
entry.value().load(Ordering::Relaxed),
)
})
.collect();
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
pub fn get_handshake_failure_class_counts(&self) -> Vec<(String, u64)> {
let mut out: Vec<(String, u64)> = self
.handshake_failure_classes
.iter()
.map(|entry| {
(
entry.key().to_string(),
entry.value().load(Ordering::Relaxed),
)
})
.collect();
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
pub fn get_accept_permit_timeout_total(&self) -> u64 {
self.accept_permit_timeout_total.load(Ordering::Relaxed)
}
@@ -2221,19 +2371,52 @@ impl Stats {
self.ip_reservation_rollback_quota_limit_total
.load(Ordering::Relaxed)
}
pub fn get_quota_refund_bytes_total(&self) -> u64 {
self.quota_refund_bytes_total.load(Ordering::Relaxed)
}
pub fn get_quota_contention_total(&self) -> u64 {
self.quota_contention_total.load(Ordering::Relaxed)
}
pub fn get_quota_contention_timeout_total(&self) -> u64 {
self.quota_contention_timeout_total.load(Ordering::Relaxed)
}
pub fn get_quota_acquire_cancelled_total(&self) -> u64 {
self.quota_acquire_cancelled_total.load(Ordering::Relaxed)
}
pub fn get_quota_write_fail_bytes_total(&self) -> u64 {
self.quota_write_fail_bytes_total.load(Ordering::Relaxed)
}
pub fn get_quota_write_fail_events_total(&self) -> u64 {
self.quota_write_fail_events_total.load(Ordering::Relaxed)
}
pub fn get_me_child_join_timeout_total(&self) -> u64 {
self.me_child_join_timeout_total.load(Ordering::Relaxed)
}
pub fn get_me_child_abort_total(&self) -> u64 {
self.me_child_abort_total.load(Ordering::Relaxed)
}
pub fn get_flow_wait_middle_rate_limit_total(&self) -> u64 {
self.flow_wait_middle_rate_limit_total
.load(Ordering::Relaxed)
}
pub fn get_flow_wait_middle_rate_limit_cancelled_total(&self) -> u64 {
self.flow_wait_middle_rate_limit_cancelled_total
.load(Ordering::Relaxed)
}
pub fn get_flow_wait_middle_rate_limit_ms_total(&self) -> u64 {
self.flow_wait_middle_rate_limit_ms_total
.load(Ordering::Relaxed)
}
pub fn get_session_drop_fallback_total(&self) -> u64 {
self.session_drop_fallback_total.load(Ordering::Relaxed)
}
pub fn increment_user_connects(&self, user: &str) {
if !self.telemetry_user_enabled() {
return;
}
let stats = self.get_or_create_user_stats_handle(user);
Self::touch_user_stats(stats.as_ref());
self.touch_user_stats(stats.as_ref());
stats.connects.fetch_add(1, Ordering::Relaxed);
}
@@ -2242,7 +2425,7 @@ impl Stats {
return;
}
let stats = self.get_or_create_user_stats_handle(user);
Self::touch_user_stats(stats.as_ref());
self.touch_user_stats(stats.as_ref());
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
}
@@ -2252,7 +2435,7 @@ impl Stats {
}
let stats = self.get_or_create_user_stats_handle(user);
Self::touch_user_stats(stats.as_ref());
self.touch_user_stats(stats.as_ref());
let counter = &stats.curr_connects;
let mut current = counter.load(Ordering::Relaxed);
@@ -2275,9 +2458,8 @@ impl Stats {
}
pub fn decrement_user_curr_connects(&self, user: &str) {
self.maybe_cleanup_user_stats();
if let Some(stats) = self.user_stats.get(user) {
Self::touch_user_stats(stats.value().as_ref());
self.touch_user_stats(stats.value().as_ref());
let counter = &stats.curr_connects;
let mut current = counter.load(Ordering::Relaxed);
loop {
@@ -2353,6 +2535,47 @@ impl Stats {
.unwrap_or(0)
}
pub fn load_user_quota_state(&self, user: &str, used_bytes: u64, last_reset_epoch_secs: u64) {
let stats = self.get_or_create_user_stats_handle(user);
stats.quota_used.store(used_bytes, Ordering::Relaxed);
stats
.quota_last_reset_epoch_secs
.store(last_reset_epoch_secs, Ordering::Relaxed);
}
pub fn reset_user_quota(&self, user: &str) -> UserQuotaSnapshot {
let stats = self.get_or_create_user_stats_handle(user);
let last_reset_epoch_secs = Self::now_epoch_secs();
stats.quota_used.store(0, Ordering::Relaxed);
stats
.quota_last_reset_epoch_secs
.store(last_reset_epoch_secs, Ordering::Relaxed);
UserQuotaSnapshot {
used_bytes: 0,
last_reset_epoch_secs,
}
}
pub fn user_quota_snapshot(&self) -> HashMap<String, UserQuotaSnapshot> {
let mut out = HashMap::new();
for entry in self.user_stats.iter() {
let stats = entry.value();
let used_bytes = stats.quota_used.load(Ordering::Relaxed);
let last_reset_epoch_secs = stats.quota_last_reset_epoch_secs.load(Ordering::Relaxed);
if used_bytes == 0 && last_reset_epoch_secs == 0 {
continue;
}
out.insert(
entry.key().clone(),
UserQuotaSnapshot {
used_bytes,
last_reset_epoch_secs,
},
);
}
out
}
pub fn get_handshake_timeouts(&self) -> u64 {
self.handshake_timeouts.load(Ordering::Relaxed)
}
@@ -2422,6 +2645,11 @@ impl Stats {
self.user_stats.iter()
}
/// Current number of retained per-user stats entries.
pub fn user_stats_len(&self) -> usize {
self.user_stats.len()
}
pub fn uptime_secs(&self) -> f64 {
self.start_time
.read()

View File

@@ -277,6 +277,7 @@ impl StreamState for TlsReaderState {
pub struct FakeTlsReader<R> {
upstream: R,
state: TlsReaderState,
body_scratch: Vec<u8>,
}
impl<R> FakeTlsReader<R> {
@@ -284,6 +285,7 @@ impl<R> FakeTlsReader<R> {
Self {
upstream,
state: TlsReaderState::Idle,
body_scratch: Vec::new(),
}
}
@@ -439,7 +441,13 @@ impl<R: AsyncRead + Unpin> AsyncRead for FakeTlsReader<R> {
length,
mut buffer,
} => {
let result = poll_read_body(&mut this.upstream, cx, &mut buffer, length);
let result = poll_read_body(
&mut this.upstream,
cx,
&mut buffer,
length,
&mut this.body_scratch,
);
match result {
BodyPollResult::Pending => {
@@ -558,34 +566,36 @@ fn poll_read_body<R: AsyncRead + Unpin>(
cx: &mut Context<'_>,
buffer: &mut BytesMut,
target_len: usize,
scratch: &mut Vec<u8>,
) -> BodyPollResult {
// NOTE: This implementation uses a temporary Vec to avoid tricky borrow/lifetime
// issues with BytesMut spare capacity and ReadBuf across polls.
// It's safe and correct; optimization is possible if needed.
while buffer.len() < target_len {
let remaining = target_len - buffer.len();
let chunk_len = remaining.min(8192);
let mut temp = vec![0u8; remaining.min(8192)];
let mut read_buf = ReadBuf::new(&mut temp);
match Pin::new(&mut *upstream).poll_read(cx, &mut read_buf) {
Poll::Pending => return BodyPollResult::Pending,
Poll::Ready(Err(e)) => return BodyPollResult::Error(e),
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
return BodyPollResult::Error(Error::new(
ErrorKind::UnexpectedEof,
format!(
"unexpected EOF in TLS body (got {} of {} bytes)",
buffer.len(),
target_len
),
));
}
buffer.extend_from_slice(&temp[..n]);
}
if scratch.len() < chunk_len {
scratch.resize(chunk_len, 0);
}
let n = {
let mut read_buf = ReadBuf::new(&mut scratch[..chunk_len]);
match Pin::new(&mut *upstream).poll_read(cx, &mut read_buf) {
Poll::Pending => return BodyPollResult::Pending,
Poll::Ready(Err(e)) => return BodyPollResult::Error(e),
Poll::Ready(Ok(())) => read_buf.filled().len(),
}
};
if n == 0 {
return BodyPollResult::Error(Error::new(
ErrorKind::UnexpectedEof,
format!(
"unexpected EOF in TLS body (got {} of {} bytes)",
buffer.len(),
target_len
),
));
}
buffer.extend_from_slice(&scratch[..n]);
}
BodyPollResult::Complete(buffer.split().freeze())

View File

@@ -559,9 +559,7 @@ async fn mass_reconnect_sync_cleanup_prevents_temporary_reservation_bloat() {
}
#[tokio::test]
async fn adversarial_drain_cleanup_queue_race_does_not_cause_false_rejections() {
// Regression guard: concurrent cleanup draining must not produce false
// limit denials for a new IP when the previous IP is already queued.
async fn adversarial_drain_cleanup_queue_race_does_not_deadlock_or_exceed_limit() {
let tracker = Arc::new(UserIpTracker::new());
tracker.set_user_limit("racer", 1).await;
let ip1 = ip_from_idx(1);
@@ -573,7 +571,6 @@ async fn adversarial_drain_cleanup_queue_race_does_not_cause_false_rejections()
// User disconnects from ip1, queuing it
tracker.enqueue_cleanup("racer".to_string(), ip1);
let mut saw_false_rejection = false;
for _ in 0..100 {
// Queue cleanup then race explicit drain and check-and-add on the alternative IP.
tracker.enqueue_cleanup("racer".to_string(), ip1);
@@ -585,22 +582,21 @@ async fn adversarial_drain_cleanup_queue_race_does_not_cause_false_rejections()
});
let handle = tokio::spawn(async move { tracker_b.check_and_add("racer", ip2).await });
drain_handle.await.unwrap();
let res = handle.await.unwrap();
if res.is_err() {
saw_false_rejection = true;
break;
}
tokio::time::timeout(Duration::from_secs(1), drain_handle)
.await
.expect("cleanup drain must not deadlock")
.unwrap();
let _ = tokio::time::timeout(Duration::from_secs(1), handle)
.await
.expect("admission must not deadlock")
.unwrap();
// Restore baseline for next iteration.
assert!(tracker.get_active_ip_count("racer").await <= 1);
tracker.drain_cleanup_queue().await;
tracker.remove_ip("racer", ip2).await;
tracker.remove_ip("racer", ip1).await;
tracker.check_and_add("racer", ip1).await.unwrap();
}
assert!(
!saw_false_rejection,
"Concurrent cleanup draining must not cause false-positive IP denials"
);
}
#[tokio::test]
@@ -649,6 +645,25 @@ async fn duplicate_cleanup_entries_do_not_break_future_admission() {
);
}
#[tokio::test]
async fn duplicate_cleanup_entries_are_coalesced_until_drain() {
let tracker = UserIpTracker::new();
let ip = ip_from_idx(7150);
tracker.enqueue_cleanup("coalesced-cleanup".to_string(), ip);
tracker.enqueue_cleanup("coalesced-cleanup".to_string(), ip);
tracker.enqueue_cleanup("coalesced-cleanup".to_string(), ip);
assert_eq!(
tracker.cleanup_queue_len_for_tests(),
1,
"duplicate queued cleanup entries must retain one allocation slot"
);
tracker.drain_cleanup_queue().await;
assert_eq!(tracker.cleanup_queue_len_for_tests(), 0);
}
#[tokio::test]
async fn stress_repeated_queue_poison_recovery_preserves_admission_progress() {
let tracker = UserIpTracker::new();

View File

@@ -1,26 +1,71 @@
use std::collections::HashMap;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tokio::time::sleep;
use tracing::{debug, info, warn};
use crate::tls_front::types::{
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult,
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, TlsProfileSource,
};
const FULL_CERT_SENT_SWEEP_INTERVAL_SECS: u64 = 30;
const FULL_CERT_SENT_MAX_IPS: usize = 65_536;
const FULL_CERT_SENT_SHARDS: usize = 64;
static FULL_CERT_SENT_IPS_GAUGE: AtomicU64 = AtomicU64::new(0);
static FULL_CERT_SENT_CAP_DROPS: AtomicU64 = AtomicU64::new(0);
/// Current number of IPs tracked by the TLS full-cert budget gate.
pub(crate) fn full_cert_sent_ips_for_metrics() -> u64 {
FULL_CERT_SENT_IPS_GAUGE.load(Ordering::Relaxed)
}
/// Number of new IPs denied a full-cert budget slot because the cap was reached.
pub(crate) fn full_cert_sent_cap_drops_for_metrics() -> u64 {
FULL_CERT_SENT_CAP_DROPS.load(Ordering::Relaxed)
}
/// Lightweight in-memory + optional on-disk cache for TLS fronting data.
#[derive(Debug)]
pub struct TlsFrontCache {
memory: RwLock<HashMap<String, Arc<CachedTlsData>>>,
default: Arc<CachedTlsData>,
full_cert_sent: RwLock<HashMap<IpAddr, Instant>>,
full_cert_sent_shards: Vec<RwLock<HashMap<IpAddr, Instant>>>,
full_cert_sent_last_sweep_epoch_secs: AtomicU64,
disk_path: PathBuf,
}
/// Read-only health view for one configured TLS front domain.
#[derive(Debug, Clone)]
pub(crate) struct TlsFrontProfileHealth {
pub(crate) domain: String,
pub(crate) source: &'static str,
pub(crate) age_seconds: u64,
pub(crate) is_default: bool,
pub(crate) has_cert_info: bool,
pub(crate) has_cert_payload: bool,
pub(crate) app_data_records: usize,
pub(crate) ticket_records: usize,
pub(crate) change_cipher_spec_count: u8,
pub(crate) total_app_data_len: usize,
}
fn profile_source_label(source: TlsProfileSource) -> &'static str {
match source {
TlsProfileSource::Default => "default",
TlsProfileSource::Raw => "raw",
TlsProfileSource::Rustls => "rustls",
TlsProfileSource::Merged => "merged",
}
}
#[allow(dead_code)]
impl TlsFrontCache {
pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef<Path>) -> Self {
@@ -52,7 +97,10 @@ impl TlsFrontCache {
Self {
memory: RwLock::new(map),
default,
full_cert_sent: RwLock::new(HashMap::new()),
full_cert_sent_shards: (0..FULL_CERT_SENT_SHARDS)
.map(|_| RwLock::new(HashMap::new()))
.collect(),
full_cert_sent_last_sweep_epoch_secs: AtomicU64::new(0),
disk_path: disk_path.as_ref().to_path_buf(),
}
}
@@ -69,22 +117,129 @@ impl TlsFrontCache {
self.memory.read().await.contains_key(domain)
}
pub(crate) async fn profile_health_snapshot(
&self,
domains: &[String],
max_domains: usize,
) -> (Vec<TlsFrontProfileHealth>, usize) {
let guard = self.memory.read().await;
let now = SystemTime::now();
let mut snapshot = Vec::with_capacity(domains.len().min(max_domains));
let mut suppressed = 0usize;
for domain in domains {
if snapshot.len() >= max_domains {
suppressed = suppressed.saturating_add(1);
continue;
}
let cached = guard
.get(domain)
.cloned()
.unwrap_or_else(|| self.default.clone());
let behavior = &cached.behavior_profile;
let age_seconds = now
.duration_since(cached.fetched_at)
.map(|duration| duration.as_secs())
.unwrap_or(0);
snapshot.push(TlsFrontProfileHealth {
domain: domain.clone(),
source: profile_source_label(behavior.source),
age_seconds,
is_default: cached.domain == "default",
has_cert_info: cached.cert_info.is_some(),
has_cert_payload: cached.cert_payload.is_some(),
app_data_records: cached
.app_data_records_sizes
.len()
.max(behavior.app_data_record_sizes.len()),
ticket_records: behavior.ticket_record_sizes.len(),
change_cipher_spec_count: behavior.change_cipher_spec_count,
total_app_data_len: cached.total_app_data_len,
});
}
(snapshot, suppressed)
}
fn full_cert_sent_shard_index(client_ip: IpAddr) -> usize {
let mut hasher = DefaultHasher::new();
client_ip.hash(&mut hasher);
(hasher.finish() as usize) % FULL_CERT_SENT_SHARDS
}
fn full_cert_sent_shard(&self, client_ip: IpAddr) -> &RwLock<HashMap<IpAddr, Instant>> {
&self.full_cert_sent_shards[Self::full_cert_sent_shard_index(client_ip)]
}
fn decrement_full_cert_sent_entries(amount: usize) {
if amount == 0 {
return;
}
let amount = amount as u64;
let _ =
FULL_CERT_SENT_IPS_GAUGE.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| {
Some(current.saturating_sub(amount))
});
}
fn try_reserve_full_cert_sent_entry() -> bool {
let mut current = FULL_CERT_SENT_IPS_GAUGE.load(Ordering::Relaxed);
loop {
if current >= FULL_CERT_SENT_MAX_IPS as u64 {
return false;
}
match FULL_CERT_SENT_IPS_GAUGE.compare_exchange_weak(
current,
current.saturating_add(1),
Ordering::AcqRel,
Ordering::Relaxed,
) {
Ok(_) => return true,
Err(actual) => current = actual,
}
}
}
async fn sweep_full_cert_sent_shards(&self, now: Instant, ttl: Duration) {
for shard in &self.full_cert_sent_shards {
let mut guard = shard.write().await;
let before = guard.len();
guard.retain(|_, seen_at| now.duration_since(*seen_at) < ttl);
Self::decrement_full_cert_sent_entries(before.saturating_sub(guard.len()));
}
}
/// Returns true when full cert payload should be sent for client_ip
/// according to TTL policy.
pub async fn take_full_cert_budget_for_ip(&self, client_ip: IpAddr, ttl: Duration) -> bool {
if ttl.is_zero() {
self.full_cert_sent
.write()
.await
.insert(client_ip, Instant::now());
return true;
}
let now = Instant::now();
let mut guard = self.full_cert_sent.write().await;
guard.retain(|_, seen_at| now.duration_since(*seen_at) < ttl);
let now_epoch_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let should_sweep = self
.full_cert_sent_last_sweep_epoch_secs
.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |last_sweep| {
if now_epoch_secs.saturating_sub(last_sweep) >= FULL_CERT_SENT_SWEEP_INTERVAL_SECS {
Some(now_epoch_secs)
} else {
None
}
})
.is_ok();
match guard.get_mut(&client_ip) {
if should_sweep {
self.sweep_full_cert_sent_shards(now, ttl).await;
}
let mut guard = self.full_cert_sent_shard(client_ip).write().await;
let allowed = match guard.get_mut(&client_ip) {
Some(seen_at) => {
if now.duration_since(*seen_at) >= ttl {
*seen_at = now;
@@ -94,12 +249,43 @@ impl TlsFrontCache {
}
}
None => {
if !Self::try_reserve_full_cert_sent_entry() {
FULL_CERT_SENT_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return false;
}
guard.insert(client_ip, now);
true
}
};
allowed
}
#[cfg(test)]
async fn insert_full_cert_sent_for_tests(&self, client_ip: IpAddr, seen_at: Instant) {
let mut guard = self.full_cert_sent_shard(client_ip).write().await;
if guard.insert(client_ip, seen_at).is_none() {
FULL_CERT_SENT_IPS_GAUGE.fetch_add(1, Ordering::Relaxed);
}
}
#[cfg(test)]
async fn full_cert_sent_is_empty_for_tests(&self) -> bool {
for shard in &self.full_cert_sent_shards {
if !shard.read().await.is_empty() {
return false;
}
}
true
}
#[cfg(test)]
async fn full_cert_sent_contains_for_tests(&self, client_ip: IpAddr) -> bool {
self.full_cert_sent_shard(client_ip)
.read()
.await
.contains_key(&client_ip)
}
pub async fn set(&self, domain: &str, data: CachedTlsData) {
let mut guard = self.memory.write().await;
guard.insert(domain.to_string(), Arc::new(data));
@@ -130,6 +316,14 @@ impl TlsFrontCache {
warn!(file = %name, "Skipping TLS cache entry with invalid domain");
continue;
}
if !cert_info_matches_domain(&cached) {
warn!(
file = %name,
domain = %cached.domain,
"Skipping TLS cache entry with mismatched certificate metadata"
);
continue;
}
// fetched_at is skipped during deserialization; approximate with file mtime if available.
if let Ok(meta) = entry.metadata().await
&& let Ok(modified) = meta.modified()
@@ -209,10 +403,100 @@ impl TlsFrontCache {
}
}
fn cert_info_matches_domain(cached: &CachedTlsData) -> bool {
let Some(cert_info) = cached.cert_info.as_ref() else {
return true;
};
if !cert_info.san_names.is_empty() {
return cert_info
.san_names
.iter()
.any(|name| dns_name_matches_domain(name, &cached.domain));
}
cert_info
.subject_cn
.as_deref()
.map_or(true, |name| dns_name_matches_domain(name, &cached.domain))
}
fn dns_name_matches_domain(pattern: &str, domain: &str) -> bool {
let pattern = normalize_dns_name(pattern);
let domain = normalize_dns_name(domain);
if pattern == domain {
return true;
}
let Some(suffix) = pattern.strip_prefix("*.") else {
return false;
};
let Some(prefix) = domain.strip_suffix(suffix) else {
return false;
};
prefix.ends_with('.') && !prefix[..prefix.len() - 1].contains('.')
}
fn normalize_dns_name(value: &str) -> String {
value.trim().trim_end_matches('.').to_ascii_lowercase()
}
#[cfg(test)]
mod tests {
use super::*;
fn cached_with_cert_info(
domain: &str,
subject_cn: Option<&str>,
san_names: Vec<&str>,
) -> CachedTlsData {
CachedTlsData {
server_hello_template: ParsedServerHello {
version: [0x03, 0x03],
random: [0u8; 32],
session_id: Vec::new(),
cipher_suite: [0x13, 0x01],
compression: 0,
extensions: Vec::new(),
},
cert_info: Some(crate::tls_front::types::ParsedCertificateInfo {
not_after_unix: None,
not_before_unix: None,
issuer_cn: None,
subject_cn: subject_cn.map(str::to_string),
san_names: san_names.into_iter().map(str::to_string).collect(),
}),
cert_payload: None,
app_data_records_sizes: vec![1024],
total_app_data_len: 1024,
behavior_profile: TlsBehaviorProfile::default(),
fetched_at: SystemTime::now(),
domain: domain.to_string(),
}
}
#[test]
fn cert_info_domain_match_accepts_exact_san() {
let cached = cached_with_cert_info("b.com", Some("a.com"), vec!["b.com"]);
assert!(cert_info_matches_domain(&cached));
}
#[test]
fn cert_info_domain_match_rejects_wrong_san() {
let cached = cached_with_cert_info("b.com", Some("b.com"), vec!["a.com"]);
assert!(!cert_info_matches_domain(&cached));
}
#[test]
fn cert_info_domain_match_accepts_single_label_wildcard_san() {
let cached = cached_with_cert_info("api.b.com", None, vec!["*.b.com"]);
assert!(cert_info_matches_domain(&cached));
}
#[test]
fn cert_info_domain_match_rejects_multi_label_wildcard_san() {
let cached = cached_with_cert_info("deep.api.b.com", None, vec!["*.b.com"]);
assert!(!cert_info_matches_domain(&cached));
}
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_uses_ttl() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
@@ -230,10 +514,68 @@ mod tests {
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_zero_ttl_always_allows_full_payload() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
let ttl = Duration::ZERO;
assert!(cache.take_full_cert_budget_for_ip(ip, ttl).await);
assert!(cache.take_full_cert_budget_for_ip(ip, ttl).await);
for idx in 0..100_000u32 {
let ip = IpAddr::V4(std::net::Ipv4Addr::new(
10,
((idx >> 16) & 0xff) as u8,
((idx >> 8) & 0xff) as u8,
(idx & 0xff) as u8,
));
assert!(cache.take_full_cert_budget_for_ip(ip, ttl).await);
}
assert!(cache.full_cert_sent_is_empty_for_tests().await);
}
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_sweeps_expired_entries_when_due() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
let ttl = Duration::from_secs(1);
let stale_seen_at = Instant::now()
.checked_sub(Duration::from_secs(10))
.unwrap_or_else(Instant::now);
cache
.insert_full_cert_sent_for_tests(stale_ip, stale_seen_at)
.await;
cache
.full_cert_sent_last_sweep_epoch_secs
.store(0, Ordering::Relaxed);
assert!(cache.take_full_cert_budget_for_ip(new_ip, ttl).await);
assert!(!cache.full_cert_sent_contains_for_tests(stale_ip).await);
assert!(cache.full_cert_sent_contains_for_tests(new_ip).await);
}
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_does_not_sweep_every_call() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
let ttl = Duration::from_secs(1);
let stale_seen_at = Instant::now()
.checked_sub(Duration::from_secs(10))
.unwrap_or_else(Instant::now);
let now_epoch_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
cache
.insert_full_cert_sent_for_tests(stale_ip, stale_seen_at)
.await;
cache
.full_cert_sent_last_sweep_epoch_secs
.store(now_epoch_secs, Ordering::Relaxed);
assert!(cache.take_full_cert_budget_for_ip(new_ip, ttl).await);
assert!(cache.full_cert_sent_contains_for_tests(stale_ip).await);
assert!(cache.full_cert_sent_contains_for_tests(new_ip).await);
}
}

View File

@@ -5,7 +5,9 @@ use crate::protocol::constants::{
MAX_TLS_CIPHERTEXT_SIZE, TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER,
TLS_RECORD_HANDSHAKE, TLS_VERSION,
};
use crate::protocol::tls::{TLS_DIGEST_LEN, TLS_DIGEST_POS, gen_fake_x25519_key};
use crate::protocol::tls::{
ClientHelloTlsVersion, TLS_DIGEST_LEN, TLS_DIGEST_POS, gen_fake_x25519_key,
};
use crate::tls_front::types::{CachedTlsData, ParsedCertificateInfo, TlsProfileSource};
use crc32fast::Hasher;
@@ -18,9 +20,6 @@ fn jitter_and_clamp_sizes(sizes: &[usize], rng: &SecureRandom) -> Vec<usize> {
.iter()
.map(|&size| {
let base = size.clamp(MIN_APP_DATA, MAX_APP_DATA);
if base == MIN_APP_DATA || base == MAX_APP_DATA {
return base;
}
let jitter_range = ((base as f64) * 0.03).round() as i64;
if jitter_range == 0 {
return base;
@@ -69,9 +68,19 @@ fn ensure_payload_capacity(mut sizes: Vec<usize>, payload_len: usize) -> Vec<usi
fn emulated_app_data_sizes(cached: &CachedTlsData) -> Vec<usize> {
match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => {
if !cached.behavior_profile.app_data_record_sizes.is_empty() {
return cached.behavior_profile.app_data_record_sizes.clone();
}
return cached
.app_data_records_sizes
.first()
.copied()
.or_else(|| {
cached
.behavior_profile
.app_data_record_sizes
.first()
.copied()
})
.map(|size| vec![size])
.unwrap_or_else(|| vec![cached.total_app_data_len.max(1024)]);
}
TlsProfileSource::Default | TlsProfileSource::Rustls => {}
}
@@ -83,8 +92,8 @@ fn emulated_app_data_sizes(cached: &CachedTlsData) -> Vec<usize> {
sizes
}
fn emulated_change_cipher_spec_count(cached: &CachedTlsData) -> usize {
usize::from(cached.behavior_profile.change_cipher_spec_count.max(1))
fn emulated_change_cipher_spec_count(_cached: &CachedTlsData) -> usize {
1
}
fn emulated_ticket_record_sizes(
@@ -92,19 +101,20 @@ fn emulated_ticket_record_sizes(
new_session_tickets: u8,
rng: &SecureRandom,
) -> Vec<usize> {
let mut sizes = match cached.behavior_profile.source {
let target_count = usize::from(new_session_tickets.min(MAX_TICKET_RECORDS as u8));
if target_count == 0 {
return Vec::new();
}
let profiled_sizes = match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => {
cached.behavior_profile.ticket_record_sizes.clone()
cached.behavior_profile.ticket_record_sizes.as_slice()
}
TlsProfileSource::Default | TlsProfileSource::Rustls => Vec::new(),
TlsProfileSource::Default | TlsProfileSource::Rustls => &[],
};
let target_count = sizes
.len()
.max(usize::from(
new_session_tickets.min(MAX_TICKET_RECORDS as u8),
))
.min(MAX_TICKET_RECORDS);
let mut sizes = Vec::with_capacity(target_count);
sizes.extend(profiled_sizes.iter().copied().take(target_count));
while sizes.len() < target_count {
sizes.push(rng.range(48) + 48);
@@ -182,6 +192,8 @@ pub fn build_emulated_server_hello(
session_id: &[u8],
cached: &CachedTlsData,
use_full_cert_payload: bool,
serverhello_compact: bool,
client_tls_version: ClientHelloTlsVersion,
rng: &SecureRandom,
alpn: Option<Vec<u8>>,
new_session_tickets: u8,
@@ -245,21 +257,45 @@ pub fn build_emulated_server_hello(
}
// --- ApplicationData (fake encrypted records) ---
let mut sizes = jitter_and_clamp_sizes(&emulated_app_data_sizes(cached), rng);
let compact_payload = cached
.cert_info
.as_ref()
.and_then(build_compact_cert_info_payload)
.and_then(hash_compact_cert_info_payload);
let selected_payload: Option<&[u8]> = if use_full_cert_payload {
let mut sizes = {
let base_sizes = emulated_app_data_sizes(cached);
match cached.behavior_profile.source {
TlsProfileSource::Raw | TlsProfileSource::Merged => base_sizes
.into_iter()
.map(|size| size.clamp(MIN_APP_DATA, MAX_APP_DATA))
.collect(),
TlsProfileSource::Default | TlsProfileSource::Rustls => {
jitter_and_clamp_sizes(&base_sizes, rng)
}
}
};
let compact_payload = if serverhello_compact {
cached
.cert_payload
.cert_info
.as_ref()
.map(|payload| payload.certificate_message.as_slice())
.filter(|payload| !payload.is_empty())
.or(compact_payload.as_deref())
.and_then(build_compact_cert_info_payload)
.and_then(hash_compact_cert_info_payload)
} else {
compact_payload.as_deref()
None
};
let full_payload = cached
.cert_payload
.as_ref()
.map(|payload| payload.certificate_message.as_slice())
.filter(|payload| !payload.is_empty());
let selected_payload: Option<&[u8]> = match client_tls_version {
ClientHelloTlsVersion::Tls13 => None,
ClientHelloTlsVersion::Tls12 => {
if serverhello_compact {
if use_full_cert_payload {
full_payload.or(compact_payload.as_deref())
} else {
compact_payload.as_deref()
}
} else {
full_payload
}
}
};
if let Some(payload) = selected_payload {
@@ -383,6 +419,7 @@ mod tests {
use crate::protocol::constants::{
TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE,
};
use crate::protocol::tls::ClientHelloTlsVersion;
fn first_app_data_payload(response: &[u8]) -> &[u8] {
let hello_len = u16::from_be_bytes([response[3], response[4]]) as usize;
@@ -429,6 +466,8 @@ mod tests {
&[0x22; 16],
&cached,
true,
true,
ClientHelloTlsVersion::Tls12,
&rng,
None,
0,
@@ -455,6 +494,8 @@ mod tests {
&[0x33; 16],
&cached,
true,
true,
ClientHelloTlsVersion::Tls12,
&rng,
None,
0,
@@ -487,6 +528,8 @@ mod tests {
&[0x55; 16],
&cached,
false,
true,
ClientHelloTlsVersion::Tls12,
&rng,
None,
0,
@@ -511,7 +554,69 @@ mod tests {
}
#[test]
fn test_build_emulated_server_hello_replays_tail_records_for_profiled_tls() {
fn test_build_emulated_server_hello_tls13_never_uses_cert_payload() {
let cert_msg = vec![0x0b, 0x00, 0x00, 0x05, 0x00, 0xaa, 0xbb, 0xcc, 0xdd];
let cached = make_cached(Some(TlsCertPayload {
cert_chain_der: vec![vec![0x30, 0x01, 0x00]],
certificate_message: cert_msg.clone(),
}));
let rng = SecureRandom::new();
let response = build_emulated_server_hello(
b"secret",
&[0x56; 32],
&[0x78; 16],
&cached,
true,
true,
ClientHelloTlsVersion::Tls13,
&rng,
None,
0,
);
let payload = first_app_data_payload(&response);
assert!(
!payload.starts_with(&cert_msg),
"TLS 1.3 response path must not expose certificate payload bytes"
);
}
#[test]
fn test_build_emulated_server_hello_compact_disabled_skips_compact_payload() {
let mut cached = make_cached(None);
cached.cert_info = Some(crate::tls_front::types::ParsedCertificateInfo {
not_after_unix: Some(1_900_000_000),
not_before_unix: Some(1_700_000_000),
issuer_cn: Some("Issuer".to_string()),
subject_cn: Some("example.com".to_string()),
san_names: vec!["example.com".to_string()],
});
let rng = SecureRandom::new();
let response = build_emulated_server_hello(
b"secret",
&[0x90; 32],
&[0x91; 16],
&cached,
false,
false,
ClientHelloTlsVersion::Tls12,
&rng,
Some(b"h2".to_vec()),
0,
);
let payload = first_app_data_payload(&response);
let expected_alpn_marker = [0x00u8, 0x10, 0x00, 0x05, 0x00, 0x03, 0x02, b'h', b'2'];
assert!(
payload.starts_with(&expected_alpn_marker),
"when compact mode is disabled and no full cert payload exists, the random/alpn path must be used"
);
}
#[test]
fn test_build_emulated_server_hello_ignores_tail_records_for_profiled_tls() {
let mut cached = make_cached(None);
cached.app_data_records_sizes = vec![27, 3905, 537, 69];
cached.total_app_data_len = 4538;
@@ -526,6 +631,8 @@ mod tests {
&[0x34; 16],
&cached,
false,
true,
ClientHelloTlsVersion::Tls13,
&rng,
None,
0,
@@ -533,19 +640,11 @@ mod tests {
let hello_len = u16::from_be_bytes([response[3], response[4]]) as usize;
let ccs_start = 5 + hello_len;
let mut pos = ccs_start + 6;
let mut app_lengths = Vec::new();
while pos + 5 <= response.len() {
assert_eq!(response[pos], TLS_RECORD_APPLICATION);
let record_len = u16::from_be_bytes([response[pos + 3], response[pos + 4]]) as usize;
app_lengths.push(record_len);
pos += 5 + record_len;
}
assert_eq!(app_lengths.len(), 4);
assert_eq!(app_lengths[0], 64);
assert_eq!(app_lengths[3], 69);
assert!(app_lengths[1] >= 64);
assert!(app_lengths[2] >= 64);
let app_start = ccs_start + 6;
let app_len =
u16::from_be_bytes([response[app_start + 3], response[app_start + 4]]) as usize;
assert_eq!(response[app_start], TLS_RECORD_APPLICATION);
assert_eq!(app_len, 64);
assert_eq!(app_start + 5 + app_len, response.len());
}
}

View File

@@ -3,7 +3,9 @@
use dashmap::DashMap;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use anyhow::{Result, anyhow};
@@ -20,6 +22,7 @@ use rustls::client::ClientConfig;
use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
use rustls::{DigitallySignedStruct, Error as RustlsError};
use x25519_dalek::{X25519_BASEPOINT_BYTES, x25519};
use x509_parser::certificate::X509Certificate;
use x509_parser::prelude::FromDer;
@@ -143,12 +146,37 @@ enum FetchErrorKind {
Other,
}
const PROFILE_CACHE_MAX_ENTRIES: usize = 4096;
static PROFILE_CACHE: OnceLock<DashMap<ProfileCacheKey, ProfileCacheValue>> = OnceLock::new();
static PROFILE_CACHE_INSERT_GUARD: OnceLock<Mutex<()>> = OnceLock::new();
static PROFILE_CACHE_CAP_DROPS: AtomicU64 = AtomicU64::new(0);
fn profile_cache() -> &'static DashMap<ProfileCacheKey, ProfileCacheValue> {
PROFILE_CACHE.get_or_init(DashMap::new)
}
fn profile_cache_insert_guard() -> &'static Mutex<()> {
PROFILE_CACHE_INSERT_GUARD.get_or_init(|| Mutex::new(()))
}
fn sweep_expired_profile_cache(ttl: Duration, now: Instant) {
if ttl.is_zero() {
return;
}
profile_cache().retain(|_, value| now.saturating_duration_since(value.updated_at) <= ttl);
}
/// Current number of adaptive TLS fetch profile-cache entries.
pub(crate) fn profile_cache_entries_for_metrics() -> usize {
profile_cache().len()
}
/// Number of fresh profile-cache winners skipped because the cache was full.
pub(crate) fn profile_cache_cap_drops_for_metrics() -> u64 {
PROFILE_CACHE_CAP_DROPS.load(Ordering::Relaxed)
}
fn route_hint(
upstream: Option<&std::sync::Arc<crate::transport::UpstreamManager>>,
unix_sock: Option<&str>,
@@ -266,6 +294,43 @@ fn remember_profile_success(
let Some(key) = cache_key else {
return;
};
remember_profile_success_with_cap(strategy, key, profile, now, PROFILE_CACHE_MAX_ENTRIES);
}
fn remember_profile_success_with_cap(
strategy: &TlsFetchStrategy,
key: ProfileCacheKey,
profile: TlsFetchProfile,
now: Instant,
max_entries: usize,
) {
let Ok(_guard) = profile_cache_insert_guard().lock() else {
PROFILE_CACHE_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return;
};
if max_entries == 0 {
PROFILE_CACHE_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return;
}
if profile_cache().contains_key(&key) {
profile_cache().insert(
key,
ProfileCacheValue {
profile,
updated_at: now,
},
);
return;
}
if profile_cache().len() >= max_entries {
// TLS fetch is control-plane work; sweeping under a tiny mutex keeps
// profile-cache cardinality hard-bounded without touching relay hot paths.
sweep_expired_profile_cache(strategy.profile_cache_ttl, now);
}
if profile_cache().len() >= max_entries {
PROFILE_CACHE_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return;
}
profile_cache().insert(
key,
ProfileCacheValue {
@@ -275,7 +340,7 @@ fn remember_profile_success(
);
}
fn build_client_config() -> Arc<ClientConfig> {
fn build_client_config(alpn_protocols: &[&[u8]]) -> Arc<ClientConfig> {
let root = rustls::RootCertStore::empty();
let provider = rustls::crypto::ring::default_provider();
@@ -288,6 +353,7 @@ fn build_client_config() -> Arc<ClientConfig> {
config
.dangerous()
.set_certificate_verifier(Arc::new(NoVerify));
config.alpn_protocols = alpn_protocols.iter().map(|proto| proto.to_vec()).collect();
Arc::new(config)
}
@@ -359,6 +425,22 @@ fn profile_alpn(profile: TlsFetchProfile) -> &'static [&'static [u8]] {
}
}
fn profile_alpn_labels(profile: TlsFetchProfile) -> &'static [&'static str] {
const H2_HTTP11: &[&str] = &["h2", "http/1.1"];
const HTTP11: &[&str] = &["http/1.1"];
match profile {
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => H2_HTTP11,
TlsFetchProfile::CompatTls12 | TlsFetchProfile::LegacyMinimal => HTTP11,
}
}
fn profile_session_id_len(profile: TlsFetchProfile) -> usize {
match profile {
TlsFetchProfile::ModernChromeLike | TlsFetchProfile::ModernFirefoxLike => 32,
TlsFetchProfile::CompatTls12 | TlsFetchProfile::LegacyMinimal => 0,
}
}
fn profile_supported_versions(profile: TlsFetchProfile) -> &'static [u16] {
const MODERN: &[u16] = &[0x0304, 0x0303];
const COMPAT: &[u16] = &[0x0303, 0x0304];
@@ -413,8 +495,20 @@ fn build_client_hello(
body.extend_from_slice(&rng.bytes(32));
}
// Session ID: empty
body.push(0);
// Use non-empty Session ID for modern TLS 1.3-like profiles to reduce middlebox friction.
let session_id_len = profile_session_id_len(profile);
let session_id = if session_id_len == 0 {
Vec::new()
} else if deterministic {
deterministic_bytes(
&format!("tls-fetch-session:{sni}:{}", profile.as_str()),
session_id_len,
)
} else {
rng.bytes(session_id_len)
};
body.push(session_id.len() as u8);
body.extend_from_slice(&session_id);
let mut cipher_suites = profile_cipher_suites(profile).to_vec();
if grease_enabled {
@@ -433,16 +527,26 @@ fn build_client_hello(
// === Extensions ===
let mut exts = Vec::new();
let mut push_extension = |ext_type: u16, data: &[u8]| {
exts.extend_from_slice(&ext_type.to_be_bytes());
exts.extend_from_slice(&(data.len() as u16).to_be_bytes());
exts.extend_from_slice(data);
};
// server_name (SNI)
let sni_bytes = sni.as_bytes();
let mut sni_ext = Vec::with_capacity(5 + sni_bytes.len());
sni_ext.extend_from_slice(&(sni_bytes.len() as u16 + 3).to_be_bytes());
sni_ext.push(0); // host_name
sni_ext.push(0);
sni_ext.extend_from_slice(&(sni_bytes.len() as u16).to_be_bytes());
sni_ext.extend_from_slice(sni_bytes);
exts.extend_from_slice(&0x0000u16.to_be_bytes());
exts.extend_from_slice(&(sni_ext.len() as u16).to_be_bytes());
exts.extend_from_slice(&sni_ext);
push_extension(0x0000, &sni_ext);
// Chrome-like profile keeps browser-like ordering and extension set.
if matches!(profile, TlsFetchProfile::ModernChromeLike) {
// ec_point_formats: uncompressed only.
push_extension(0x000b, &[0x01, 0x00]);
}
// supported_groups
let mut groups = profile_groups(profile).to_vec();
@@ -450,11 +554,16 @@ fn build_client_hello(
let grease = grease_value(rng, deterministic, &format!("group:{sni}"));
groups.insert(0, grease);
}
exts.extend_from_slice(&0x000au16.to_be_bytes());
exts.extend_from_slice(&((2 + groups.len() * 2) as u16).to_be_bytes());
exts.extend_from_slice(&(groups.len() as u16 * 2).to_be_bytes());
let mut groups_ext = Vec::with_capacity(2 + groups.len() * 2);
groups_ext.extend_from_slice(&(groups.len() as u16 * 2).to_be_bytes());
for g in groups {
exts.extend_from_slice(&g.to_be_bytes());
groups_ext.extend_from_slice(&g.to_be_bytes());
}
push_extension(0x000a, &groups_ext);
if matches!(profile, TlsFetchProfile::ModernChromeLike) {
// session_ticket
push_extension(0x0023, &[]);
}
// signature_algorithms
@@ -463,12 +572,12 @@ fn build_client_hello(
let grease = grease_value(rng, deterministic, &format!("sigalg:{sni}"));
sig_algs.insert(0, grease);
}
exts.extend_from_slice(&0x000du16.to_be_bytes());
exts.extend_from_slice(&((2 + sig_algs.len() * 2) as u16).to_be_bytes());
exts.extend_from_slice(&(sig_algs.len() as u16 * 2).to_be_bytes());
let mut sig_algs_ext = Vec::with_capacity(2 + sig_algs.len() * 2);
sig_algs_ext.extend_from_slice(&(sig_algs.len() as u16 * 2).to_be_bytes());
for a in sig_algs {
exts.extend_from_slice(&a.to_be_bytes());
sig_algs_ext.extend_from_slice(&a.to_be_bytes());
}
push_extension(0x000d, &sig_algs_ext);
// supported_versions
let mut versions = profile_supported_versions(profile).to_vec();
@@ -476,30 +585,32 @@ fn build_client_hello(
let grease = grease_value(rng, deterministic, &format!("version:{sni}"));
versions.insert(0, grease);
}
exts.extend_from_slice(&0x002bu16.to_be_bytes());
exts.extend_from_slice(&((1 + versions.len() * 2) as u16).to_be_bytes());
exts.push((versions.len() * 2) as u8);
let mut versions_ext = Vec::with_capacity(1 + versions.len() * 2);
versions_ext.push((versions.len() * 2) as u8);
for v in versions {
exts.extend_from_slice(&v.to_be_bytes());
versions_ext.extend_from_slice(&v.to_be_bytes());
}
push_extension(0x002b, &versions_ext);
if matches!(profile, TlsFetchProfile::ModernChromeLike) {
// psk_key_exchange_modes: psk_dhe_ke
push_extension(0x002d, &[0x01, 0x01]);
}
// key_share (x25519)
let key = if deterministic {
let det = deterministic_bytes(&format!("keyshare:{sni}"), 32);
let mut key = [0u8; 32];
key.copy_from_slice(&det);
key
} else {
gen_key_share(rng)
};
let key = gen_key_share(
rng,
deterministic,
&format!("tls-fetch-keyshare:{sni}:{}", profile.as_str()),
);
let mut keyshare = Vec::with_capacity(4 + key.len());
keyshare.extend_from_slice(&0x001du16.to_be_bytes()); // group
keyshare.extend_from_slice(&0x001du16.to_be_bytes());
keyshare.extend_from_slice(&(key.len() as u16).to_be_bytes());
keyshare.extend_from_slice(&key);
exts.extend_from_slice(&0x0033u16.to_be_bytes());
exts.extend_from_slice(&((2 + keyshare.len()) as u16).to_be_bytes());
exts.extend_from_slice(&(keyshare.len() as u16).to_be_bytes());
exts.extend_from_slice(&keyshare);
let mut keyshare_ext = Vec::with_capacity(2 + keyshare.len());
keyshare_ext.extend_from_slice(&(keyshare.len() as u16).to_be_bytes());
keyshare_ext.extend_from_slice(&keyshare);
push_extension(0x0033, &keyshare_ext);
// ALPN
let mut alpn_list = Vec::new();
@@ -508,16 +619,15 @@ fn build_client_hello(
alpn_list.extend_from_slice(proto);
}
if !alpn_list.is_empty() {
exts.extend_from_slice(&0x0010u16.to_be_bytes());
exts.extend_from_slice(&((2 + alpn_list.len()) as u16).to_be_bytes());
exts.extend_from_slice(&(alpn_list.len() as u16).to_be_bytes());
exts.extend_from_slice(&alpn_list);
let mut alpn_ext = Vec::with_capacity(2 + alpn_list.len());
alpn_ext.extend_from_slice(&(alpn_list.len() as u16).to_be_bytes());
alpn_ext.extend_from_slice(&alpn_list);
push_extension(0x0010, &alpn_ext);
}
if grease_enabled {
let grease = grease_value(rng, deterministic, &format!("ext:{sni}"));
exts.extend_from_slice(&grease.to_be_bytes());
exts.extend_from_slice(&0u16.to_be_bytes());
push_extension(grease, &[]);
}
// padding to reduce recognizability and keep length ~500 bytes
@@ -553,10 +663,14 @@ fn build_client_hello(
record
}
fn gen_key_share(rng: &SecureRandom) -> [u8; 32] {
let mut key = [0u8; 32];
key.copy_from_slice(&rng.bytes(32));
key
fn gen_key_share(rng: &SecureRandom, deterministic: bool, seed: &str) -> [u8; 32] {
let mut scalar = [0u8; 32];
if deterministic {
scalar.copy_from_slice(&deterministic_bytes(seed, 32));
} else {
scalar.copy_from_slice(&rng.bytes(32));
}
x25519(scalar, X25519_BASEPOINT_BYTES)
}
async fn read_tls_record<S>(stream: &mut S) -> Result<(u8, Vec<u8>)>
@@ -1018,6 +1132,7 @@ async fn fetch_via_rustls_stream<S>(
host: &str,
sni: &str,
proxy_header: Option<Vec<u8>>,
alpn_protocols: &[&[u8]],
) -> Result<TlsFetchResult>
where
S: AsyncRead + AsyncWrite + Unpin,
@@ -1028,7 +1143,7 @@ where
stream.flush().await?;
}
let config = build_client_config();
let config = build_client_config(alpn_protocols);
let connector = TlsConnector::from(config);
let server_name = ServerName::try_from(sni.to_owned())
@@ -1113,6 +1228,7 @@ async fn fetch_via_rustls(
proxy_protocol: u8,
unix_sock: Option<&str>,
strict_route: bool,
alpn_protocols: &[&[u8]],
) -> Result<TlsFetchResult> {
#[cfg(unix)]
if let Some(sock_path) = unix_sock {
@@ -1124,7 +1240,8 @@ async fn fetch_via_rustls(
"Rustls fetch using mask unix socket"
);
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, None, None);
return fetch_via_rustls_stream(stream, host, sni, proxy_header).await;
return fetch_via_rustls_stream(stream, host, sni, proxy_header, alpn_protocols)
.await;
}
Ok(Err(e)) => {
warn!(
@@ -1152,7 +1269,7 @@ async fn fetch_via_rustls(
.await?;
let (src_addr, dst_addr) = socket_addrs_from_upstream_stream(&stream);
let proxy_header = build_tls_fetch_proxy_header(proxy_protocol, src_addr, dst_addr);
fetch_via_rustls_stream(stream, host, sni, proxy_header).await
fetch_via_rustls_stream(stream, host, sni, proxy_header, alpn_protocols).await
}
/// Fetch real TLS metadata with an adaptive multi-profile strategy.
@@ -1191,6 +1308,14 @@ pub async fn fetch_real_tls_with_strategy(
break;
}
let timeout_for_attempt = attempt_timeout.min(total_budget - elapsed);
debug!(
sni = %sni,
profile = profile.as_str(),
alpn = ?profile_alpn_labels(profile),
grease_enabled = strategy.grease_enabled,
deterministic = strategy.deterministic,
"TLS fetch ClientHello params (raw)"
);
match fetch_via_raw_tls(
host,
@@ -1256,6 +1381,16 @@ pub async fn fetch_real_tls_with_strategy(
}
let rustls_timeout = attempt_timeout.min(total_budget - elapsed);
let rustls_profile = selected_profile.unwrap_or(TlsFetchProfile::ModernChromeLike);
let rustls_alpn_protocols = profile_alpn(rustls_profile);
debug!(
sni = %sni,
profile = rustls_profile.as_str(),
alpn = ?profile_alpn_labels(rustls_profile),
grease_enabled = strategy.grease_enabled,
deterministic = strategy.deterministic,
"TLS fetch ClientHello params (rustls)"
);
let rustls_result = fetch_via_rustls(
host,
port,
@@ -1266,6 +1401,7 @@ pub async fn fetch_real_tls_with_strategy(
proxy_protocol,
unix_sock,
strategy.strict_route,
rustls_alpn_protocols,
)
.await;
@@ -1327,8 +1463,8 @@ mod tests {
use super::{
ProfileCacheValue, TlsFetchStrategy, build_client_hello, build_tls_fetch_proxy_header,
derive_behavior_profile, encode_tls13_certificate_message, order_profiles, profile_cache,
profile_cache_key,
derive_behavior_profile, encode_tls13_certificate_message, fetch_via_rustls_stream,
order_profiles, profile_alpn, profile_cache, profile_cache_key,
};
use crate::config::TlsFetchProfile;
use crate::crypto::SecureRandom;
@@ -1336,11 +1472,115 @@ mod tests {
TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE,
};
use crate::tls_front::types::TlsProfileSource;
use tokio::io::AsyncReadExt;
struct ParsedClientHelloForTest {
session_id: Vec<u8>,
extensions: Vec<(u16, Vec<u8>)>,
}
fn read_u24(bytes: &[u8]) -> usize {
((bytes[0] as usize) << 16) | ((bytes[1] as usize) << 8) | (bytes[2] as usize)
}
fn parse_client_hello_for_test(record: &[u8]) -> ParsedClientHelloForTest {
assert!(record.len() >= 9, "record too short");
assert_eq!(record[0], TLS_RECORD_HANDSHAKE, "not a handshake record");
let record_len = u16::from_be_bytes([record[3], record[4]]) as usize;
assert_eq!(record.len(), 5 + record_len, "record length mismatch");
let handshake = &record[5..];
assert_eq!(handshake[0], 0x01, "not a ClientHello handshake");
let hello_len = read_u24(&handshake[1..4]);
assert_eq!(handshake.len(), 4 + hello_len, "handshake length mismatch");
let hello = &handshake[4..];
let mut pos = 0usize;
pos += 2;
pos += 32;
let session_len = hello[pos] as usize;
pos += 1;
let session_id = hello[pos..pos + session_len].to_vec();
pos += session_len;
let cipher_len = u16::from_be_bytes([hello[pos], hello[pos + 1]]) as usize;
pos += 2 + cipher_len;
let compression_len = hello[pos] as usize;
pos += 1 + compression_len;
let ext_len = u16::from_be_bytes([hello[pos], hello[pos + 1]]) as usize;
pos += 2;
let ext_end = pos + ext_len;
assert_eq!(ext_end, hello.len(), "extensions length mismatch");
let mut extensions = Vec::new();
while pos + 4 <= ext_end {
let ext_type = u16::from_be_bytes([hello[pos], hello[pos + 1]]);
let data_len = u16::from_be_bytes([hello[pos + 2], hello[pos + 3]]) as usize;
pos += 4;
let data = hello[pos..pos + data_len].to_vec();
pos += data_len;
extensions.push((ext_type, data));
}
assert_eq!(pos, ext_end, "extension parse did not consume all bytes");
ParsedClientHelloForTest {
session_id,
extensions,
}
}
fn parse_alpn_protocols(data: &[u8]) -> Vec<Vec<u8>> {
assert!(data.len() >= 2, "ALPN extension is too short");
let protocols_len = u16::from_be_bytes([data[0], data[1]]) as usize;
assert_eq!(protocols_len + 2, data.len(), "ALPN list length mismatch");
let mut pos = 2usize;
let mut out = Vec::new();
while pos < data.len() {
let len = data[pos] as usize;
pos += 1;
out.push(data[pos..pos + len].to_vec());
pos += len;
}
out
}
async fn capture_rustls_client_hello_record(
alpn_protocols: &'static [&'static [u8]],
) -> Vec<u8> {
let (client, mut server) = tokio::io::duplex(32 * 1024);
let fetch_task = tokio::spawn(async move {
fetch_via_rustls_stream(client, "example.com", "example.com", None, alpn_protocols)
.await
});
let mut header = [0u8; 5];
server
.read_exact(&mut header)
.await
.expect("must read client hello record header");
let body_len = u16::from_be_bytes([header[3], header[4]]) as usize;
let mut body = vec![0u8; body_len];
server
.read_exact(&mut body)
.await
.expect("must read client hello record body");
drop(server);
let result = fetch_task.await.expect("fetch task must join");
assert!(
result.is_err(),
"capture task should end with handshake error"
);
let mut record = Vec::with_capacity(5 + body_len);
record.extend_from_slice(&header);
record.extend_from_slice(&body);
record
}
#[test]
fn test_encode_tls13_certificate_message_single_cert() {
let cert = vec![0x30, 0x03, 0x02, 0x01, 0x01];
@@ -1470,6 +1710,186 @@ mod tests {
assert_eq!(first, second);
}
#[test]
fn test_raw_client_hello_alpn_matches_profile() {
let rng = SecureRandom::new();
for profile in [
TlsFetchProfile::ModernChromeLike,
TlsFetchProfile::ModernFirefoxLike,
TlsFetchProfile::CompatTls12,
TlsFetchProfile::LegacyMinimal,
] {
let hello = build_client_hello("alpn.example", &rng, profile, false, true);
let parsed = parse_client_hello_for_test(&hello);
let alpn_ext = parsed
.extensions
.iter()
.find(|(ext_type, _)| *ext_type == 0x0010)
.expect("ALPN extension must exist");
let parsed_alpn = parse_alpn_protocols(&alpn_ext.1);
let expected_alpn = profile_alpn(profile)
.iter()
.map(|proto| proto.to_vec())
.collect::<Vec<_>>();
assert_eq!(
parsed_alpn,
expected_alpn,
"ALPN mismatch for {}",
profile.as_str()
);
}
}
#[test]
fn test_modern_chrome_like_browser_extension_layout() {
let rng = SecureRandom::new();
let hello = build_client_hello(
"chrome.example",
&rng,
TlsFetchProfile::ModernChromeLike,
false,
true,
);
let parsed = parse_client_hello_for_test(&hello);
assert_eq!(
parsed.session_id.len(),
32,
"modern chrome must use non-empty session id"
);
let extension_ids = parsed
.extensions
.iter()
.map(|(ext_type, _)| *ext_type)
.collect::<Vec<_>>();
let expected_prefix = [
0x0000, 0x000b, 0x000a, 0x0023, 0x000d, 0x002b, 0x002d, 0x0033, 0x0010,
];
assert!(
extension_ids.as_slice().starts_with(&expected_prefix),
"unexpected extension order: {extension_ids:?}"
);
assert!(
extension_ids.contains(&0x0015),
"modern chrome profile should include padding extension"
);
let key_share = parsed
.extensions
.iter()
.find(|(ext_type, _)| *ext_type == 0x0033)
.expect("key_share extension must exist");
let key_share_data = &key_share.1;
assert!(
key_share_data.len() >= 2 + 4 + 32,
"key_share payload is too short"
);
let entry_len = u16::from_be_bytes([key_share_data[0], key_share_data[1]]) as usize;
assert_eq!(
entry_len,
key_share_data.len() - 2,
"key_share list length mismatch"
);
let group = u16::from_be_bytes([key_share_data[2], key_share_data[3]]);
let key_len = u16::from_be_bytes([key_share_data[4], key_share_data[5]]) as usize;
let key = &key_share_data[6..6 + key_len];
assert_eq!(group, 0x001d, "key_share group must be x25519");
assert_eq!(key_len, 32, "x25519 key length must be 32");
assert!(
key.iter().any(|b| *b != 0),
"x25519 key must not be all zero"
);
}
#[test]
fn test_fallback_profiles_keep_compat_extension_set() {
let rng = SecureRandom::new();
for profile in [
TlsFetchProfile::ModernFirefoxLike,
TlsFetchProfile::CompatTls12,
TlsFetchProfile::LegacyMinimal,
] {
let hello = build_client_hello("fallback.example", &rng, profile, false, true);
let parsed = parse_client_hello_for_test(&hello);
let extension_ids = parsed
.extensions
.iter()
.map(|(ext_type, _)| *ext_type)
.collect::<Vec<_>>();
assert!(extension_ids.contains(&0x0000), "SNI extension must exist");
assert!(
extension_ids.contains(&0x000a),
"supported_groups extension must exist"
);
assert!(
extension_ids.contains(&0x000d),
"signature_algorithms extension must exist"
);
assert!(
extension_ids.contains(&0x002b),
"supported_versions extension must exist"
);
assert!(
extension_ids.contains(&0x0033),
"key_share extension must exist"
);
assert!(extension_ids.contains(&0x0010), "ALPN extension must exist");
assert!(
!extension_ids.contains(&0x000b),
"ec_point_formats must stay chrome-only"
);
assert!(
!extension_ids.contains(&0x0023),
"session_ticket must stay chrome-only"
);
assert!(
!extension_ids.contains(&0x002d),
"psk_key_exchange_modes must stay chrome-only"
);
let expected_session_len = if matches!(profile, TlsFetchProfile::ModernFirefoxLike) {
32
} else {
0
};
assert_eq!(
parsed.session_id.len(),
expected_session_len,
"unexpected session id length for {}",
profile.as_str()
);
}
}
#[tokio::test(flavor = "current_thread")]
async fn test_rustls_client_hello_alpn_matches_selected_profile() {
for profile in [
TlsFetchProfile::ModernChromeLike,
TlsFetchProfile::CompatTls12,
TlsFetchProfile::LegacyMinimal,
] {
let record = capture_rustls_client_hello_record(profile_alpn(profile)).await;
let parsed = parse_client_hello_for_test(&record);
let alpn_ext = parsed
.extensions
.iter()
.find(|(ext_type, _)| *ext_type == 0x0010)
.expect("ALPN extension must exist");
let parsed_alpn = parse_alpn_protocols(&alpn_ext.1);
let expected_alpn = profile_alpn(profile)
.iter()
.map(|proto| proto.to_vec())
.collect::<Vec<_>>();
assert_eq!(
parsed_alpn,
expected_alpn,
"rustls ALPN mismatch for {}",
profile.as_str()
);
}
}
#[test]
fn test_build_tls_fetch_proxy_header_v2_with_tcp_addrs() {
let src: SocketAddr = "198.51.100.10:42000".parse().expect("valid src");

View File

@@ -4,6 +4,7 @@ use crate::crypto::SecureRandom;
use crate::protocol::constants::{
TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE,
};
use crate::protocol::tls::ClientHelloTlsVersion;
use crate::tls_front::emulator::build_emulated_server_hello;
use crate::tls_front::types::{
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsProfileSource,
@@ -52,7 +53,7 @@ fn record_lengths_by_type(response: &[u8], wanted_type: u8) -> Vec<usize> {
}
#[test]
fn emulated_server_hello_replays_profile_change_cipher_spec_count() {
fn emulated_server_hello_keeps_single_change_cipher_spec_for_client_compatibility() {
let cached = make_cached();
let rng = SecureRandom::new();
@@ -62,6 +63,8 @@ fn emulated_server_hello_replays_profile_change_cipher_spec_count() {
&[0x72; 16],
&cached,
false,
true,
ClientHelloTlsVersion::Tls13,
&rng,
None,
0,
@@ -69,12 +72,12 @@ fn emulated_server_hello_replays_profile_change_cipher_spec_count() {
assert_eq!(response[0], TLS_RECORD_HANDSHAKE);
let ccs_records = record_lengths_by_type(&response, TLS_RECORD_CHANGE_CIPHER);
assert_eq!(ccs_records.len(), 2);
assert_eq!(ccs_records.len(), 1);
assert!(ccs_records.iter().all(|len| *len == 1));
}
#[test]
fn emulated_server_hello_replays_profile_ticket_tail_lengths() {
fn emulated_server_hello_does_not_emit_profile_ticket_tail_when_disabled() {
let cached = make_cached();
let rng = SecureRandom::new();
@@ -84,12 +87,35 @@ fn emulated_server_hello_replays_profile_ticket_tail_lengths() {
&[0x82; 16],
&cached,
false,
true,
ClientHelloTlsVersion::Tls13,
&rng,
None,
0,
);
let app_records = record_lengths_by_type(&response, TLS_RECORD_APPLICATION);
assert!(app_records.len() >= 4);
assert_eq!(&app_records[app_records.len() - 2..], &[220, 180]);
assert_eq!(app_records, vec![1200]);
}
#[test]
fn emulated_server_hello_uses_profile_ticket_lengths_when_enabled() {
let cached = make_cached();
let rng = SecureRandom::new();
let response = build_emulated_server_hello(
b"secret",
&[0x91; 32],
&[0x92; 16],
&cached,
false,
true,
ClientHelloTlsVersion::Tls13,
&rng,
None,
2,
);
let app_records = record_lengths_by_type(&response, TLS_RECORD_APPLICATION);
assert_eq!(app_records, vec![1200, 220, 180]);
}

View File

@@ -4,6 +4,7 @@ use crate::crypto::SecureRandom;
use crate::protocol::constants::{
TLS_RECORD_APPLICATION, TLS_RECORD_CHANGE_CIPHER, TLS_RECORD_HANDSHAKE,
};
use crate::protocol::tls::ClientHelloTlsVersion;
use crate::tls_front::emulator::build_emulated_server_hello;
use crate::tls_front::types::{
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
@@ -55,6 +56,8 @@ fn emulated_server_hello_ignores_oversized_alpn_when_marker_would_not_fit() {
&[0x22; 16],
&cached,
true,
true,
ClientHelloTlsVersion::Tls13,
&rng,
Some(oversized_alpn),
0,
@@ -91,6 +94,8 @@ fn emulated_server_hello_embeds_full_alpn_marker_when_body_can_fit() {
&[0x41; 16],
&cached,
true,
true,
ClientHelloTlsVersion::Tls13,
&rng,
Some(b"h2".to_vec()),
0,
@@ -119,6 +124,8 @@ fn emulated_server_hello_prefers_cert_payload_over_alpn_marker() {
&[0x42; 16],
&cached,
true,
true,
ClientHelloTlsVersion::Tls12,
&rng,
Some(b"h2".to_vec()),
0,

View File

@@ -321,7 +321,14 @@ async fn run_update_cycle(
let mut maps_changed = false;
let mut ready_v4: Option<(ProxyConfigData, u64)> = None;
let cfg_v4 = retry_fetch("https://core.telegram.org/getProxyConfig", upstream.clone()).await;
let cfg_v4 = retry_fetch(
cfg.general
.proxy_config_v4_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfig"),
upstream.clone(),
)
.await;
if let Some(cfg_v4) = cfg_v4
&& snapshot_passes_guards(cfg, &cfg_v4, "getProxyConfig")
{
@@ -346,7 +353,10 @@ async fn run_update_cycle(
let mut ready_v6: Option<(ProxyConfigData, u64)> = None;
let cfg_v6 = retry_fetch(
"https://core.telegram.org/getProxyConfigV6",
cfg.general
.proxy_config_v6_url
.as_deref()
.unwrap_or("https://core.telegram.org/getProxyConfigV6"),
upstream.clone(),
)
.await;
@@ -430,6 +440,7 @@ async fn run_update_cycle(
match download_proxy_secret_with_max_len_via_upstream(
cfg.general.proxy_secret_len_max,
upstream,
cfg.general.proxy_secret_url.as_deref(),
)
.await
{

View File

@@ -7,7 +7,6 @@ mod model;
mod pressure;
mod scheduler;
#[cfg(test)]
pub(crate) use model::PressureState;
pub(crate) use model::{AdmissionDecision, DispatchAction, DispatchFeedback, SchedulerDecision};
pub(crate) use scheduler::{WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState};

View File

@@ -77,11 +77,12 @@ pub(crate) struct FlowFairnessState {
pub(crate) standing_state: StandingQueueState,
pub(crate) scheduler_state: FlowSchedulerState,
pub(crate) bucket_id: usize,
pub(crate) weight_quanta: u8,
pub(crate) in_active_ring: bool,
}
impl FlowFairnessState {
pub(crate) fn new(flow_id: u64, worker_id: u16, bucket_id: usize) -> Self {
pub(crate) fn new(flow_id: u64, worker_id: u16, bucket_id: usize, weight_quanta: u8) -> Self {
Self {
_flow_id: flow_id,
_worker_id: worker_id,
@@ -97,6 +98,7 @@ impl FlowFairnessState {
standing_state: StandingQueueState::Transient,
scheduler_state: FlowSchedulerState::Idle,
bucket_id,
weight_quanta: weight_quanta.max(1),
in_active_ring: false,
}
}

View File

@@ -12,6 +12,7 @@ pub(crate) struct PressureSignals {
#[derive(Debug, Clone)]
pub(crate) struct PressureConfig {
pub(crate) backpressure_enabled: bool,
pub(crate) evaluate_every_rounds: u32,
pub(crate) transition_hysteresis_rounds: u8,
pub(crate) standing_ratio_pressured_pct: u8,
@@ -32,6 +33,7 @@ pub(crate) struct PressureConfig {
impl Default for PressureConfig {
fn default() -> Self {
Self {
backpressure_enabled: true,
evaluate_every_rounds: 8,
transition_hysteresis_rounds: 3,
standing_ratio_pressured_pct: 20,
@@ -99,6 +101,13 @@ impl PressureEvaluator {
force: bool,
) -> PressureState {
self.rotate_window_if_needed(now, cfg);
if !cfg.backpressure_enabled {
self.state = PressureState::Normal;
self.candidate_state = PressureState::Normal;
self.candidate_hits = 0;
self.rounds_since_eval = 0;
return self.state;
}
self.rounds_since_eval = self.rounds_since_eval.saturating_add(1);
if !force && self.rounds_since_eval < cfg.evaluate_every_rounds.max(1) {
return self.state;
@@ -133,6 +142,10 @@ impl PressureEvaluator {
max_total_queued_bytes: u64,
signals: PressureSignals,
) -> PressureState {
if !cfg.backpressure_enabled {
return PressureState::Normal;
}
let queue_ratio_pct = if max_total_queued_bytes == 0 {
100
} else {

View File

@@ -1,6 +1,7 @@
use std::collections::{HashMap, VecDeque};
use std::collections::{HashMap, HashSet, VecDeque};
use std::time::{Duration, Instant};
use crate::protocol::constants::RPC_FLAG_QUICKACK;
use bytes::Bytes;
use super::model::{
@@ -13,6 +14,7 @@ use super::pressure::{PressureConfig, PressureEvaluator, PressureSignals};
#[derive(Debug, Clone)]
pub(crate) struct WorkerFairnessConfig {
pub(crate) worker_id: u16,
pub(crate) backpressure_enabled: bool,
pub(crate) max_active_flows: usize,
pub(crate) max_total_queued_bytes: u64,
pub(crate) max_flow_queued_bytes: u64,
@@ -26,6 +28,8 @@ pub(crate) struct WorkerFairnessConfig {
pub(crate) max_consecutive_stalls_before_close: u8,
pub(crate) soft_bucket_count: usize,
pub(crate) soft_bucket_share_pct: u8,
pub(crate) default_flow_weight: u8,
pub(crate) quickack_flow_weight: u8,
pub(crate) pressure: PressureConfig,
}
@@ -33,6 +37,7 @@ impl Default for WorkerFairnessConfig {
fn default() -> Self {
Self {
worker_id: 0,
backpressure_enabled: true,
max_active_flows: 4096,
max_total_queued_bytes: 16 * 1024 * 1024,
max_flow_queued_bytes: 512 * 1024,
@@ -46,6 +51,8 @@ impl Default for WorkerFairnessConfig {
max_consecutive_stalls_before_close: 16,
soft_bucket_count: 64,
soft_bucket_share_pct: 25,
default_flow_weight: 1,
quickack_flow_weight: 4,
pressure: PressureConfig::default(),
}
}
@@ -57,9 +64,9 @@ struct FlowEntry {
}
impl FlowEntry {
fn new(flow_id: u64, worker_id: u16, bucket_id: usize) -> Self {
fn new(flow_id: u64, worker_id: u16, bucket_id: usize, weight_quanta: u8) -> Self {
Self {
fairness: FlowFairnessState::new(flow_id, worker_id, bucket_id),
fairness: FlowFairnessState::new(flow_id, worker_id, bucket_id, weight_quanta),
queue: VecDeque::new(),
}
}
@@ -86,6 +93,7 @@ pub(crate) struct WorkerFairnessState {
pressure: PressureEvaluator,
flows: HashMap<u64, FlowEntry>,
active_ring: VecDeque<u64>,
active_ring_members: HashSet<u64>,
total_queued_bytes: u64,
bucket_queued_bytes: Vec<u64>,
bucket_active_flows: Vec<usize>,
@@ -101,13 +109,15 @@ pub(crate) struct WorkerFairnessState {
}
impl WorkerFairnessState {
pub(crate) fn new(config: WorkerFairnessConfig, now: Instant) -> Self {
pub(crate) fn new(mut config: WorkerFairnessConfig, now: Instant) -> Self {
config.pressure.backpressure_enabled = config.backpressure_enabled;
let bucket_count = config.soft_bucket_count.max(1);
Self {
config,
pressure: PressureEvaluator::new(now),
flows: HashMap::new(),
active_ring: VecDeque::new(),
active_ring_members: HashSet::new(),
total_queued_bytes: 0,
bucket_queued_bytes: vec![0; bucket_count],
bucket_active_flows: vec![0; bucket_count],
@@ -127,6 +137,15 @@ impl WorkerFairnessState {
self.pressure.state()
}
pub(crate) fn set_backpressure_enabled(&mut self, enabled: bool) {
if self.config.backpressure_enabled == enabled {
return;
}
self.config.backpressure_enabled = enabled;
self.config.pressure.backpressure_enabled = enabled;
self.evaluate_pressure(Instant::now(), true);
}
pub(crate) fn snapshot(&self) -> WorkerFairnessSnapshot {
WorkerFairnessSnapshot {
pressure_state: self.pressure.state(),
@@ -159,7 +178,7 @@ impl WorkerFairnessState {
};
let frame_bytes = frame.queued_bytes();
if self.pressure.state() == PressureState::Saturated {
if self.config.backpressure_enabled && self.pressure.state() == PressureState::Saturated {
self.pressure
.note_admission_reject(now, &self.config.pressure);
self.enqueue_rejects = self.enqueue_rejects.saturating_add(1);
@@ -184,6 +203,7 @@ impl WorkerFairnessState {
}
let bucket_id = self.bucket_for(conn_id);
let frame_weight = Self::weight_for_flags(&self.config, flags);
let bucket_cap = self
.config
.max_total_queued_bytes
@@ -205,12 +225,13 @@ impl WorkerFairnessState {
self.bucket_active_flows[bucket_id].saturating_add(1);
self.flows.insert(
conn_id,
FlowEntry::new(conn_id, self.config.worker_id, bucket_id),
FlowEntry::new(conn_id, self.config.worker_id, bucket_id, frame_weight),
);
self.flows
.get_mut(&conn_id)
.expect("flow inserted must be retrievable")
};
entry.fairness.weight_quanta = entry.fairness.weight_quanta.max(frame_weight);
if entry.fairness.pending_bytes.saturating_add(frame_bytes)
> self.config.max_flow_queued_bytes
@@ -222,7 +243,8 @@ impl WorkerFairnessState {
return AdmissionDecision::RejectFlowCap;
}
if self.pressure.state() >= PressureState::Shedding
if self.config.backpressure_enabled
&& self.pressure.state() >= PressureState::Shedding
&& entry.fairness.standing_state == StandingQueueState::Standing
{
self.pressure
@@ -242,11 +264,24 @@ impl WorkerFairnessState {
self.bucket_queued_bytes[bucket_id] =
self.bucket_queued_bytes[bucket_id].saturating_add(frame_bytes);
let mut enqueue_active = false;
if !entry.fairness.in_active_ring {
entry.fairness.in_active_ring = true;
self.active_ring.push_back(conn_id);
enqueue_active = true;
}
let pressure_state = self.pressure.state();
let (before_membership, after_membership) = {
let before = Self::flow_membership(&entry.fairness);
Self::classify_flow(&self.config, pressure_state, now, &mut entry.fairness);
let after = Self::flow_membership(&entry.fairness);
(before, after)
};
if enqueue_active {
self.enqueue_active_conn(conn_id);
}
self.apply_flow_membership_delta(before_membership, after_membership);
self.evaluate_pressure(now, true);
AdmissionDecision::Admit
}
@@ -260,62 +295,89 @@ impl WorkerFairnessState {
let Some(conn_id) = self.active_ring.pop_front() else {
break;
};
if !self.active_ring_members.remove(&conn_id) {
continue;
}
let mut candidate = None;
let mut requeue_active = false;
let mut drained_bytes = 0u64;
let mut bucket_id = 0usize;
let mut should_continue = false;
let mut enqueue_active = false;
let mut membership_delta = None;
let pressure_state = self.pressure.state();
if let Some(flow) = self.flows.get_mut(&conn_id) {
bucket_id = flow.fairness.bucket_id;
flow.fairness.in_active_ring = false;
let before_membership = Self::flow_membership(&flow.fairness);
if flow.queue.is_empty() {
flow.fairness.in_active_ring = false;
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
flow.fairness.pending_bytes = 0;
flow.fairness.deficit_bytes = 0;
flow.fairness.queue_started_at = None;
continue;
}
should_continue = true;
} else {
Self::classify_flow(&self.config, pressure_state, now, &mut flow.fairness);
Self::classify_flow(&self.config, pressure_state, now, &mut flow.fairness);
let quantum =
Self::effective_quantum_bytes(&self.config, pressure_state, &flow.fairness);
flow.fairness.deficit_bytes = flow
.fairness
.deficit_bytes
.saturating_add(i64::from(quantum));
self.deficit_grants = self.deficit_grants.saturating_add(1);
let front_len = flow.queue.front().map_or(0, |front| front.queued_bytes());
if flow.fairness.deficit_bytes < front_len as i64 {
flow.fairness.consecutive_skips =
flow.fairness.consecutive_skips.saturating_add(1);
self.deficit_skips = self.deficit_skips.saturating_add(1);
requeue_active = true;
} else if let Some(frame) = flow.queue.pop_front() {
drained_bytes = frame.queued_bytes();
flow.fairness.pending_bytes =
flow.fairness.pending_bytes.saturating_sub(drained_bytes);
let quantum =
Self::effective_quantum_bytes(&self.config, pressure_state, &flow.fairness);
flow.fairness.deficit_bytes = flow
.fairness
.deficit_bytes
.saturating_sub(drained_bytes as i64);
flow.fairness.consecutive_skips = 0;
flow.fairness.queue_started_at =
flow.queue.front().map(|front| front.enqueued_at);
requeue_active = !flow.queue.is_empty();
if !requeue_active {
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
flow.fairness.in_active_ring = false;
.saturating_add(i64::from(quantum));
Self::clamp_deficit_bytes(&self.config, &mut flow.fairness);
self.deficit_grants = self.deficit_grants.saturating_add(1);
let front_len = flow.queue.front().map_or(0, |front| front.queued_bytes());
if flow.fairness.deficit_bytes < front_len as i64 {
flow.fairness.consecutive_skips =
flow.fairness.consecutive_skips.saturating_add(1);
self.deficit_skips = self.deficit_skips.saturating_add(1);
requeue_active = true;
flow.fairness.in_active_ring = true;
enqueue_active = true;
} else if let Some(frame) = flow.queue.pop_front() {
drained_bytes = frame.queued_bytes();
flow.fairness.pending_bytes =
flow.fairness.pending_bytes.saturating_sub(drained_bytes);
flow.fairness.deficit_bytes = flow
.fairness
.deficit_bytes
.saturating_sub(drained_bytes as i64);
Self::clamp_deficit_bytes(&self.config, &mut flow.fairness);
flow.fairness.consecutive_skips = 0;
flow.fairness.queue_started_at =
flow.queue.front().map(|front| front.enqueued_at);
requeue_active = !flow.queue.is_empty();
if !requeue_active {
flow.fairness.scheduler_state = FlowSchedulerState::Idle;
flow.fairness.in_active_ring = false;
flow.fairness.deficit_bytes = 0;
} else {
flow.fairness.in_active_ring = true;
enqueue_active = true;
}
candidate = Some(DispatchCandidate {
pressure_state,
flow_class: flow.fairness.pressure_class,
frame,
});
}
candidate = Some(DispatchCandidate {
pressure_state,
flow_class: flow.fairness.pressure_class,
frame,
});
}
membership_delta = Some((before_membership, Self::flow_membership(&flow.fairness)));
}
if let Some((before_membership, after_membership)) = membership_delta {
self.apply_flow_membership_delta(before_membership, after_membership);
}
if should_continue {
continue;
}
if drained_bytes > 0 {
@@ -324,11 +386,8 @@ impl WorkerFairnessState {
self.bucket_queued_bytes[bucket_id].saturating_sub(drained_bytes);
}
if requeue_active {
if let Some(flow) = self.flows.get_mut(&conn_id) {
flow.fairness.in_active_ring = true;
}
self.active_ring.push_back(conn_id);
if requeue_active && enqueue_active {
self.enqueue_active_conn(conn_id);
}
if let Some(candidate) = candidate {
@@ -348,7 +407,9 @@ impl WorkerFairnessState {
) -> DispatchAction {
match feedback {
DispatchFeedback::Routed => {
let mut membership_delta = None;
if let Some(flow) = self.flows.get_mut(&conn_id) {
let before_membership = Self::flow_membership(&flow.fairness);
flow.fairness.last_drain_at = Some(now);
flow.fairness.recent_drain_bytes = flow
.fairness
@@ -358,54 +419,89 @@ impl WorkerFairnessState {
if flow.fairness.scheduler_state != FlowSchedulerState::Idle {
flow.fairness.scheduler_state = FlowSchedulerState::Active;
}
Self::classify_flow(
&self.config,
self.pressure.state(),
now,
&mut flow.fairness,
);
membership_delta =
Some((before_membership, Self::flow_membership(&flow.fairness)));
}
if let Some((before_membership, after_membership)) = membership_delta {
self.apply_flow_membership_delta(before_membership, after_membership);
}
self.evaluate_pressure(now, false);
DispatchAction::Continue
}
DispatchFeedback::QueueFull => {
self.pressure.note_route_stall(now, &self.config.pressure);
self.downstream_stalls = self.downstream_stalls.saturating_add(1);
if self.config.backpressure_enabled {
self.pressure.note_route_stall(now, &self.config.pressure);
self.downstream_stalls = self.downstream_stalls.saturating_add(1);
}
let state = self.pressure.state();
let Some(flow) = self.flows.get_mut(&conn_id) else {
self.evaluate_pressure(now, true);
return DispatchAction::Continue;
};
let (before_membership, after_membership, should_close_flow, enqueue_active) = {
let before_membership = Self::flow_membership(&flow.fairness);
let mut enqueue_active = false;
flow.fairness.consecutive_stalls =
flow.fairness.consecutive_stalls.saturating_add(1);
flow.fairness.scheduler_state = FlowSchedulerState::Backpressured;
flow.fairness.pressure_class = FlowPressureClass::Backpressured;
let state = self.pressure.state();
let should_shed_frame = matches!(state, PressureState::Saturated)
|| (matches!(state, PressureState::Shedding)
&& flow.fairness.standing_state == StandingQueueState::Standing
&& flow.fairness.consecutive_stalls
>= self.config.max_consecutive_stalls_before_shed);
if should_shed_frame {
self.shed_drops = self.shed_drops.saturating_add(1);
self.fairness_penalties = self.fairness_penalties.saturating_add(1);
} else {
let frame_bytes = candidate.frame.queued_bytes();
flow.queue.push_front(candidate.frame);
flow.fairness.pending_bytes =
flow.fairness.pending_bytes.saturating_add(frame_bytes);
flow.fairness.queue_started_at =
flow.queue.front().map(|front| front.enqueued_at);
self.total_queued_bytes = self.total_queued_bytes.saturating_add(frame_bytes);
self.bucket_queued_bytes[flow.fairness.bucket_id] = self.bucket_queued_bytes
[flow.fairness.bucket_id]
.saturating_add(frame_bytes);
if !flow.fairness.in_active_ring {
flow.fairness.in_active_ring = true;
self.active_ring.push_back(conn_id);
if self.config.backpressure_enabled {
flow.fairness.consecutive_stalls =
flow.fairness.consecutive_stalls.saturating_add(1);
flow.fairness.scheduler_state = FlowSchedulerState::Backpressured;
flow.fairness.pressure_class = FlowPressureClass::Backpressured;
}
}
if flow.fairness.consecutive_stalls
>= self.config.max_consecutive_stalls_before_close
&& self.pressure.state() == PressureState::Saturated
{
let should_shed_frame = self.config.backpressure_enabled
&& (matches!(state, PressureState::Saturated)
|| (matches!(state, PressureState::Shedding)
&& flow.fairness.standing_state == StandingQueueState::Standing
&& flow.fairness.consecutive_stalls
>= self.config.max_consecutive_stalls_before_shed));
if should_shed_frame {
self.shed_drops = self.shed_drops.saturating_add(1);
self.fairness_penalties = self.fairness_penalties.saturating_add(1);
} else {
let frame_bytes = candidate.frame.queued_bytes();
flow.queue.push_front(candidate.frame);
flow.fairness.pending_bytes =
flow.fairness.pending_bytes.saturating_add(frame_bytes);
flow.fairness.queue_started_at =
flow.queue.front().map(|front| front.enqueued_at);
self.total_queued_bytes =
self.total_queued_bytes.saturating_add(frame_bytes);
self.bucket_queued_bytes[flow.fairness.bucket_id] = self
.bucket_queued_bytes[flow.fairness.bucket_id]
.saturating_add(frame_bytes);
if !flow.fairness.in_active_ring {
flow.fairness.in_active_ring = true;
enqueue_active = true;
}
}
Self::classify_flow(&self.config, state, now, &mut flow.fairness);
let after_membership = Self::flow_membership(&flow.fairness);
let should_close_flow = self.config.backpressure_enabled
&& flow.fairness.consecutive_stalls
>= self.config.max_consecutive_stalls_before_close
&& self.pressure.state() == PressureState::Saturated;
(
before_membership,
after_membership,
should_close_flow,
enqueue_active,
)
};
if enqueue_active {
self.enqueue_active_conn(conn_id);
}
self.apply_flow_membership_delta(before_membership, after_membership);
if should_close_flow {
self.remove_flow(conn_id);
self.evaluate_pressure(now, true);
return DispatchAction::CloseFlow;
@@ -426,6 +522,16 @@ impl WorkerFairnessState {
let Some(entry) = self.flows.remove(&conn_id) else {
return;
};
self.active_ring_members.remove(&conn_id);
self.active_ring
.retain(|queued_conn_id| *queued_conn_id != conn_id);
let (was_standing, was_backpressured) = Self::flow_membership(&entry.fairness);
if was_standing {
self.standing_flow_count = self.standing_flow_count.saturating_sub(1);
}
if was_backpressured {
self.backpressured_flow_count = self.backpressured_flow_count.saturating_sub(1);
}
self.bucket_active_flows[entry.fairness.bucket_id] =
self.bucket_active_flows[entry.fairness.bucket_id].saturating_sub(1);
@@ -440,27 +546,6 @@ impl WorkerFairnessState {
}
fn evaluate_pressure(&mut self, now: Instant, force: bool) {
let mut standing = 0usize;
let mut backpressured = 0usize;
for flow in self.flows.values_mut() {
Self::classify_flow(&self.config, self.pressure.state(), now, &mut flow.fairness);
if flow.fairness.standing_state == StandingQueueState::Standing {
standing = standing.saturating_add(1);
}
if matches!(
flow.fairness.scheduler_state,
FlowSchedulerState::Backpressured
| FlowSchedulerState::Penalized
| FlowSchedulerState::SheddingCandidate
) {
backpressured = backpressured.saturating_add(1);
}
}
self.standing_flow_count = standing;
self.backpressured_flow_count = backpressured;
let _ = self.pressure.maybe_evaluate(
now,
&self.config.pressure,
@@ -468,8 +553,8 @@ impl WorkerFairnessState {
PressureSignals {
active_flows: self.flows.len(),
total_queued_bytes: self.total_queued_bytes,
standing_flows: standing,
backpressured_flows: backpressured,
standing_flows: self.standing_flow_count,
backpressured_flows: self.backpressured_flow_count,
},
force,
);
@@ -481,12 +566,39 @@ impl WorkerFairnessState {
now: Instant,
fairness: &mut FlowFairnessState,
) {
if fairness.pending_bytes == 0 {
fairness.pressure_class = FlowPressureClass::Healthy;
fairness.standing_state = StandingQueueState::Transient;
fairness.scheduler_state = FlowSchedulerState::Idle;
let (pressure_class, standing_state, scheduler_state, standing) =
Self::derive_flow_classification(config, pressure_state, now, fairness);
fairness.pressure_class = pressure_class;
fairness.standing_state = standing_state;
fairness.scheduler_state = scheduler_state;
if scheduler_state == FlowSchedulerState::Idle {
fairness.deficit_bytes = 0;
}
if standing {
fairness.penalty_score = fairness.penalty_score.saturating_add(1);
} else {
fairness.penalty_score = fairness.penalty_score.saturating_sub(1);
return;
}
}
fn derive_flow_classification(
config: &WorkerFairnessConfig,
pressure_state: PressureState,
now: Instant,
fairness: &FlowFairnessState,
) -> (
FlowPressureClass,
StandingQueueState,
FlowSchedulerState,
bool,
) {
if fairness.pending_bytes == 0 {
return (
FlowPressureClass::Healthy,
StandingQueueState::Transient,
FlowSchedulerState::Idle,
false,
);
}
let queue_age = fairness
@@ -503,29 +615,165 @@ impl WorkerFairnessState {
&& (fairness.consecutive_stalls >= config.standing_stall_threshold || drain_stalled);
if standing {
fairness.standing_state = StandingQueueState::Standing;
fairness.pressure_class = FlowPressureClass::Standing;
fairness.penalty_score = fairness.penalty_score.saturating_add(1);
fairness.scheduler_state = if pressure_state >= PressureState::Shedding {
let scheduler_state = if pressure_state >= PressureState::Shedding {
FlowSchedulerState::SheddingCandidate
} else {
FlowSchedulerState::Penalized
};
return;
return (
FlowPressureClass::Standing,
StandingQueueState::Standing,
scheduler_state,
true,
);
}
fairness.standing_state = StandingQueueState::Transient;
if fairness.consecutive_stalls > 0 {
fairness.pressure_class = FlowPressureClass::Backpressured;
fairness.scheduler_state = FlowSchedulerState::Backpressured;
} else if fairness.pending_bytes >= config.standing_queue_min_backlog_bytes {
fairness.pressure_class = FlowPressureClass::Bursty;
fairness.scheduler_state = FlowSchedulerState::Active;
} else {
fairness.pressure_class = FlowPressureClass::Healthy;
fairness.scheduler_state = FlowSchedulerState::Active;
return (
FlowPressureClass::Backpressured,
StandingQueueState::Transient,
FlowSchedulerState::Backpressured,
false,
);
}
fairness.penalty_score = fairness.penalty_score.saturating_sub(1);
if fairness.pending_bytes >= config.standing_queue_min_backlog_bytes {
return (
FlowPressureClass::Bursty,
StandingQueueState::Transient,
FlowSchedulerState::Active,
false,
);
}
(
FlowPressureClass::Healthy,
StandingQueueState::Transient,
FlowSchedulerState::Active,
false,
)
}
#[inline]
fn flow_membership(fairness: &FlowFairnessState) -> (bool, bool) {
(
fairness.standing_state == StandingQueueState::Standing,
Self::scheduler_state_is_backpressured(fairness.scheduler_state),
)
}
#[inline]
fn scheduler_state_is_backpressured(state: FlowSchedulerState) -> bool {
matches!(
state,
FlowSchedulerState::Backpressured
| FlowSchedulerState::Penalized
| FlowSchedulerState::SheddingCandidate
)
}
fn apply_flow_membership_delta(
&mut self,
before_membership: (bool, bool),
after_membership: (bool, bool),
) {
if before_membership.0 != after_membership.0 {
if after_membership.0 {
self.standing_flow_count = self.standing_flow_count.saturating_add(1);
} else {
self.standing_flow_count = self.standing_flow_count.saturating_sub(1);
}
}
if before_membership.1 != after_membership.1 {
if after_membership.1 {
self.backpressured_flow_count = self.backpressured_flow_count.saturating_add(1);
} else {
self.backpressured_flow_count = self.backpressured_flow_count.saturating_sub(1);
}
}
}
#[inline]
fn clamp_deficit_bytes(config: &WorkerFairnessConfig, fairness: &mut FlowFairnessState) {
let max_deficit = config.max_flow_queued_bytes.min(i64::MAX as u64) as i64;
fairness.deficit_bytes = fairness.deficit_bytes.clamp(0, max_deficit);
}
#[inline]
fn enqueue_active_conn(&mut self, conn_id: u64) {
if self.active_ring_members.insert(conn_id) {
self.active_ring.push_back(conn_id);
}
}
#[inline]
fn weight_for_flags(config: &WorkerFairnessConfig, flags: u32) -> u8 {
if (flags & RPC_FLAG_QUICKACK) != 0 {
return config.quickack_flow_weight.max(1);
}
config.default_flow_weight.max(1)
}
#[cfg(test)]
pub(crate) fn debug_recompute_flow_counters(&self, now: Instant) -> (usize, usize) {
let pressure_state = self.pressure.state();
let mut standing = 0usize;
let mut backpressured = 0usize;
for flow in self.flows.values() {
let (_, standing_state, scheduler_state, _) =
Self::derive_flow_classification(&self.config, pressure_state, now, &flow.fairness);
if standing_state == StandingQueueState::Standing {
standing = standing.saturating_add(1);
}
if Self::scheduler_state_is_backpressured(scheduler_state) {
backpressured = backpressured.saturating_add(1);
}
}
(standing, backpressured)
}
#[cfg(test)]
pub(crate) fn debug_check_active_ring_consistency(&self) -> bool {
if self.active_ring.len() != self.active_ring_members.len() {
return false;
}
let mut seen = HashSet::with_capacity(self.active_ring.len());
for conn_id in self.active_ring.iter().copied() {
if !seen.insert(conn_id) {
return false;
}
if !self.active_ring_members.contains(&conn_id) {
return false;
}
let Some(flow) = self.flows.get(&conn_id) else {
return false;
};
if !flow.fairness.in_active_ring || flow.queue.is_empty() {
return false;
}
}
for (conn_id, flow) in self.flows.iter() {
let in_ring = self.active_ring_members.contains(conn_id);
if flow.fairness.in_active_ring != in_ring {
return false;
}
if in_ring && flow.queue.is_empty() {
return false;
}
}
true
}
#[cfg(test)]
pub(crate) fn debug_max_deficit_bytes(&self) -> i64 {
self.flows
.values()
.map(|entry| entry.fairness.deficit_bytes)
.max()
.unwrap_or(0)
}
fn effective_quantum_bytes(
@@ -542,12 +790,14 @@ impl WorkerFairnessState {
return config.penalized_quantum_bytes.max(1);
}
match pressure_state {
let base_quantum = match pressure_state {
PressureState::Normal => config.base_quantum_bytes.max(1),
PressureState::Pressured => config.pressured_quantum_bytes.max(1),
PressureState::Shedding => config.pressured_quantum_bytes.max(1),
PressureState::Saturated => config.penalized_quantum_bytes.max(1),
}
};
let weighted_quantum = base_quantum.saturating_mul(fairness.weight_quanta.max(1) as u32);
weighted_quantum.max(1)
}
fn bucket_for(&self, conn_id: u64) -> usize {

View File

@@ -1794,6 +1794,8 @@ mod tests {
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -46,6 +46,7 @@ mod send_adversarial_tests;
mod wire;
use bytes::Bytes;
use tokio::sync::OwnedSemaphorePermit;
#[allow(unused_imports)]
pub use config_updater::{
@@ -68,9 +69,32 @@ pub use secret::{fetch_proxy_secret, fetch_proxy_secret_with_upstream};
pub(crate) use selftest::{bnd_snapshot, timeskew_snapshot, upstream_bnd_snapshots};
pub use wire::proto_flags_for_tag;
/// Holds D2C queued-byte capacity until a routed payload is consumed or dropped.
pub struct RouteBytePermit {
_permit: OwnedSemaphorePermit,
}
impl std::fmt::Debug for RouteBytePermit {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RouteBytePermit").finish_non_exhaustive()
}
}
impl RouteBytePermit {
pub(crate) fn new(permit: OwnedSemaphorePermit) -> Self {
Self { _permit: permit }
}
}
/// Response routed from middle proxy readers to client relay tasks.
#[derive(Debug)]
pub enum MeResponse {
Data { flags: u32, data: Bytes },
/// Downstream payload with its queued-byte reservation.
Data {
flags: u32,
data: Bytes,
route_permit: Option<RouteBytePermit>,
},
Ack(u32),
Close,
}

View File

@@ -396,6 +396,8 @@ pub(super) struct WriterSelectionPolicyCore {
pub(super) struct TransportPolicyCore {
pub(super) me_socks_kdf_policy: AtomicU8,
pub(super) me_route_backpressure_enabled: Arc<AtomicBool>,
pub(super) me_route_fairshare_enabled: Arc<AtomicBool>,
pub(super) me_reader_route_data_wait_ms: Arc<AtomicU64>,
}
@@ -548,6 +550,8 @@ impl MePool {
me_socks_kdf_policy: MeSocksKdfPolicy,
me_writer_cmd_channel_capacity: usize,
me_route_channel_capacity: usize,
me_route_backpressure_enabled: bool,
me_route_fairshare_enabled: bool,
me_route_backpressure_base_timeout_ms: u64,
me_route_backpressure_high_timeout_ms: u64,
me_route_backpressure_high_watermark_pct: u8,
@@ -614,13 +618,9 @@ impl MePool {
me_route_hybrid_max_wait: Duration::from_millis(
me_route_hybrid_max_wait_ms.max(50),
),
me_route_blocking_send_timeout: if me_route_blocking_send_timeout_ms == 0 {
None
} else {
Some(Duration::from_millis(
me_route_blocking_send_timeout_ms.min(5_000),
))
},
me_route_blocking_send_timeout: Some(Duration::from_millis(
me_route_blocking_send_timeout_ms.clamp(1, 5_000),
)),
me_route_last_success_epoch_ms: AtomicU64::new(0),
me_route_hybrid_timeout_warn_epoch_ms: AtomicU64::new(0),
me_async_recovery_last_trigger_epoch_ms: AtomicU64::new(0),
@@ -783,6 +783,10 @@ impl MePool {
}),
transport_policy: Arc::new(TransportPolicyCore {
me_socks_kdf_policy: AtomicU8::new(me_socks_kdf_policy.as_u8()),
me_route_backpressure_enabled: Arc::new(AtomicBool::new(
me_route_backpressure_enabled,
)),
me_route_fairshare_enabled: Arc::new(AtomicBool::new(me_route_fairshare_enabled)),
me_reader_route_data_wait_ms: Arc::new(AtomicU64::new(
me_reader_route_data_wait_ms,
)),
@@ -1245,6 +1249,8 @@ impl MePool {
pub fn update_runtime_transport_policy(
&self,
socks_kdf_policy: MeSocksKdfPolicy,
route_backpressure_enabled: bool,
route_fairshare_enabled: bool,
route_backpressure_base_timeout_ms: u64,
route_backpressure_high_timeout_ms: u64,
route_backpressure_high_watermark_pct: u8,
@@ -1253,6 +1259,12 @@ impl MePool {
self.transport_policy
.me_socks_kdf_policy
.store(socks_kdf_policy.as_u8(), Ordering::Relaxed);
self.transport_policy
.me_route_backpressure_enabled
.store(route_backpressure_enabled, Ordering::Relaxed);
self.transport_policy
.me_route_fairshare_enabled
.store(route_fairshare_enabled, Ordering::Relaxed);
self.transport_policy
.me_reader_route_data_wait_ms
.store(reader_route_data_wait_ms, Ordering::Relaxed);

View File

@@ -365,7 +365,10 @@ impl MePool {
}
}
pub async fn zero_downtime_reinit_after_map_change(self: &Arc<Self>, rng: &SecureRandom) {
pub async fn zero_downtime_reinit_after_map_change(
self: &Arc<Self>,
rng: &SecureRandom,
) -> bool {
let desired_by_dc = self.desired_dc_endpoints().await;
let now_epoch_secs = Self::now_epoch_secs();
let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs);
@@ -380,7 +383,7 @@ impl MePool {
MeDrainGateReason::CoverageQuorum
};
self.set_last_drain_gate(false, false, reason, now_epoch_secs);
return;
return false;
}
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
@@ -490,7 +493,7 @@ impl MePool {
missing_dc = ?missing_dc,
"ME reinit coverage below threshold; keeping stale writers"
);
return;
return false;
}
if hardswap {
@@ -520,7 +523,7 @@ impl MePool {
missing_dc = ?fresh_missing_dc,
"ME hardswap pending: fresh generation DC coverage incomplete"
);
return;
return false;
}
}
@@ -567,7 +570,7 @@ impl MePool {
self.clear_pending_hardswap_state();
}
debug!("ME reinit cycle completed with no stale writers");
return;
return true;
}
let drain_timeout = self.force_close_timeout();
@@ -606,10 +609,11 @@ impl MePool {
if hardswap {
self.clear_pending_hardswap_state();
}
true
}
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) {
self.zero_downtime_reinit_after_map_change(rng).await;
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) -> bool {
self.zero_downtime_reinit_after_map_change(rng).await
}
}

View File

@@ -436,6 +436,9 @@ impl MePool {
let cancel_signal = cancel.clone();
let cancel_select = cancel.clone();
let cancel_cleanup = cancel.clone();
let route_backpressure_enabled =
self.transport_policy.me_route_backpressure_enabled.clone();
let route_fairshare_enabled = self.transport_policy.me_route_fairshare_enabled.clone();
let reader_route_data_wait_ms = self.transport_policy.me_reader_route_data_wait_ms.clone();
tokio::spawn(async move {
@@ -458,6 +461,8 @@ impl MePool {
writer_id,
degraded,
rtt_ema_ms_x10,
route_backpressure_enabled,
route_fairshare_enabled,
reader_route_data_wait_ms,
cancel_reader,
) => WriterLifecycleExit::Reader(reader_res),

View File

@@ -4,7 +4,7 @@ use std::collections::HashMap;
use std::io::ErrorKind;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
use std::time::Instant;
use std::time::{Duration, Instant};
use bytes::{Bytes, BytesMut};
use tokio::io::AsyncReadExt;
@@ -21,8 +21,8 @@ use crate::stats::Stats;
use super::codec::{RpcChecksumMode, WriterCommand, rpc_crc};
use super::fairness::{
AdmissionDecision, DispatchAction, DispatchFeedback, SchedulerDecision, WorkerFairnessConfig,
WorkerFairnessSnapshot, WorkerFairnessState,
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState,
};
use super::registry::RouteResult;
use super::{ConnRegistry, MeResponse};
@@ -45,10 +45,30 @@ fn is_data_route_queue_full(result: RouteResult) -> bool {
)
}
fn should_close_on_queue_full_streak(streak: u8) -> bool {
fn should_close_on_queue_full_streak_with_policy(
streak: u8,
pressure_state: PressureState,
backpressure_enabled: bool,
) -> bool {
if !backpressure_enabled {
return false;
}
if pressure_state < PressureState::Shedding {
return false;
}
streak >= DATA_ROUTE_QUEUE_FULL_STARVATION_THRESHOLD
}
fn should_schedule_fairness_retry(snapshot: &WorkerFairnessSnapshot) -> bool {
snapshot.total_queued_bytes > 0
}
fn fairness_retry_delay(route_wait_ms: u64) -> Duration {
Duration::from_millis(route_wait_ms.max(1))
}
async fn route_data_with_retry(
reg: &ConnRegistry,
conn_id: u64,
@@ -64,6 +84,7 @@ async fn route_data_with_retry(
MeResponse::Data {
flags,
data: data.clone(),
route_permit: None,
},
timeout_ms,
)
@@ -148,6 +169,7 @@ async fn drain_fairness_scheduler(
reg: &ConnRegistry,
tx: &mpsc::Sender<WriterCommand>,
data_route_queue_full_streak: &mut HashMap<u64, u8>,
backpressure_enabled: bool,
route_wait_ms: u64,
stats: &Stats,
) {
@@ -157,7 +179,7 @@ async fn drain_fairness_scheduler(
break;
};
let cid = candidate.frame.conn_id;
let _pressure_state = candidate.pressure_state;
let pressure_state = candidate.pressure_state;
let _flow_class = candidate.flow_class;
let routed = route_data_with_retry(
reg,
@@ -176,7 +198,11 @@ async fn drain_fairness_scheduler(
if is_data_route_queue_full(routed) {
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
if should_close_on_queue_full_streak(*streak) {
if should_close_on_queue_full_streak_with_policy(
*streak,
pressure_state,
backpressure_enabled,
) {
fairness.remove_flow(cid);
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
@@ -208,6 +234,8 @@ pub(crate) async fn reader_loop(
writer_id: u64,
degraded: Arc<AtomicBool>,
writer_rtt_ema_ms_x10: Arc<AtomicU32>,
route_backpressure_enabled: Arc<AtomicBool>,
route_fairshare_enabled: Arc<AtomicBool>,
reader_route_data_wait_ms: Arc<AtomicU64>,
cancel: CancellationToken,
) -> Result<()> {
@@ -224,17 +252,46 @@ pub(crate) async fn reader_loop(
max_flow_queued_bytes: (reg.route_channel_capacity() as u64)
.saturating_mul(2 * 1024)
.clamp(64 * 1024, 2 * 1024 * 1024),
backpressure_enabled: route_backpressure_enabled.load(Ordering::Relaxed),
..WorkerFairnessConfig::default()
},
Instant::now(),
);
let mut fairness_snapshot = fairness.snapshot();
loop {
let backpressure_enabled = route_backpressure_enabled.load(Ordering::Relaxed);
let fairshare_enabled = route_fairshare_enabled.load(Ordering::Relaxed);
fairness.set_backpressure_enabled(backpressure_enabled);
let fairness_has_backlog = should_schedule_fairness_retry(&fairness_snapshot);
let mut tmp = [0u8; 65_536];
let backlog_retry_enabled = fairness_has_backlog;
let backlog_retry_delay =
fairness_retry_delay(reader_route_data_wait_ms.load(Ordering::Relaxed));
let mut retry_only = false;
let n = tokio::select! {
res = rd.read(&mut tmp) => res.map_err(ProxyError::Io)?,
_ = tokio::time::sleep(backlog_retry_delay), if backlog_retry_enabled => {
retry_only = true;
0usize
},
_ = cancel.cancelled() => return Ok(()),
};
if retry_only {
let route_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed);
drain_fairness_scheduler(
&mut fairness,
reg.as_ref(),
&tx,
&mut data_route_queue_full_streak,
backpressure_enabled,
route_wait_ms,
stats.as_ref(),
)
.await;
let current_snapshot = fairness.snapshot();
apply_fairness_metrics_delta(stats.as_ref(), &mut fairness_snapshot, current_snapshot);
continue;
}
if n == 0 {
stats.increment_me_reader_eof_total();
return Err(ProxyError::Io(std::io::Error::new(
@@ -311,23 +368,56 @@ pub(crate) async fn reader_loop(
let data = body.slice(12..);
trace!(cid, flags, len = data.len(), "RPC_PROXY_ANS");
let admission = fairness.enqueue_data(cid, flags, data, Instant::now());
if !matches!(admission, AdmissionDecision::Admit) {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_high();
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
if should_close_on_queue_full_streak(*streak)
|| matches!(
admission,
AdmissionDecision::RejectSaturated
| AdmissionDecision::RejectStandingFlow
)
{
if fairshare_enabled {
let admission = fairness.enqueue_data(cid, flags, data, Instant::now());
if !matches!(admission, AdmissionDecision::Admit) {
stats.increment_me_route_drop_queue_full();
stats.increment_me_route_drop_queue_full_high();
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
let pressure_state = fairness.pressure_state();
if should_close_on_queue_full_streak_with_policy(
*streak,
pressure_state,
backpressure_enabled,
) || (backpressure_enabled
&& matches!(admission, AdmissionDecision::RejectSaturated))
{
fairness.remove_flow(cid);
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
}
}
} else {
let route_wait_ms = reader_route_data_wait_ms.load(Ordering::Relaxed);
let routed =
route_data_with_retry(reg.as_ref(), cid, flags, data, route_wait_ms).await;
if matches!(routed, RouteResult::Routed) {
data_route_queue_full_streak.remove(&cid);
continue;
}
report_route_drop(routed, stats.as_ref());
if should_close_on_route_result_for_data(routed) {
fairness.remove_flow(cid);
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
continue;
}
if is_data_route_queue_full(routed) {
let streak = data_route_queue_full_streak.entry(cid).or_insert(0);
*streak = streak.saturating_add(1);
if should_close_on_queue_full_streak_with_policy(
*streak,
PressureState::Shedding,
backpressure_enabled,
) {
fairness.remove_flow(cid);
data_route_queue_full_streak.remove(&cid);
reg.unregister(cid).await;
send_close_conn(&tx, cid).await;
}
}
}
} else if pt == RPC_SIMPLE_ACK_U32 && body.len() >= 12 {
@@ -433,6 +523,7 @@ pub(crate) async fn reader_loop(
reg.as_ref(),
&tx,
&mut data_route_queue_full_streak,
backpressure_enabled,
route_wait_ms,
stats.as_ref(),
)
@@ -445,14 +536,18 @@ pub(crate) async fn reader_loop(
#[cfg(test)]
mod tests {
use std::time::Duration;
use bytes::Bytes;
use super::PressureState;
use crate::transport::middle_proxy::ConnRegistry;
use super::{
MeResponse, RouteResult, is_data_route_queue_full, route_data_with_retry,
should_close_on_queue_full_streak, should_close_on_route_result_for_ack,
should_close_on_route_result_for_data,
MeResponse, RouteResult, WorkerFairnessSnapshot, fairness_retry_delay,
is_data_route_queue_full, route_data_with_retry,
should_close_on_queue_full_streak_with_policy, should_close_on_route_result_for_ack,
should_close_on_route_result_for_data, should_schedule_fairness_retry,
};
#[test]
@@ -475,10 +570,51 @@ mod tests {
assert!(is_data_route_queue_full(RouteResult::QueueFullBase));
assert!(is_data_route_queue_full(RouteResult::QueueFullHigh));
assert!(!is_data_route_queue_full(RouteResult::NoConn));
assert!(!should_close_on_queue_full_streak(1));
assert!(!should_close_on_queue_full_streak(2));
assert!(should_close_on_queue_full_streak(3));
assert!(should_close_on_queue_full_streak(u8::MAX));
assert!(!should_close_on_queue_full_streak_with_policy(
1,
PressureState::Normal,
true
));
assert!(!should_close_on_queue_full_streak_with_policy(
2,
PressureState::Pressured,
true
));
assert!(!should_close_on_queue_full_streak_with_policy(
3,
PressureState::Pressured,
true
));
assert!(should_close_on_queue_full_streak_with_policy(
3,
PressureState::Shedding,
true
));
assert!(should_close_on_queue_full_streak_with_policy(
u8::MAX,
PressureState::Saturated,
true
));
assert!(!should_close_on_queue_full_streak_with_policy(
u8::MAX,
PressureState::Saturated,
false
));
}
#[test]
fn fairness_retry_is_scheduled_only_when_queue_has_pending_bytes() {
let mut snapshot = WorkerFairnessSnapshot::default();
assert!(!should_schedule_fairness_retry(&snapshot));
snapshot.total_queued_bytes = 1;
assert!(should_schedule_fairness_retry(&snapshot));
}
#[test]
fn fairness_retry_delay_never_drops_below_one_millisecond() {
assert_eq!(fairness_retry_delay(0), Duration::from_millis(1));
assert_eq!(fairness_retry_delay(2), Duration::from_millis(2));
}
#[test]
@@ -504,7 +640,7 @@ mod tests {
let routed = route_data_with_retry(&reg, conn_id, 0, Bytes::from_static(b"a"), 20).await;
assert!(matches!(routed, RouteResult::Routed));
match rx.recv().await {
Some(MeResponse::Data { flags, data }) => {
Some(MeResponse::Data { flags, data, .. }) => {
assert_eq!(flags, 0);
assert_eq!(data, Bytes::from_static(b"a"));
}

View File

@@ -1,18 +1,22 @@
use std::collections::{HashMap, HashSet};
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use dashmap::DashMap;
use tokio::sync::mpsc::error::TrySendError;
use tokio::sync::{Mutex, mpsc};
use tokio::sync::{Mutex, Semaphore, mpsc};
use super::MeResponse;
use super::codec::WriterCommand;
use super::{MeResponse, RouteBytePermit};
const ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS: u64 = 25;
const ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS: u64 = 120;
const ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT: u8 = 80;
const ROUTE_QUEUED_BYTE_PERMIT_UNIT: usize = 16 * 1024;
const ROUTE_QUEUED_PERMITS_PER_SLOT: usize = 4;
const ROUTE_QUEUED_MAX_FRAME_PERMITS: usize = 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RouteResult {
@@ -53,6 +57,7 @@ pub(super) struct WriterActivitySnapshot {
struct RoutingTable {
map: DashMap<u64, mpsc::Sender<MeResponse>>,
byte_budget: DashMap<u64, Arc<Semaphore>>,
}
struct WriterTable {
@@ -105,6 +110,7 @@ pub struct ConnRegistry {
route_backpressure_base_timeout_ms: AtomicU64,
route_backpressure_high_timeout_ms: AtomicU64,
route_backpressure_high_watermark_pct: AtomicU8,
route_byte_permits_per_conn: usize,
}
impl ConnRegistry {
@@ -116,10 +122,23 @@ impl ConnRegistry {
}
pub fn with_route_channel_capacity(route_channel_capacity: usize) -> Self {
let route_channel_capacity = route_channel_capacity.max(1);
Self::with_route_limits(
route_channel_capacity,
Self::route_byte_permit_budget(route_channel_capacity),
)
}
fn with_route_limits(
route_channel_capacity: usize,
route_byte_permits_per_conn: usize,
) -> Self {
let start = rand::random::<u64>() | 1;
let route_channel_capacity = route_channel_capacity.max(1);
Self {
routing: RoutingTable {
map: DashMap::new(),
byte_budget: DashMap::new(),
},
writers: WriterTable {
map: DashMap::new(),
@@ -131,15 +150,30 @@ impl ConnRegistry {
inner: Mutex::new(BindingInner::new()),
},
next_id: AtomicU64::new(start),
route_channel_capacity: route_channel_capacity.max(1),
route_channel_capacity,
route_backpressure_base_timeout_ms: AtomicU64::new(ROUTE_BACKPRESSURE_BASE_TIMEOUT_MS),
route_backpressure_high_timeout_ms: AtomicU64::new(ROUTE_BACKPRESSURE_HIGH_TIMEOUT_MS),
route_backpressure_high_watermark_pct: AtomicU8::new(
ROUTE_BACKPRESSURE_HIGH_WATERMARK_PCT,
),
route_byte_permits_per_conn: route_byte_permits_per_conn.max(1),
}
}
fn route_data_permits(data_len: usize) -> u32 {
data_len
.max(1)
.div_ceil(ROUTE_QUEUED_BYTE_PERMIT_UNIT)
.min(u32::MAX as usize) as u32
}
fn route_byte_permit_budget(route_channel_capacity: usize) -> usize {
route_channel_capacity
.saturating_mul(ROUTE_QUEUED_PERMITS_PER_SLOT)
.max(ROUTE_QUEUED_MAX_FRAME_PERMITS)
.max(1)
}
pub fn route_channel_capacity(&self) -> usize {
self.route_channel_capacity
}
@@ -149,6 +183,14 @@ impl ConnRegistry {
Self::with_route_channel_capacity(4096)
}
#[cfg(test)]
fn with_route_byte_permits_for_tests(
route_channel_capacity: usize,
route_byte_permits_per_conn: usize,
) -> Self {
Self::with_route_limits(route_channel_capacity, route_byte_permits_per_conn)
}
pub fn update_route_backpressure_policy(
&self,
base_timeout_ms: u64,
@@ -170,6 +212,10 @@ impl ConnRegistry {
let id = self.next_id.fetch_add(1, Ordering::Relaxed);
let (tx, rx) = mpsc::channel(self.route_channel_capacity);
self.routing.map.insert(id, tx);
self.routing.byte_budget.insert(
id,
Arc::new(Semaphore::new(self.route_byte_permits_per_conn)),
);
(id, rx)
}
@@ -186,6 +232,7 @@ impl ConnRegistry {
/// Unregister connection, returning associated writer_id if any.
pub async fn unregister(&self, id: u64) -> Option<u64> {
self.routing.map.remove(&id);
self.routing.byte_budget.remove(&id);
self.hot_binding.map.remove(&id);
let mut binding = self.binding.inner.lock().await;
binding.meta.remove(&id);
@@ -206,6 +253,64 @@ impl ConnRegistry {
None
}
async fn attach_route_byte_permit(
&self,
id: u64,
resp: MeResponse,
timeout_ms: Option<u64>,
) -> std::result::Result<MeResponse, RouteResult> {
let MeResponse::Data {
flags,
data,
route_permit,
} = resp
else {
return Ok(resp);
};
if route_permit.is_some() {
return Ok(MeResponse::Data {
flags,
data,
route_permit,
});
}
let Some(semaphore) = self
.routing
.byte_budget
.get(&id)
.map(|entry| entry.value().clone())
else {
return Err(RouteResult::NoConn);
};
let permits = Self::route_data_permits(data.len());
let permit = match timeout_ms {
Some(0) => semaphore
.try_acquire_many_owned(permits)
.map_err(|_| RouteResult::QueueFullHigh)?,
Some(timeout_ms) => {
let acquire = semaphore.acquire_many_owned(permits);
match tokio::time::timeout(Duration::from_millis(timeout_ms.max(1)), acquire).await
{
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(RouteResult::ChannelClosed),
Err(_) => return Err(RouteResult::QueueFullHigh),
}
}
None => semaphore
.acquire_many_owned(permits)
.await
.map_err(|_| RouteResult::ChannelClosed)?,
};
Ok(MeResponse::Data {
flags,
data,
route_permit: Some(RouteBytePermit::new(permit)),
})
}
#[allow(dead_code)]
pub async fn route(&self, id: u64, resp: MeResponse) -> RouteResult {
let tx = self.routing.map.get(&id).map(|entry| entry.value().clone());
@@ -214,15 +319,23 @@ impl ConnRegistry {
return RouteResult::NoConn;
};
let base_timeout_ms = self
.route_backpressure_base_timeout_ms
.load(Ordering::Relaxed)
.max(1);
let resp = match self
.attach_route_byte_permit(id, resp, Some(base_timeout_ms))
.await
{
Ok(resp) => resp,
Err(result) => return result,
};
match tx.try_send(resp) {
Ok(()) => RouteResult::Routed,
Err(TrySendError::Closed(_)) => RouteResult::ChannelClosed,
Err(TrySendError::Full(resp)) => {
// Absorb short bursts without dropping/closing the session immediately.
let base_timeout_ms = self
.route_backpressure_base_timeout_ms
.load(Ordering::Relaxed)
.max(1);
let high_timeout_ms = self
.route_backpressure_high_timeout_ms
.load(Ordering::Relaxed)
@@ -266,6 +379,10 @@ impl ConnRegistry {
let Some(tx) = tx else {
return RouteResult::NoConn;
};
let resp = match self.attach_route_byte_permit(id, resp, Some(0)).await {
Ok(resp) => resp,
Err(result) => return result,
};
match tx.try_send(resp) {
Ok(()) => RouteResult::Routed,
@@ -289,6 +406,13 @@ impl ConnRegistry {
let Some(tx) = tx else {
return RouteResult::NoConn;
};
let resp = match self
.attach_route_byte_permit(id, resp, Some(timeout_ms))
.await
{
Ok(resp) => resp,
Err(result) => return result,
};
match tx.try_send(resp) {
Ok(()) => RouteResult::Routed,
@@ -541,8 +665,10 @@ impl ConnRegistry {
mod tests {
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use super::ConnMeta;
use super::ConnRegistry;
use bytes::Bytes;
use super::{ConnMeta, ConnRegistry, RouteResult};
use crate::transport::middle_proxy::MeResponse;
#[tokio::test]
async fn writer_activity_snapshot_tracks_writer_and_dc_load() {
@@ -608,6 +734,55 @@ mod tests {
assert_eq!(snapshot.active_sessions_by_target_dc.get(&4), Some(&1));
}
#[tokio::test]
async fn route_data_is_bounded_by_byte_permits_before_channel_capacity() {
let registry = ConnRegistry::with_route_byte_permits_for_tests(4, 1);
let (conn_id, mut rx) = registry.register().await;
let routed = registry
.route_nowait(
conn_id,
MeResponse::Data {
flags: 0,
data: Bytes::from_static(&[0xAA]),
route_permit: None,
},
)
.await;
assert!(matches!(routed, RouteResult::Routed));
let blocked = registry
.route_nowait(
conn_id,
MeResponse::Data {
flags: 0,
data: Bytes::from_static(&[0xBB]),
route_permit: None,
},
)
.await;
assert!(
matches!(blocked, RouteResult::QueueFullHigh),
"byte budget must reject data before count capacity is exhausted"
);
drop(rx.recv().await);
let routed_after_drain = registry
.route_nowait(
conn_id,
MeResponse::Data {
flags: 0,
data: Bytes::from_static(&[0xCC]),
route_permit: None,
},
)
.await;
assert!(
matches!(routed_after_drain, RouteResult::Routed),
"receiving queued data must release byte permits"
);
}
#[tokio::test]
async fn bind_writer_rebinds_conn_atomically() {
let registry = ConnRegistry::new();

View File

@@ -47,6 +47,7 @@ pub async fn me_reinit_scheduler(
rng: Arc<SecureRandom>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
mut trigger_rx: mpsc::Receiver<MeReinitTrigger>,
me_ready_tx: watch::Sender<u64>,
) {
info!("ME reinit scheduler started");
loop {
@@ -90,15 +91,25 @@ pub async fn me_reinit_scheduler(
if cfg.general.me_reinit_singleflight {
debug!(reason, "ME reinit scheduled (single-flight)");
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
if pool.zero_downtime_reinit_periodic(rng.as_ref()).await {
me_ready_tx.send_modify(|version| {
*version = version.saturating_add(1);
});
}
} else {
debug!(reason, "ME reinit scheduled (concurrent mode)");
let pool_clone = pool.clone();
let rng_clone = rng.clone();
let me_ready_tx_clone = me_ready_tx.clone();
tokio::spawn(async move {
pool_clone
if pool_clone
.zero_downtime_reinit_periodic(rng_clone.as_ref())
.await;
.await
{
me_ready_tx_clone.send_modify(|version| {
*version = version.saturating_add(1);
});
}
});
}
}

View File

@@ -37,20 +37,26 @@ pub(super) fn validate_proxy_secret_len(data_len: usize, max_len: usize) -> Resu
/// Fetch Telegram proxy-secret binary.
#[allow(dead_code)]
pub async fn fetch_proxy_secret(cache_path: Option<&str>, max_len: usize) -> Result<Vec<u8>> {
fetch_proxy_secret_with_upstream(cache_path, max_len, None).await
pub async fn fetch_proxy_secret(
cache_path: Option<&str>,
max_len: usize,
proxy_secret_url: Option<&str>,
) -> Result<Vec<u8>> {
fetch_proxy_secret_with_upstream(cache_path, max_len, proxy_secret_url, None).await
}
/// Fetch Telegram proxy-secret binary, optionally through upstream routing.
pub async fn fetch_proxy_secret_with_upstream(
cache_path: Option<&str>,
max_len: usize,
proxy_secret_url: Option<&str>,
upstream: Option<Arc<UpstreamManager>>,
) -> Result<Vec<u8>> {
let cache = cache_path.unwrap_or("proxy-secret");
// 1) Try fresh download first.
match download_proxy_secret_with_max_len_via_upstream(max_len, upstream).await {
match download_proxy_secret_with_max_len_via_upstream(max_len, upstream, proxy_secret_url).await
{
Ok(data) => {
if let Err(e) = tokio::fs::write(cache, &data).await {
warn!(error = %e, "Failed to cache proxy-secret (non-fatal)");
@@ -91,14 +97,19 @@ pub async fn fetch_proxy_secret_with_upstream(
#[allow(dead_code)]
pub async fn download_proxy_secret_with_max_len(max_len: usize) -> Result<Vec<u8>> {
download_proxy_secret_with_max_len_via_upstream(max_len, None).await
download_proxy_secret_with_max_len_via_upstream(max_len, None, None).await
}
pub async fn download_proxy_secret_with_max_len_via_upstream(
max_len: usize,
upstream: Option<Arc<UpstreamManager>>,
proxy_secret_url: Option<&str>,
) -> Result<Vec<u8>> {
let resp = https_get("https://core.telegram.org/getProxySecret", upstream).await?;
let resp = https_get(
proxy_secret_url.unwrap_or("https://core.telegram.org/getProxySecret"),
upstream,
)
.await?;
if !(200..=299).contains(&resp.status) {
return Err(ProxyError::Proxy(format!(

View File

@@ -2,6 +2,7 @@ use std::time::{Duration, Instant};
use bytes::Bytes;
use crate::protocol::constants::RPC_FLAG_QUICKACK;
use crate::transport::middle_proxy::fairness::{
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
WorkerFairnessConfig, WorkerFairnessState,
@@ -114,6 +115,62 @@ fn fairness_keeps_fast_flow_progress_under_slow_neighbor() {
assert!(snapshot.total_queued_bytes <= 64 * 1024);
}
#[test]
fn fairness_prioritizes_quickack_flow_when_weights_enabled() {
let mut now = Instant::now();
let mut fairness = WorkerFairnessState::new(
WorkerFairnessConfig {
max_total_queued_bytes: 256 * 1024,
max_flow_queued_bytes: 128 * 1024,
base_quantum_bytes: 8 * 1024,
pressured_quantum_bytes: 8 * 1024,
penalized_quantum_bytes: 8 * 1024,
default_flow_weight: 1,
quickack_flow_weight: 4,
..WorkerFairnessConfig::default()
},
now,
);
for _ in 0..8 {
assert_eq!(
fairness.enqueue_data(10, RPC_FLAG_QUICKACK, enqueue_payload(16 * 1024), now),
AdmissionDecision::Admit
);
assert_eq!(
fairness.enqueue_data(20, 0, enqueue_payload(16 * 1024), now),
AdmissionDecision::Admit
);
}
let mut quickack_dispatched = 0u64;
let mut bulk_dispatched = 0u64;
for _ in 0..64 {
now += Duration::from_millis(1);
let SchedulerDecision::Dispatch(candidate) = fairness.next_decision(now) else {
break;
};
if candidate.frame.conn_id == 10 {
quickack_dispatched = quickack_dispatched.saturating_add(1);
} else if candidate.frame.conn_id == 20 {
bulk_dispatched = bulk_dispatched.saturating_add(1);
}
let _ = fairness.apply_dispatch_feedback(
candidate.frame.conn_id,
candidate,
DispatchFeedback::Routed,
now,
);
}
assert!(
quickack_dispatched > bulk_dispatched,
"quickack flow must receive higher dispatch rate with larger weight"
);
}
#[test]
fn fairness_pressure_hysteresis_prevents_instant_flapping() {
let mut now = Instant::now();
@@ -128,7 +185,7 @@ fn fairness_pressure_hysteresis_prevents_instant_flapping() {
let mut fairness = WorkerFairnessState::new(cfg, now);
for _ in 0..4 {
for _ in 0..3 {
assert_eq!(
fairness.enqueue_data(9, 0, enqueue_payload(900), now),
AdmissionDecision::Admit
@@ -180,6 +237,12 @@ fn fairness_randomized_sequence_preserves_memory_bounds() {
}
let snapshot = fairness.snapshot();
let (standing_recomputed, backpressured_recomputed) =
fairness.debug_recompute_flow_counters(now);
assert!(snapshot.total_queued_bytes <= 32 * 1024);
assert_eq!(snapshot.standing_flows, standing_recomputed);
assert_eq!(snapshot.backpressured_flows, backpressured_recomputed);
assert!(fairness.debug_check_active_ring_consistency());
assert!(fairness.debug_max_deficit_bytes() <= 4 * 1024);
}
}

View File

@@ -104,6 +104,8 @@ async fn make_pool(
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -102,6 +102,8 @@ async fn make_pool(
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -97,6 +97,8 @@ async fn make_pool(me_pool_drain_threshold: u64) -> Arc<MePool> {
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -86,6 +86,8 @@ async fn make_pool() -> Arc<MePool> {
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -91,6 +91,8 @@ async fn make_pool() -> Arc<MePool> {
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -97,6 +97,8 @@ async fn make_pool() -> (Arc<MePool>, Arc<SecureRandom>) {
MeSocksKdfPolicy::default(),
general.me_writer_cmd_channel_capacity,
general.me_route_channel_capacity,
general.me_route_backpressure_enabled,
general.me_route_fairshare_enabled,
general.me_route_backpressure_base_timeout_ms,
general.me_route_backpressure_high_timeout_ms,
general.me_route_backpressure_high_watermark_pct,

View File

@@ -18,6 +18,9 @@ const PROXY_V1_MIN_LEN: usize = 6;
/// Minimum length for v2 header
const PROXY_V2_MIN_LEN: usize = 16;
/// Maximum accepted PROXY v2 address and TLV payload.
const PROXY_V2_MAX_ADDR_LEN: usize = 216;
/// Address families for v2
mod address_family {
pub const UNSPEC: u8 = 0x0;
@@ -169,6 +172,9 @@ async fn parse_v2<R: AsyncRead + Unpin>(
let family_protocol = header[13];
let addr_len = u16::from_be_bytes([header[14], header[15]]) as usize;
if addr_len > PROXY_V2_MAX_ADDR_LEN {
return Err(ProxyError::InvalidProxyProtocol);
}
// Read address data
let mut addr_data = vec![0u8; addr_len];

View File

@@ -279,6 +279,12 @@ pub struct UpstreamApiSummarySnapshot {
pub shadowsocks_total: usize,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct UpstreamApiHealthSummary {
pub configured_total: usize,
pub healthy_total: usize,
}
#[derive(Debug, Clone)]
pub struct UpstreamApiSnapshot {
pub summary: UpstreamApiSummarySnapshot,
@@ -444,6 +450,20 @@ impl UpstreamManager {
Some(UpstreamApiSnapshot { summary, upstreams })
}
pub async fn api_health_summary(&self) -> UpstreamApiHealthSummary {
let guard = self.upstreams.read().await;
let mut summary = UpstreamApiHealthSummary {
configured_total: guard.len(),
healthy_total: 0,
};
for upstream in guard.iter() {
if upstream.healthy {
summary.healthy_total += 1;
}
}
summary
}
fn describe_upstream(upstream_type: &UpstreamType) -> (UpstreamRouteKind, String) {
match upstream_type {
UpstreamType::Direct { .. } => (UpstreamRouteKind::Direct, "direct".to_string()),

File diff suppressed because it is too large Load Diff