Compare commits

..

69 Commits
3.4.8 ... flow

Author SHA1 Message Date
Alexey
9e877e45c9 Merge pull request #788 from amirotin/feature/users-quota-endpoint
Add GET /v1/users/quota endpoint
2026-05-17 11:06:06 +03:00
Mirotin Artem
0af64a4d0a Add GET /v1/users/quota endpoint 2026-05-15 16:25:56 +03:00
Alexey
f77e9b8881 Merge pull request #783 from astronaut808/feature/user-rate-limits-api
Expose user rate limits through the API
2026-05-14 18:20:04 +03:00
astronaut808
25ca64de1b Document Docker config layout for API mutations 2026-05-13 16:42:01 +05:00
astronaut808
8895947414 Expose user rate limits through the API 2026-05-13 16:35:40 +05:00
Alexey
7a284623d6 Update API.md 2026-05-10 15:38:27 +03:00
Alexey
3bd5637e47 ME + Admission + Cleanup Correctness: merge pull request #779 from telemt/flow
ME + Admission + Cleanup Correctness
2026-05-10 14:23:09 +03:00
Alexey
57b2aa0453 Rustfmt 2026-05-10 14:14:52 +03:00
Alexey
10c7cb2e0c Middle Relay Cancellation Errors 2026-05-10 14:12:15 +03:00
Alexey
900b574fb8 Harden ME Writer Cancellation paths 2026-05-10 14:09:10 +03:00
Alexey
beed6b4679 Middle Wait Deadlines + Tighten Session Release State 2026-05-10 13:58:02 +03:00
Alexey
eef2a38c75 Type Route Cutovers + Reduce IP Tracker cleanup pressure 2026-05-10 13:55:01 +03:00
Alexey
6cb72b3b6c Explicit Reasons of Session Fallback Cleanup + ME Close 2026-05-10 13:50:36 +03:00
Alexey
090b2ca636 Stats and Cleanup-proccess beyond Hot-path 2026-05-10 13:43:41 +03:00
Alexey
e10c070dc1 Observability + Cancellation for Middle Quota + Traffic Waits 2026-05-10 13:38:11 +03:00
Alexey
3f9ac87daf Bounded Rate Bursts + Cancel ME Waits 2026-05-10 13:33:54 +03:00
Alexey
36de807096 Merge branch 'flow' of https://github.com/telemt/telemt into flow 2026-05-10 13:31:11 +03:00
Alexey
844a912b38 Expose Quota Contention + Cleanup fallback metrics 2026-05-10 13:30:59 +03:00
Alexey
5c5a3fae06 Update AGENTS.md 2026-05-10 13:29:02 +03:00
Alexey
ba1d9be5d4 Hardened Relays and API Security paths 2026-05-10 13:22:54 +03:00
Alexey
b2aa9b8c9e Hardened API & Management-plane Admission
- bound API and metrics connection handling
- default metrics listener to localhost
- reject untrusted PROXY protocol peers before parsing headers
- cap API request body size and PROXY v2 payload allocation
- validate route usernames and TLS domains consistently
2026-05-09 20:50:23 +03:00
Alexey
73c82bda7a Update AGENTS.md 2026-05-09 16:34:54 +03:00
Alexey
b3510aa8b8 Bound HTTP API+Metrics Connection Admission 2026-05-09 16:29:30 +03:00
Alexey
dd3c5eff1c Merge pull request #776 from pavlozt/fix/sudo_path
Fix(installer): workaround for systems with secure PATH
2026-05-09 13:09:25 +03:00
PavelZ
dd4fb71959 Fix(installer): workaround for systems with secure PATH 2026-05-08 17:31:50 +03:00
Alexey
f0f2bc0482 Limit&Quota Saving as File + API 2026-05-08 14:38:24 +03:00
Alexey
86573be493 Event-driven Wakeup for ME Admission-gate 2026-05-08 13:34:41 +03:00
Alexey
658a565cb3 Merge pull request #770 from konstpic/feat/user-source-deny-list
feat(access): add per-user source IP deny list checks
2026-05-07 11:56:54 +03:00
Alexey
29fabcb199 Merge pull request #772 from agrofx1/user_check
Add root switch or login check
2026-05-07 11:53:50 +03:00
Alexey
efdf3bcc1b Fix root detection by checking UID 2026-05-07 11:53:29 +03:00
Agrofx
66c37ad6fd Merge branch 'flow' into user_check 2026-05-07 10:30:57 +03:00
Agrofx
0fcf67ca34 Update install.sh
Co-authored-by: Dimasssss <Dimasssss2000@gmail.com>
2026-05-07 10:30:47 +03:00
Agrofx
df14762a12 Add root switch or login check 2026-05-07 06:27:51 +03:00
Alexey
4995e83236 Config Strict and Validator 2026-05-06 20:38:55 +03:00
Alexey
e0f251ad82 TLS Domains masking fixes 2026-05-06 20:29:24 +03:00
Konstantin Pichugin
b605b1ba7c docs(access): document user_source_deny usage and API path
Add config examples and behavior notes for access.user_source_deny, and clarify that it is configured through config.toml rather than dedicated user API request fields.
2026-05-06 19:17:06 +03:00
Konstantin Pichugin
b859fb95c3 feat(access): add per-user source IP deny list checks
Add access.user_source_deny and enforce it in TLS and MTProto handshake paths after successful authentication to fail closed for blocked source IPs.
2026-05-06 19:11:18 +03:00
Alexey
8c303ab2b6 Merge pull request #765 from Misha20062006/patch-2
Correct saving instructions in QUICK_START_GUIDE.ru.md
2026-05-06 17:13:49 +03:00
Misha20062006
f70c2936c7 Correct saving instructions in QUICK_START_GUIDE.ru.md
Updated instructions for saving changes in the guide.
2026-05-06 00:07:14 +03:00
Alexey
d67c37afd7 Merge pull request #762 from astronaut808/feature/tls-front-profile-health
Add TLS Front Profile Health metrics
2026-05-05 15:23:01 +03:00
astronaut808
9f9ca9f270 Add TLS front profile health metrics 2026-05-03 18:07:24 +05:00
Alexey
cdd2239047 Merge pull request #758 from mammuthus/feature/metrics-bad-class-export-dashboard
Add class-based error metrics and dashboard panels
2026-05-02 00:46:53 +03:00
Alexey
9ee341a94f Merge pull request #757 from Dimasssss/docs
Update CONFIG_PARAMS
2026-05-02 00:36:46 +03:00
mamuthus
a7a2f4ab27 Adjust General metrics dashboard layout 2026-05-01 19:19:00 +00:00
mamuthus
9dae14aa66 Add class-based error metrics and dashboard panels 2026-05-01 18:26:32 +00:00
Dimasssss
f76c847c44 Update CONFIG_PARAMS.en.md 2026-05-01 21:10:34 +03:00
Dimasssss
1aaa9c0bc6 Update CONFIG_PARAMS.ru.md 2026-05-01 21:09:38 +03:00
Alexey
e50026e776 Update README.md 2026-04-30 19:41:40 +03:00
Alexey
7106f38fae Update Cargo.lock 2026-04-30 11:38:33 +03:00
Alexey
2a694470d5 Update Cargo.toml 2026-04-30 11:37:18 +03:00
Alexey
b98cd37211 TLS Full Certificate Budget Bookkeeping + Hot-path Cleanup and Timeout Invariants + IP-Tracker refactoring + Shard TLS Full-Cert Budget: merge pull request #753 from telemt/flow
TLS Full Certificate Budget Bookkeeping + Hot-path Cleanup and Timeout Invariants + IP-Tracker refactoring + Shard TLS Full-Cert Budget
2026-04-30 11:36:30 +03:00
Alexey
8b62965978 Stabilize unknown-DC symlink race test setup 2026-04-30 11:11:04 +03:00
Alexey
d46bda9880 Preserve synchronous IP cleanup queue contract + Rustfmt 2026-04-30 11:05:18 +03:00
Alexey
c3de07db6a Shard TLS full-cert budget tracking + Bound user-labeled metrics export cardinality 2026-04-30 11:01:10 +03:00
Alexey
61f9af7ffc Reduce Lock-free IP-Tracker Cleanup backlog 2026-04-30 10:51:04 +03:00
Alexey
1f90e28871 Cap scanner-sensitive Caches and IP-Tracker Cardinality 2026-04-30 10:43:27 +03:00
Alexey
876b74ebf7 Hot-path Cleanup and Timeout Invariants 2026-04-29 23:16:11 +03:00
Alexey
b34e1d71ae TLS Full Certificate Budget Bookkeeping 2026-04-29 23:00:25 +03:00
Alexey
b1c947e8e3 Limit only new ip when TimeWindow + Fix WorkingDirectory behavior + Atomically updates with Includes + Expose tls_domains links as domain-link pairs + TLS Fetcher on multiple tls_domains: merge pull request #751 from telemt/flow
Limit only new ip when TimeWindow + Fix WorkingDirectory behavior + Atomically updates with Includes + Expose tls_domains links as domain-link pairs + TLS Fetcher on multiple tls_domains
2026-04-29 16:04:36 +03:00
Alexey
cfe01dced2 Bump
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 15:54:22 +03:00
Alexey
8520955a5f Update helpers.rs
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 15:53:27 +03:00
Alexey
065786b839 TLS Fetcher on multiple tls_domains by #750
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 11:47:42 +03:00
Alexey
f0e1a6cf1c Expose tls_domains links as domain-link pairs
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-29 11:34:47 +03:00
Alexey
236bbb4970 Atomically updates with Includes
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-28 13:00:13 +03:00
Alexey
8ef5263fce Fix WorkingDirectory behavior
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
Co-Authored-By: mikhailnov <m@mikhailnov.ru>
Signed-off-by: Alexey <247128645+axkurcom@users.noreply.github.com>
2026-04-28 12:31:21 +03:00
Alexey
893cef22e3 Update README.md 2026-04-27 23:49:47 +03:00
Alexey
bdfa641843 Merge pull request #735 from sanekb/fix_timewindow_same_ip
fix: limit only new ip when TimeWindow mode enabled
2026-04-25 19:08:36 +03:00
Alexey
007fc86189 Merge branch 'flow' into fix_timewindow_same_ip 2026-04-25 18:56:27 +03:00
sanekb
d567dfe40b fix: limit only new ip when TimeWindow mode enabled 2026-04-25 14:36:43 +03:00
55 changed files with 11398 additions and 5471 deletions

View File

@@ -191,6 +191,11 @@ When facing a non-trivial modification, follow this sequence:
4. **Implement**: Make the minimal, isolated change.
5. **Verify**: Explain why the change preserves existing behavior and architectural integrity.
When the repository contains a `PLAN.md` for the current task, maintain it as
a working checkbox plan while implementing changes. Mark completed and partial
items in `PLAN.md` as the code changes land, so the remaining work stays
explicit and future passes do not waste time rediscovering status.
---
### 9. Context Awareness
@@ -222,10 +227,9 @@ Your response MUST consist of two sections:
**Section 2: `## Changes`**
- For each modified or created file: the filename on a separate line in backticks, followed by the code block.
- For files **under 200 lines**: return the full file with all changes applied.
- For files **over 200 lines**: return only the changed functions/blocks with at least 3 lines of surrounding context above and below. If the user requests the full file, provide it.
- New files: full file content.
- For each modified or created file: the filename on a separate line in backticks, followed by a concise description of what changed.
- Do not include full file contents or long code blocks in `## Changes` unless the user explicitly asks for code text.
- If code snippets are necessary, include only the minimal relevant excerpt.
- End with a suggested git commit message in English.
#### Reporting Out-of-Scope Issues
@@ -429,4 +433,3 @@ Every patch must be **atomic and production-safe**.
* **No transitional states** — no placeholders, incomplete refactors, or temporary inconsistencies.
**Invariant:** After any single patch, the repository remains fully functional and buildable.

6
Cargo.lock generated
View File

@@ -2404,9 +2404,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
version = "0.103.12"
version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
@@ -2791,7 +2791,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "telemt"
version = "3.4.8"
version = "3.4.11"
dependencies = [
"aes",
"anyhow",

View File

@@ -1,6 +1,6 @@
[package]
name = "telemt"
version = "3.4.8"
version = "3.4.11"
edition = "2024"
[features]

View File

@@ -1,6 +1,6 @@
# Telemt - MTProxy on Rust + Tokio
[![Latest Release](https://img.shields.io/github/v/release/telemt/telemt?color=neon)](https://github.com/telemt/telemt/releases/latest) [![Stars](https://img.shields.io/github/stars/telemt/telemt?style=social)](https://github.com/telemt/telemt/stargazers) [![Forks](https://img.shields.io/github/forks/telemt/telemt?style=social)](https://github.com/telemt/telemt/network/members) [![Telegram](https://img.shields.io/badge/Telegram-Chat-24a1de?logo=telegram&logoColor=24a1de)](https://t.me/telemtrs)
[![Latest Release](https://img.shields.io/github/v/release/telemt/telemt?color=neon)](https://github.com/telemt/telemt/releases/latest) [![Stars](https://img.shields.io/github/stars/telemt/telemt?style=social)](https://github.com/telemt/telemt/stargazers) [![Forks](https://img.shields.io/github/forks/telemt/telemt?style=social)](https://github.com/telemt/telemt/network/members)
[🇷🇺 README на русском](https://github.com/telemt/telemt/blob/main/README.ru.md)
@@ -14,7 +14,7 @@
<p align="center">
<a href="https://t.me/telemtrs">
<img src="/docs/assets/telegram_button.svg" width="150"/>
<img src="https://github.com/user-attachments/assets/30b7e7b9-974a-4e3d-aab6-b58a85de4507" width="240"/>
</a>
</p>
@@ -99,7 +99,7 @@ Monero (XMR) directly:
8Bk4tZEYPQWSypeD2hrUXG2rKbAKF16GqEN942ZdAP5cFdSqW6h4DwkP5cJMAdszzuPeHeHZPTyjWWFwzeFdjuci3ktfMoB
```
All donations go toward infrastructure, development, and research.
All donations go toward infrastructure, development and research
![telemt_scheme](docs/assets/telemt.png)

View File

@@ -13,7 +13,7 @@ API runtime is configured in `[server.api]`.
| `listen` | `string` (`IP:PORT`) | `0.0.0.0:9091` | API bind address. |
| `whitelist` | `CIDR[]` | `127.0.0.0/8` | Source IP allowlist. Empty list means allow all. |
| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. |
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be within `[1, 1048576]`. |
| `minimal_runtime_enabled` | `bool` | `true` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
@@ -26,7 +26,7 @@ API runtime is configured in `[server.api]`.
Runtime validation for API config:
- `server.api.listen` must be a valid `IP:PORT`.
- `server.api.request_body_limit_bytes` must be `> 0`.
- `server.api.request_body_limit_bytes` must be within `[1, 1048576]`.
- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`.
- `server.api.runtime_edge_cache_ttl_ms` must be within `[0, 60000]`.
- `server.api.runtime_edge_top_n` must be within `[1, 1000]`.
@@ -76,13 +76,14 @@ Requests are processed in this order:
Notes:
- Whitelist is evaluated against the direct TCP peer IP (`SocketAddr::ip`), without `X-Forwarded-For` support.
- `Authorization` check is exact string equality against configured `auth_header`.
- `Authorization` check is exact constant-time byte equality against configured `auth_header`.
## Endpoint Matrix
| Method | Path | Body | Success | `data` contract |
| --- | --- | --- | --- | --- |
| `GET` | `/v1/health` | none | `200` | `HealthData` |
| `GET` | `/v1/health/ready` | none | `200` or `503` | `HealthReadyData` |
| `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` |
| `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` |
| `GET` | `/v1/runtime/initialization` | none | `200` | `RuntimeInitializationData` |
@@ -102,13 +103,50 @@ Notes:
| `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` |
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
| `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` |
| `POST` | `/v1/users` | `CreateUserRequest` | `201` or `202` | `CreateUserResponse` |
| `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` |
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` | `UserInfo` |
| `DELETE` | `/v1/users/{username}` | none | `200` | `string` (deleted username) |
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `404` | `ErrorResponse` (`not_found`, current runtime behavior) |
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` or `202` | `UserInfo` |
| `DELETE` | `/v1/users/{username}` | none | `200` or `202` | `DeleteUserResponse` |
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` or `202` | `CreateUserResponse` |
| `POST` | `/v1/users/{username}/reset-quota` | empty body | `200` | `ResetUserQuotaResponse` |
## Endpoint Behavior
| Endpoint | Function |
| --- | --- |
| `GET /v1/health` | Returns basic API liveness and current `read_only` flag. |
| `GET /v1/health/ready` | Returns readiness based on admission state and upstream health; returns `503` when not ready. |
| `GET /v1/system/info` | Returns binary/build metadata, process uptime, config path/hash, and reload counters. |
| `GET /v1/runtime/gates` | Returns admission, ME readiness, fallback/reroute, and startup gate state. |
| `GET /v1/runtime/initialization` | Returns startup progress, ME initialization status, and per-component timeline. |
| `GET /v1/limits/effective` | Returns effective timeout, upstream, ME, unique-IP, and TCP policy values after config defaults/resolution. |
| `GET /v1/security/posture` | Returns current API/security/telemetry posture flags. |
| `GET /v1/security/whitelist` | Returns configured API whitelist CIDRs. |
| `GET /v1/stats/summary` | Returns compact core counters and classed failure counters. |
| `GET /v1/stats/zero/all` | Returns zero-cost core, upstream, ME, pool, and desync counters. |
| `GET /v1/stats/upstreams` | Returns upstream zero counters and, when enabled/available, runtime upstream health rows. |
| `GET /v1/stats/minimal/all` | Returns cached minimal ME writer/DC/runtime/network-path snapshot. |
| `GET /v1/stats/me-writers` | Returns cached ME writer coverage and per-writer status rows. |
| `GET /v1/stats/dcs` | Returns cached per-DC endpoint/writer/load status rows. |
| `GET /v1/runtime/me_pool_state` | Returns active/warm/pending/draining generation state, writer contour/health, and refill state. |
| `GET /v1/runtime/me_quality` | Returns ME lifecycle counters, route-drop counters, family states, drain gate, and per-DC RTT/coverage. |
| `GET /v1/runtime/upstream_quality` | Returns upstream policy/counters plus runtime upstream health rows when available. |
| `GET /v1/runtime/nat_stun` | Returns NAT/STUN runtime flags, configured/live STUN servers, reflection cache, and backoff. |
| `GET /v1/runtime/me-selftest` | Returns ME self-test state for KDF, time skew, IP family, PID, and SOCKS BND observations. |
| `GET /v1/runtime/connections/summary` | Returns runtime-edge connection totals and top-N users by connections/throughput. |
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
| `GET /v1/users` | Returns disk-first user views sorted by username. |
| `POST /v1/users` | Creates a user and returns the effective user view plus secret. |
| `GET /v1/users/{username}` | Returns one disk-first user view or `404` when absent. |
| `PATCH /v1/users/{username}` | Updates selected per-user fields with JSON Merge Patch semantics. |
| `DELETE /v1/users/{username}` | Deletes one user and related per-user access-map entries. |
| `POST /v1/users/{username}/rotate-secret` | Rotates one user's secret and returns the effective secret. |
| `POST /v1/users/{username}/reset-quota` | Resets one user's runtime quota counter and persists quota state. |
## Common Error Codes
@@ -118,7 +156,7 @@ Notes:
| `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. |
| `403` | `forbidden` | Source IP is not allowed by whitelist. |
| `403` | `read_only` | Mutating endpoint called while `read_only=true`. |
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route (including current `rotate-secret` route). |
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route. |
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
| `409` | `user_exists` | User already exists on create. |
@@ -132,11 +170,12 @@ Notes:
| Case | Behavior |
| --- | --- |
| Path matching | Exact match on `req.uri().path()`. Query string does not affect route matching. |
| Trailing slash | Not normalized. Example: `/v1/users/` is `404`. |
| Trailing slash | Trimmed for route matching when path length is greater than 1. Example: `/v1/users/` matches `/v1/users`. |
| Username route with extra slash | `/v1/users/{username}/...` is not treated as user route and returns `404`. |
| `PUT /v1/users/{username}` | `405 method_not_allowed`. |
| `POST /v1/users/{username}` | `404 not_found`. |
| `POST /v1/users/{username}/rotate-secret` | `404 not_found` in current release due route matcher limitation. |
| `POST /v1/users/{username}/rotate-secret/` | Trailing slash is trimmed and the route matches `rotate-secret`. |
| `POST /v1/users/{username}/reset-quota/` | Trailing slash is trimmed and the route matches `reset-quota`. |
## Body and JSON Semantics
@@ -146,7 +185,7 @@ Notes:
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
- `Content-Type` is not required for JSON parsing.
- Unknown JSON fields are ignored by deserialization.
- `PATCH` updates only provided fields and does not support explicit clearing of optional fields.
- `PATCH` uses JSON Merge Patch semantics for optional per-user fields: omitted means unchanged, explicit `null` removes the config entry, and a non-null value sets it.
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
## Query Parameters
@@ -166,24 +205,43 @@ Notes:
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
| `rate_limit_up_bps` | `u64` | no | Per-user upload rate limit in bytes per second. |
| `rate_limit_down_bps` | `u64` | no | Per-user download rate limit in bytes per second. |
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
### `PatchUserRequest`
| Field | Type | Required | Description |
| --- | --- | --- | --- |
| `secret` | `string` | no | Exactly 32 hex chars. |
| `user_ad_tag` | `string` | no | Exactly 32 hex chars. |
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
| `user_ad_tag` | `string|null` | no | Exactly 32 hex chars; `null` removes the per-user ad tag. |
| `max_tcp_conns` | `usize|null` | no | Per-user concurrent TCP limit; `null` removes the per-user override. |
| `expiration_rfc3339` | `string|null` | no | RFC3339 expiration timestamp; `null` removes the expiration. |
| `data_quota_bytes` | `u64|null` | no | Per-user traffic quota; `null` removes the per-user quota. |
| `rate_limit_up_bps` | `u64|null` | no | Per-user upload rate limit in bytes per second; `null` removes the upload direction limit. |
| `rate_limit_down_bps` | `u64|null` | no | Per-user download rate limit in bytes per second; `null` removes the download direction limit. |
| `max_unique_ips` | `usize|null` | no | Per-user unique source IP limit; `null` removes the per-user override. |
### `access.user_source_deny` via API
- In current API surface, per-user deny-list is **not** exposed as a dedicated field in `CreateUserRequest` / `PatchUserRequest`.
- Configure it in `config.toml` under `[access.user_source_deny]` and apply via normal config reload path.
- Runtime behavior after apply:
- auth succeeds for username/secret
- source IP is checked against `access.user_source_deny[username]`
- on match, handshake is rejected with the same fail-closed outcome as invalid auth
Example config:
```toml
[access.user_source_deny]
alice = ["203.0.113.0/24", "2001:db8:abcd::/48"]
bob = ["198.51.100.42/32"]
```
### `RotateSecretRequest`
| Field | Type | Required | Description |
| --- | --- | --- | --- |
| `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. |
Note: the request contract is defined, but the corresponding route currently returns `404` (see routing edge cases).
An empty request body is accepted and generates a new secret automatically.
## Response Data Contracts
@@ -193,15 +251,33 @@ Note: the request contract is defined, but the corresponding route currently ret
| `status` | `string` | Always `"ok"`. |
| `read_only` | `bool` | Mirrors current API `read_only` mode. |
### `HealthReadyData`
| Field | Type | Description |
| --- | --- | --- |
| `ready` | `bool` | `true` when admission is open and at least one upstream is healthy. |
| `status` | `string` | `"ready"` or `"not_ready"`. |
| `reason` | `string?` | `admission_closed` or `no_healthy_upstreams` when not ready. |
| `admission_open` | `bool` | Current admission-gate state. |
| `healthy_upstreams` | `usize` | Number of healthy upstream entries. |
| `total_upstreams` | `usize` | Number of configured upstream entries. |
### `SummaryData`
| Field | Type | Description |
| --- | --- | --- |
| `uptime_seconds` | `f64` | Process uptime in seconds. |
| `connections_total` | `u64` | Total accepted client connections. |
| `connections_bad_total` | `u64` | Failed/invalid client connections. |
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
| `configured_users` | `usize` | Number of configured users in config. |
#### `ClassCount`
| Field | Type | Description |
| --- | --- | --- |
| `class` | `string` | Failure class label. |
| `total` | `u64` | Counter value for this class. |
### `SystemInfoData`
| Field | Type | Description |
| --- | --- | --- |
@@ -226,7 +302,12 @@ Note: the request contract is defined, but the corresponding route currently ret
| `conditional_cast_enabled` | `bool` | Whether conditional ME admission logic is enabled (`general.use_middle_proxy`). |
| `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. |
| `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. |
| `me2dc_fast_enabled` | `bool` | Whether fast ME -> direct fallback is enabled. |
| `use_middle_proxy` | `bool` | Current transport mode preference. |
| `route_mode` | `string` | Current route mode label from route runtime controller. |
| `reroute_active` | `bool` | `true` when ME fallback currently routes new sessions to Direct-DC. |
| `reroute_to_direct_at_epoch_secs` | `u64?` | Unix timestamp when current direct reroute began. |
| `reroute_reason` | `string?` | `fast_not_ready_fallback` or `strict_grace_fallback` while reroute is active. |
| `startup_status` | `string` | Startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). |
| `startup_stage` | `string` | Current startup stage identifier. |
| `startup_progress_pct` | `f64` | Startup progress percentage (`0..100`). |
@@ -277,11 +358,13 @@ Note: the request contract is defined, but the corresponding route currently ret
| `upstream` | `EffectiveUpstreamLimits` | Effective upstream connect/retry limits. |
| `middle_proxy` | `EffectiveMiddleProxyLimits` | Effective ME pool/floor/reconnect limits. |
| `user_ip_policy` | `EffectiveUserIpPolicyLimits` | Effective unique-IP policy mode/window. |
| `user_tcp_policy` | `EffectiveUserTcpPolicyLimits` | Effective per-user TCP connection policy. |
#### `EffectiveTimeoutLimits`
| Field | Type | Description |
| --- | --- | --- |
| `client_handshake_secs` | `u64` | Client handshake timeout. |
| `client_first_byte_idle_secs` | `u64` | First-byte idle timeout before protocol classification. |
| `tg_connect_secs` | `u64` | Upstream Telegram connect timeout. |
| `client_keepalive_secs` | `u64` | Client keepalive interval. |
| `client_ack_secs` | `u64` | ACK timeout. |
@@ -320,13 +403,20 @@ Note: the request contract is defined, but the corresponding route currently ret
| `writer_pick_mode` | `string` | Writer picker mode (`sorted_rr`, `p2c`). |
| `writer_pick_sample_size` | `u8` | Candidate sample size for `p2c` picker mode. |
| `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. |
| `me2dc_fast` | `bool` | Effective fast fallback flag. |
#### `EffectiveUserIpPolicyLimits`
| Field | Type | Description |
| --- | --- | --- |
| `global_each` | `usize` | Global per-user unique-IP limit applied when no per-user override exists. |
| `mode` | `string` | Unique-IP policy mode (`active_window`, `time_window`, `combined`). |
| `window_secs` | `u64` | Time window length used by unique-IP policy. |
#### `EffectiveUserTcpPolicyLimits`
| Field | Type | Description |
| --- | --- | --- |
| `global_each` | `usize` | Global per-user concurrent TCP limit applied when no per-user override exists. |
### `SecurityPostureData`
| Field | Type | Description |
| --- | --- | --- |
@@ -430,6 +520,8 @@ Note: the request contract is defined, but the corresponding route currently ret
| --- | --- | --- |
| `counters` | `RuntimeMeQualityCountersData` | Key ME lifecycle/error counters. |
| `route_drops` | `RuntimeMeQualityRouteDropData` | Route drop counters by reason. |
| `family_states` | `RuntimeMeQualityFamilyStateData[]` | Per-family ME route/recovery state rows. |
| `drain_gate` | `RuntimeMeQualityDrainGateData` | Current ME drain-gate decision state. |
| `dc_rtt` | `RuntimeMeQualityDcRttData[]` | Per-DC RTT and writer coverage rows. |
#### `RuntimeMeQualityCountersData`
@@ -451,6 +543,24 @@ Note: the request contract is defined, but the corresponding route currently ret
| `queue_full_base_total` | `u64` | Route drops in base-queue path. |
| `queue_full_high_total` | `u64` | Route drops in high-priority queue path. |
#### `RuntimeMeQualityFamilyStateData`
| Field | Type | Description |
| --- | --- | --- |
| `family` | `string` | Address family label. |
| `state` | `string` | Current family state label. |
| `state_since_epoch_secs` | `u64` | Unix timestamp when current state began. |
| `suppressed_until_epoch_secs` | `u64?` | Unix timestamp until suppression remains active. |
| `fail_streak` | `u32` | Consecutive failure count. |
| `recover_success_streak` | `u32` | Consecutive recovery success count. |
#### `RuntimeMeQualityDrainGateData`
| Field | Type | Description |
| --- | --- | --- |
| `route_quorum_ok` | `bool` | Whether route quorum condition allows drain. |
| `redundancy_ok` | `bool` | Whether redundancy condition allows drain. |
| `block_reason` | `string` | Current drain block reason label. |
| `updated_at_epoch_secs` | `u64` | Unix timestamp of the latest gate update. |
#### `RuntimeMeQualityDcRttData`
| Field | Type | Description |
| --- | --- | --- |
@@ -713,11 +823,24 @@ Note: the request contract is defined, but the corresponding route currently ret
| `uptime_seconds` | `f64` | Process uptime. |
| `connections_total` | `u64` | Total accepted connections. |
| `connections_bad_total` | `u64` | Failed/invalid connections. |
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
| `handshake_timeouts_total` | `u64` | Handshake timeouts. |
| `accept_permit_timeout_total` | `u64` | Listener admission permit acquisition timeouts. |
| `configured_users` | `usize` | Configured user count. |
| `telemetry_core_enabled` | `bool` | Core telemetry toggle. |
| `telemetry_user_enabled` | `bool` | User telemetry toggle. |
| `telemetry_me_level` | `string` | ME telemetry level (`off|normal|verbose`). |
| `conntrack_control_enabled` | `bool` | Whether conntrack control is enabled by policy. |
| `conntrack_control_available` | `bool` | Whether conntrack control backend is currently available. |
| `conntrack_pressure_active` | `bool` | Current conntrack pressure flag. |
| `conntrack_event_queue_depth` | `u64` | Current conntrack close-event queue depth. |
| `conntrack_rule_apply_ok` | `bool` | Last conntrack rule application state. |
| `conntrack_delete_attempt_total` | `u64` | Conntrack delete attempts. |
| `conntrack_delete_success_total` | `u64` | Successful conntrack deletes. |
| `conntrack_delete_not_found_total` | `u64` | Conntrack delete misses. |
| `conntrack_delete_error_total` | `u64` | Conntrack delete errors. |
| `conntrack_close_event_drop_total` | `u64` | Dropped conntrack close events. |
#### `ZeroUpstreamData`
| Field | Type | Description |
@@ -804,6 +927,24 @@ Note: the request contract is defined, but the corresponding route currently ret
| `route_drop_queue_full_total` | `u64` | Route drops due to full queue (total). |
| `route_drop_queue_full_base_total` | `u64` | Route drops in base queue mode. |
| `route_drop_queue_full_high_total` | `u64` | Route drops in high queue mode. |
| `d2c_batches_total` | `u64` | ME D->C batch flushes. |
| `d2c_batch_frames_total` | `u64` | ME D->C frames included in batches. |
| `d2c_batch_bytes_total` | `u64` | ME D->C payload bytes included in batches. |
| `d2c_flush_reason_queue_drain_total` | `u64` | Flushes caused by queue drain. |
| `d2c_flush_reason_batch_frames_total` | `u64` | Flushes caused by frame-count batch limit. |
| `d2c_flush_reason_batch_bytes_total` | `u64` | Flushes caused by byte-count batch limit. |
| `d2c_flush_reason_max_delay_total` | `u64` | Flushes caused by max-delay budget. |
| `d2c_flush_reason_ack_immediate_total` | `u64` | Flushes caused by immediate ACK policy. |
| `d2c_flush_reason_close_total` | `u64` | Flushes caused by close path. |
| `d2c_data_frames_total` | `u64` | ME D->C data frames. |
| `d2c_ack_frames_total` | `u64` | ME D->C ACK frames. |
| `d2c_payload_bytes_total` | `u64` | ME D->C payload bytes. |
| `d2c_write_mode_coalesced_total` | `u64` | Coalesced D->C writes. |
| `d2c_write_mode_split_total` | `u64` | Split D->C writes. |
| `d2c_quota_reject_pre_write_total` | `u64` | D->C quota rejects before write. |
| `d2c_quota_reject_post_write_total` | `u64` | D->C quota rejects after write. |
| `d2c_frame_buf_shrink_total` | `u64` | D->C frame-buffer shrink operations. |
| `d2c_frame_buf_shrink_bytes_total` | `u64` | Bytes released by D->C frame-buffer shrink operations. |
| `socks_kdf_strict_reject_total` | `u64` | SOCKS KDF strict rejects. |
| `socks_kdf_compat_fallback_total` | `u64` | SOCKS KDF compat fallbacks. |
| `endpoint_quarantine_total` | `u64` | Endpoint quarantine activations. |
@@ -963,6 +1104,8 @@ Note: the request contract is defined, but the corresponding route currently ret
| `required_writers` | `usize` | Required writers based on current floor policy. |
| `alive_writers` | `usize` | Writers currently alive. |
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
| `fresh_alive_writers` | `usize` | Alive writers that match freshness requirements. |
| `fresh_coverage_pct` | `f64` | `fresh_alive_writers / required_writers * 100`. |
#### `MeWriterStatus`
| Field | Type | Description |
@@ -977,6 +1120,12 @@ Note: the request contract is defined, but the corresponding route currently ret
| `bound_clients` | `usize` | Number of currently bound clients. |
| `idle_for_secs` | `u64?` | Idle age in seconds if idle. |
| `rtt_ema_ms` | `f64?` | RTT exponential moving average. |
| `matches_active_generation` | `bool` | Whether this writer belongs to the active pool generation. |
| `in_desired_map` | `bool` | Whether this writer's endpoint remains in desired topology. |
| `allow_drain_fallback` | `bool` | Whether drain fallback is allowed for this writer. |
| `drain_started_at_epoch_secs` | `u64?` | Unix timestamp when drain started. |
| `drain_deadline_epoch_secs` | `u64?` | Unix timestamp of drain deadline. |
| `drain_over_ttl` | `bool` | Whether drain has exceeded its TTL. |
### `DcStatusData`
| Field | Type | Description |
@@ -1001,6 +1150,8 @@ Note: the request contract is defined, but the corresponding route currently ret
| `floor_capped` | `bool` | `true` when computed floor target was capped by active limits. |
| `alive_writers` | `usize` | Alive writers in this DC. |
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
| `fresh_alive_writers` | `usize` | Fresh alive writers in this DC. |
| `fresh_coverage_pct` | `f64` | `fresh_alive_writers / required_writers * 100`. |
| `rtt_ms` | `f64?` | Aggregated RTT for DC. |
| `load` | `usize` | Active client sessions bound to this DC. |
@@ -1014,10 +1165,13 @@ Note: the request contract is defined, but the corresponding route currently ret
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | Username. |
| `in_runtime` | `bool` | Whether current runtime config already contains this user. |
| `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). |
| `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. |
| `expiration_rfc3339` | `string?` | Optional expiration timestamp. |
| `data_quota_bytes` | `u64?` | Optional data quota. |
| `rate_limit_up_bps` | `u64?` | Optional upload rate limit in bytes per second. |
| `rate_limit_down_bps` | `u64?` | Optional download rate limit in bytes per second. |
| `max_unique_ips` | `usize?` | Optional unique IP limit. |
| `current_connections` | `u64` | Current live connections. |
| `active_unique_ips` | `usize` | Current active unique source IPs. |
@@ -1027,12 +1181,25 @@ Note: the request contract is defined, but the corresponding route currently ret
| `total_octets` | `u64` | Total traffic octets for this user. |
| `links` | `UserLinks` | Active connection links derived from current config. |
### `UserActiveIps`
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | Username with at least one active tracked source IP. |
| `active_ips` | `ip[]` | Active source IPs for this user. |
#### `UserLinks`
| Field | Type | Description |
| --- | --- | --- |
| `classic` | `string[]` | Active `tg://proxy` links for classic mode. |
| `secure` | `string[]` | Active `tg://proxy` links for secure/DD mode. |
| `tls` | `string[]` | Active `tg://proxy` links for EE-TLS mode (for each host+TLS domain). |
| `tls_domains` | `TlsDomainLink[]` | Extra TLS-domain links as explicit domain/link pairs for `censorship.tls_domains`. |
#### `TlsDomainLink`
| Field | Type | Description |
| --- | --- | --- |
| `domain` | `string` | TLS domain represented by the link. |
| `link` | `string` | `tg://proxy` link for this domain. |
Link generation uses active config and enabled modes:
- Link port is `general.links.public_port` when configured; otherwise `server.port`.
@@ -1052,13 +1219,27 @@ Link generation uses active config and enabled modes:
| `user` | `UserInfo` | Created or updated user view. |
| `secret` | `string` | Effective user secret. |
### `DeleteUserResponse`
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | Deleted username. |
| `in_runtime` | `bool` | `true` when runtime config still contains the user and hot-reload has not applied deletion yet. |
### `ResetUserQuotaResponse`
| Field | Type | Description |
| --- | --- | --- |
| `username` | `string` | User whose runtime quota counter was reset. |
| `used_bytes` | `u64` | Current used bytes after reset; always `0` on success. |
| `last_reset_epoch_secs` | `u64` | Unix timestamp of the reset operation. |
## Mutation Semantics
| Endpoint | Notes |
| --- | --- |
| `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). |
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. Current implementation persists full config document on success. |
| `POST /v1/users/{username}/rotate-secret` | Currently returns `404` in runtime route matcher; request schema is reserved for intended behavior. |
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged; explicit `null` removes optional per-user entries. The write path updates only affected `access.*` TOML tables. |
| `POST /v1/users/{username}/rotate-secret` | Replaces the user's secret with a provided valid 32-hex value or a generated value, then returns the effective secret in `CreateUserResponse`. |
| `POST /v1/users/{username}/reset-quota` | Resets the runtime quota counter for the route username, persists quota state to `general.quota_state_path`, and does not modify user config. |
| `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. |
All mutating endpoints:
@@ -1067,6 +1248,12 @@ All mutating endpoints:
- Return new `revision` after successful write.
- Use process-local mutation lock + atomic write (`tmp + rename`) for config persistence.
Docker deployment note:
- Mutating endpoints require `config.toml` to live inside a writable mounted directory.
- Do not mount `config.toml` as a single bind-mounted file when API mutations are enabled; atomic `tmp + rename` writes can fail with `Device or resource busy`.
- Mount the config directory instead, for example `./config:/etc/telemt:rw`, and start Telemt with `/etc/telemt/config.toml`.
- A read-only single-file mount remains valid only for read-only deployments or when `[server.api].read_only=true`.
Delete path cleanup guarantees:
- Config cleanup removes only the requested username keys.
- Runtime unique-IP cleanup removes only this user's limiter and tracked IP state.
@@ -1133,5 +1320,4 @@ When `general.use_middle_proxy=true` and `general.me2dc_fallback=true`:
## Known Limitations (Current Release)
- `POST /v1/users/{username}/rotate-secret` is currently unreachable in route matcher and returns `404`.
- API runtime controls under `server.api` are documented as restart-required; hot-reload behavior for these fields is not strictly uniform in all change combinations.

View File

@@ -128,7 +128,48 @@ Recommended for cleaner testing:
Persisted cache artifacts are useful, but they are not required if packet captures already demonstrate the runtime result.
### 4. Capture a direct-origin trace
### 4. Check TLS-front profile health metrics
If the metrics endpoint is enabled, check the TLS-front profile health before packet-capture validation:
```bash
curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert'
```
The profile-health metrics expose the runtime state of configured TLS front domains:
- `telemt_tls_front_profile_domains` shows configured, emitted, and suppressed domain series.
- `telemt_tls_front_profile_info` shows profile source and feature flags per domain.
- `telemt_tls_front_profile_age_seconds` shows cached profile age.
- `telemt_tls_front_profile_app_data_records` shows cached AppData record count.
- `telemt_tls_front_profile_ticket_records` shows cached ticket-like tail record count.
- `telemt_tls_front_profile_change_cipher_spec_records` shows cached ChangeCipherSpec count.
- `telemt_tls_front_profile_app_data_bytes` shows total cached AppData bytes.
Interpretation:
- `source="merged"` or `source="raw"` means real TLS profile data is being used.
- `source="default"` or `is_default="true"` means the domain currently uses the synthetic default fallback.
- `has_cert_payload="true"` means certificate payload data is available for TLS emulation.
- Non-zero AppData/ticket/CCS counters show captured server-flight shape.
Example healthy output:
```text
telemt_tls_front_profile_domains{status="configured"} 1
telemt_tls_front_profile_domains{status="emitted"} 1
telemt_tls_front_profile_domains{status="suppressed"} 0
telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1
telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20
telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3
telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240
```
These metrics do not prove byte-level origin equivalence. They are an operational health signal that the configured domain is backed by real cached profile data instead of default fallback data.
### 5. Capture a direct-origin trace
From a separate client host, connect directly to the origin:
@@ -142,7 +183,7 @@ Capture with:
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
```
### 5. Capture a Telemt FakeTLS success-path trace
### 6. Capture a Telemt FakeTLS success-path trace
Now connect to Telemt with a real Telegram client through an `ee` proxy link that targets the Telemt instance.
@@ -154,7 +195,7 @@ Capture with:
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
```
### 6. Decode TLS record structure
### 7. Decode TLS record structure
Use `tshark` to print record-level structure:
@@ -182,7 +223,7 @@ Focus on the server flight after ClientHello:
- `20` = ChangeCipherSpec
- `23` = ApplicationData
### 7. Build a comparison table
### 8. Build a comparison table
A compact table like the following is usually enough:

View File

@@ -126,9 +126,50 @@ openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
2. Дайте ему получить TLS front profile data для выбранного домена.
3. Если `tls_front_dir` хранится persistently, убедитесь, что TLS front cache заполнен.
Persisted cache artifacts полезны, но не обязательны, если packet capture уже показывают runtime result.
Сохранённые артефакты кэша полезны, но не обязательны, если packet capture уже показывает результат в runtime.
### 4. Снять direct-origin trace
### 4. Проверить метрики состояния TLS-front profile
Если endpoint метрик включён, перед проверкой через packet capture можно быстро проверить состояние TLS-front profile:
```bash
curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert'
```
Метрики состояния профиля показывают runtime-состояние настроенных TLS-front доменов:
- `telemt_tls_front_profile_domains` показывает количество настроенных, экспортируемых и скрытых из-за лимита доменов.
- `telemt_tls_front_profile_info` показывает источник профиля и флаги доступных данных по каждому домену.
- `telemt_tls_front_profile_age_seconds` показывает возраст закешированного профиля.
- `telemt_tls_front_profile_app_data_records` показывает количество закешированных AppData records.
- `telemt_tls_front_profile_ticket_records` показывает количество закешированных ticket-like tail records.
- `telemt_tls_front_profile_change_cipher_spec_records` показывает закешированное количество ChangeCipherSpec records.
- `telemt_tls_front_profile_app_data_bytes` показывает общий размер закешированных AppData bytes.
Интерпретация:
- `source="merged"` или `source="raw"` означает, что используются реальные данные TLS-профиля.
- `source="default"` или `is_default="true"` означает, что домен сейчас работает на synthetic default fallback.
- `has_cert_payload="true"` означает, что certificate payload доступен для TLS emulation.
- Ненулевые AppData/ticket/CCS counters показывают захваченную форму server flight.
Пример здорового состояния:
```text
telemt_tls_front_profile_domains{status="configured"} 1
telemt_tls_front_profile_domains{status="emitted"} 1
telemt_tls_front_profile_domains{status="suppressed"} 0
telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1
telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20
telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3
telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1
telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240
```
Эти метрики не доказывают побайтную эквивалентность с origin. Это эксплуатационный сигнал состояния: настроенный домен действительно основан на реальных закешированных данных профиля, а не на default fallback.
### 5. Снять direct-origin trace
С отдельной клиентской машины подключитесь напрямую к origin:
@@ -142,7 +183,7 @@ Capture:
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
```
### 5. Снять Telemt FakeTLS success-path trace
### 6. Снять Telemt FakeTLS success-path trace
Теперь подключитесь к Telemt через реальный Telegram client с `ee` proxy link, который указывает на Telemt instance.
@@ -154,7 +195,7 @@ Capture:
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
```
### 6. Декодировать структуру TLS records
### 7. Декодировать структуру TLS records
Используйте `tshark`, чтобы вывести record-level structure:
@@ -182,7 +223,7 @@ tshark -r telemt-emulated.pcap -Y "tls.record" -T fields \
- `20` = ChangeCipherSpec
- `23` = ApplicationData
### 7. Собрать сравнительную таблицу
### 8. Собрать сравнительную таблицу
Обычно достаточно короткой таблицы такого вида:

View File

@@ -162,6 +162,8 @@ This document lists all configuration keys accepted by `config.toml`.
| [`log_level`](#log_level) | `"debug"`, `"verbose"`, `"normal"`, or `"silent"` | `"normal"` |
| [`disable_colors`](#disable_colors) | `bool` | `false` |
| [`me_socks_kdf_policy`](#me_socks_kdf_policy) | `"strict"` or `"compat"` | `"strict"` |
| [`me_route_backpressure_enabled`](#me_route_backpressure_enabled) | `bool` | `false` |
| [`me_route_fairshare_enabled`](#me_route_fairshare_enabled) | `bool` | `false` |
| [`me_route_backpressure_base_timeout_ms`](#me_route_backpressure_base_timeout_ms) | `u64` | `25` |
| [`me_route_backpressure_high_timeout_ms`](#me_route_backpressure_high_timeout_ms) | `u64` | `120` |
| [`me_route_backpressure_high_watermark_pct`](#me_route_backpressure_high_watermark_pct) | `u8` | `80` |
@@ -975,6 +977,24 @@ This document lists all configuration keys accepted by `config.toml`.
[general]
me_socks_kdf_policy = "strict"
```
## me_route_backpressure_enabled
- **Constraints / validation**: `bool`.
- **Description**: Enables channel-pressure-aware route send timeouts.
- **Example**:
```toml
[general]
me_route_backpressure_enabled = false
```
## me_route_fairshare_enabled
- **Constraints / validation**: `bool`.
- **Description**: Enables fair-share routing admission across writer workers.
- **Example**:
```toml
[general]
me_route_fairshare_enabled = false
```
## me_route_backpressure_base_timeout_ms
- **Constraints / validation**: Must be within `1..=5000` (milliseconds).
- **Description**: Base backpressure timeout in milliseconds for ME route-channel send.
@@ -1753,6 +1773,7 @@ This document lists all configuration keys accepted by `config.toml`.
| [`metrics_whitelist`](#metrics_whitelist) | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` |
| [`max_connections`](#max_connections) | `u32` | `10000` |
| [`accept_permit_timeout_ms`](#accept_permit_timeout_ms) | `u64` | `250` |
| [`listen_backlog`](#listen_backlog) | `u32` | `1024` |
## port
- **Constraints / validation**: `u16`.
@@ -1763,6 +1784,15 @@ This document lists all configuration keys accepted by `config.toml`.
[server]
port = 443
```
## listen_backlog
- **Constraints / validation**: `u32`. `0` uses the OS default backlog behavior.
- **Description**: Listen backlog passed to `listen(2)` for TCP sockets.
- **Example**:
```toml
[server]
listen_backlog = 1024
```
## listen_addr_ipv4
- **Constraints / validation**: `String` (optional). When set, must be a valid IPv4 address string.
- **Description**: IPv4 bind address for TCP listener (omit this key to disable IPv4 bind).
@@ -2005,6 +2035,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
| [`runtime_edge_top_n`](#runtime_edge_top_n) | `usize` | `10` |
| [`runtime_edge_events_capacity`](#runtime_edge_events_capacity) | `usize` | `256` |
| [`read_only`](#read_only) | `bool` | `false` |
| [`gray_action`](#gray_action) | `"drop"`, `"api"`, or `"200"` | `"drop"` |
## enabled
- **Constraints / validation**: `bool`.
@@ -2015,6 +2046,15 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
[server.api]
enabled = true
```
## gray_action
- **Constraints / validation**: `"drop"`, `"api"`, or `"200"`.
- **Description**: API response policy for gray/limited states: drop request, serve normal API response, or force `200 OK`.
- **Example**:
```toml
[server.api]
gray_action = "drop"
```
## listen
- **Constraints / validation**: `String`. Must be in `IP:PORT` format.
- **Description**: API bind address in `IP:PORT` format.
@@ -2207,6 +2247,15 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
[timeouts]
client_handshake = 30
```
## client_first_byte_idle_secs
- **Constraints / validation**: `u64` (seconds). `0` disables first-byte idle enforcement.
- **Description**: Maximum idle time to wait for the first client payload byte after session setup.
- **Example**:
```toml
[timeouts]
client_first_byte_idle_secs = 300
```
## relay_idle_policy_v2_enabled
- **Constraints / validation**: `bool`.
- **Description**: Enables soft/hard middle-relay client idle policy.
@@ -2311,6 +2360,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
| [`server_hello_delay_max_ms`](#server_hello_delay_max_ms) | `u64` | `0` |
| [`tls_new_session_tickets`](#tls_new_session_tickets) | `u8` | `0` |
| [`tls_full_cert_ttl_secs`](#tls_full_cert_ttl_secs) | `u64` | `90` |
| [`serverhello_compact`](#serverhello_compact) | `bool` | `false` |
| [`alpn_enforce`](#alpn_enforce) | `bool` | `true` |
| [`mask_proxy_protocol`](#mask_proxy_protocol) | `u8` | `0` |
| [`mask_shape_hardening`](#mask_shape_hardening) | `bool` | `true` |
@@ -2488,6 +2538,15 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
[censorship]
tls_full_cert_ttl_secs = 90
```
## serverhello_compact
- **Constraints / validation**: `bool`.
- **Description**: Enables compact ServerHello/Fake-TLS profile to reduce response-size signature.
- **Example**:
```toml
[censorship]
serverhello_compact = false
```
## alpn_enforce
- **Constraints / validation**: `bool`.
- **Description**: Enforces ALPN echo behavior based on client preference.
@@ -2827,9 +2886,12 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
| [`user_max_unique_ips_global_each`](#user_max_unique_ips_global_each) | `usize` | `0` |
| [`user_max_unique_ips_mode`](#user_max_unique_ips_mode) | `"active_window"`, `"time_window"`, or `"combined"` | `"active_window"` |
| [`user_max_unique_ips_window_secs`](#user_max_unique_ips_window_secs) | `u64` | `30` |
| [`user_source_deny`](#user_source_deny) | `Map<String, IpNetwork[]>` | `{}` |
| [`replay_check_len`](#replay_check_len) | `usize` | `65536` |
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` |
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` |
| [`user_rate_limits`](#user_rate_limits) | `Map<String, RateLimitBps>` | `{}` |
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<IpNetwork, RateLimitBps>` | `{}` |
## users
- **Constraints / validation**: Must not be empty (at least one user must exist). Each value must be **exactly 32 hex characters**.
@@ -2929,6 +2991,20 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
[access]
user_max_unique_ips_window_secs = 30
```
## user_source_deny
- **Constraints / validation**: Table `username -> IpNetwork[]`. Each network must parse as CIDR (for example `203.0.113.0/24` or `2001:db8::/32`).
- **Description**: Per-user source IP/CIDR deny-list applied **after successful auth** in TLS and MTProto handshake paths. A matched source IP is rejected via the same fail-closed path as invalid auth.
- **Example**:
```toml
[access.user_source_deny]
alice = ["203.0.113.0/24", "2001:db8:abcd::/48"]
bob = ["198.51.100.42/32"]
```
- **How it works (quick check)**:
- connection from user `alice` and source `203.0.113.55` -> rejected (matches `203.0.113.0/24`)
- connection from user `alice` and source `198.51.100.10` -> allowed by this rule set (no match)
## replay_check_len
- **Constraints / validation**: `usize`.
- **Description**: Replay-protection storage length (number of entries tracked for duplicate detection).
@@ -2958,6 +3034,24 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
```
## user_rate_limits
- **Constraints / validation**: Table `username -> { up_bps, down_bps }`. At least one direction must be non-zero.
- **Description**: Per-user bandwidth caps in bytes/sec for upload (`up_bps`) and download (`down_bps`).
- **Example**:
```toml
[access.user_rate_limits]
alice = { up_bps = 1048576, down_bps = 2097152 }
```
## cidr_rate_limits
- **Constraints / validation**: Table `CIDR -> { up_bps, down_bps }`. CIDR must parse as `IpNetwork`; at least one direction must be non-zero.
- **Description**: Source-subnet bandwidth caps applied alongside per-user limits.
- **Example**:
```toml
[access.cidr_rate_limits]
"203.0.113.0/24" = { up_bps = 0, down_bps = 1048576 }
```
# [[upstreams]]

View File

@@ -162,6 +162,8 @@
| [`log_level`](#log_level) | `"debug"`, `"verbose"`, `"normal"`, or `"silent"` | `"normal"` |
| [`disable_colors`](#disable_colors) | `bool` | `false` |
| [`me_socks_kdf_policy`](#me_socks_kdf_policy) | `"strict"` or `"compat"` | `"strict"` |
| [`me_route_backpressure_enabled`](#me_route_backpressure_enabled) | `bool` | `false` |
| [`me_route_fairshare_enabled`](#me_route_fairshare_enabled) | `bool` | `false` |
| [`me_route_backpressure_base_timeout_ms`](#me_route_backpressure_base_timeout_ms) | `u64` | `25` |
| [`me_route_backpressure_high_timeout_ms`](#me_route_backpressure_high_timeout_ms) | `u64` | `120` |
| [`me_route_backpressure_high_watermark_pct`](#me_route_backpressure_high_watermark_pct) | `u8` | `80` |
@@ -975,6 +977,24 @@
[general]
me_socks_kdf_policy = "strict"
```
## me_route_backpressure_enabled
- **Ограничения / валидация**: `bool`.
- **Описание**: Включает адаптивные таймауты записи маршрута в зависимости от заполнения канала.
- **Example**:
```toml
[general]
me_route_backpressure_enabled = false
```
## me_route_fairshare_enabled
- **Ограничения / валидация**: `bool`.
- **Описание**: Включает справедливое распределение нагрузки маршрутизации между writer-потоками.
- **Example**:
```toml
[general]
me_route_fairshare_enabled = false
```
## me_route_backpressure_base_timeout_ms
- **Ограничения / валидация**: Должно быть в пределах `1..=5000` (миллисекунд).
- **Описание**: Базовый таймаут (в миллисекундах) ожидания при режиме **backpressure** (ситуация, при которой данные обрабатываются медленне, чем получаются) для отправки через ME route-channel.
@@ -1755,6 +1775,7 @@
| [`metrics_whitelist`](#metrics_whitelist) | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` |
| [`max_connections`](#max_connections) | `u32` | `10000` |
| [`accept_permit_timeout_ms`](#accept_permit_timeout_ms) | `u64` | `250` |
| [`listen_backlog`](#listen_backlog) | `u32` | `1024` |
## port
- **Ограничения / валидация**: `u16`.
@@ -1765,6 +1786,15 @@
[server]
port = 443
```
## listen_backlog
- **Ограничения / валидация**: `u32`. `0` использует системный backlog по умолчанию.
- **Описание**: Значение backlog, передаваемое в `listen(2)` для TCP-сокетов.
- **Example**:
```toml
[server]
listen_backlog = 1024
```
## listen_addr_ipv4
- **Ограничения / валидация**: `String` (необязательный параметр). Если задан, должен содержать валидный IPv4-адрес в формате строки.
- **Описание**: Прослушиваемый адрес в формате IPv4 (не задавайте этот параметр, если необходимо отключить прослушивание по IPv4).
@@ -2011,6 +2041,7 @@
| [`runtime_edge_top_n`](#runtime_edge_top_n) | `usize` | `10` |
| [`runtime_edge_events_capacity`](#runtime_edge_events_capacity) | `usize` | `256` |
| [`read_only`](#read_only) | `bool` | `false` |
| [`gray_action`](#gray_action) | `"drop"`, `"api"`, or `"200"` | `"drop"` |
## enabled
- **Ограничения / валидация**: `bool`.
@@ -2021,6 +2052,15 @@
[server.api]
enabled = true
```
## gray_action
- **Ограничения / валидация**: `"drop"`, `"api"` или `"200"`.
- **Описание**: Политика ответа API в «серых» (ограниченных) состояниях: сброс, обычный API-ответ, либо `200 OK`.
- **Example**:
```toml
[server.api]
gray_action = "drop"
```
## listen
- **Ограничения / валидация**: `String`. Должно быть в формате `IP:PORT`.
- **Описание**: Адрес биндинга API в формате `IP:PORT`.
@@ -2213,6 +2253,15 @@
[timeouts]
client_handshake = 30
```
## client_first_byte_idle_secs
- **Ограничения / валидация**: `u64` (секунды). `0` отключает проверку простоя до первого байта.
- **Описание**: Максимальное время ожидания первого байта полезной нагрузки от клиента после установления сессии.
- **Example**:
```toml
[timeouts]
client_first_byte_idle_secs = 300
```
## relay_idle_policy_v2_enabled
- **Ограничения / валидация**: `bool`.
- **Описание**: Включает политику простоя клиента для промежуточного узла.
@@ -2317,6 +2366,7 @@
| [`server_hello_delay_max_ms`](#server_hello_delay_max_ms) | `u64` | `0` |
| [`tls_new_session_tickets`](#tls_new_session_tickets) | `u8` | `0` |
| [`tls_full_cert_ttl_secs`](#tls_full_cert_ttl_secs) | `u64` | `90` |
| [`serverhello_compact`](#serverhello_compact) | `bool` | `false` |
| [`alpn_enforce`](#alpn_enforce) | `bool` | `true` |
| [`mask_proxy_protocol`](#mask_proxy_protocol) | `u8` | `0` |
| [`mask_shape_hardening`](#mask_shape_hardening) | `bool` | `true` |
@@ -2493,6 +2543,15 @@
[censorship]
tls_full_cert_ttl_secs = 90
```
## serverhello_compact
- **Ограничения / валидация**: `bool`.
- **Описание**: Включает компактный профиль ServerHello/Fake-TLS для снижения сигнатуры размера ответа.
- **Example**:
```toml
[censorship]
serverhello_compact = false
```
## alpn_enforce
- **Ограничения / валидация**: `bool`.
- **Описание**: Принудительно изменяет поведение возврата ALPN в соответствии с предпочтениями клиента.
@@ -2837,6 +2896,8 @@
| [`replay_check_len`](#replay_check_len) | `usize` | `65536` |
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` |
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` |
| [`user_rate_limits`](#user_rate_limits) | `Map<String, RateLimitBps>` | `{}` |
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<IpNetwork, RateLimitBps>` | `{}` |
## users
- **Ограничения / валидация**: Не должно быть пустым (должен существовать хотя бы один пользователь). Каждое значение должно состоять **ровно из 32 шестнадцатеричных символов**.
@@ -2965,6 +3026,24 @@
```
## user_rate_limits
- **Ограничения / валидация**: Таблица `username -> { up_bps, down_bps }`. Должно быть ненулевое значение хотя бы в одном направлении.
- **Описание**: Персональные лимиты скорости по пользователям в байтах/сек для отправки (`up_bps`) и получения (`down_bps`).
- **Example**:
```toml
[access.user_rate_limits]
alice = { up_bps = 1048576, down_bps = 2097152 }
```
## cidr_rate_limits
- **Ограничения / валидация**: Таблица `CIDR -> { up_bps, down_bps }`. CIDR должен корректно разбираться как `IpNetwork`; хотя бы одно направление должно быть ненулевым.
- **Описание**: Лимиты скорости для подсетей источников, применяются поверх пользовательских ограничений.
- **Example**:
```toml
[access.cidr_rate_limits]
"203.0.113.0/24" = { up_bps = 0, down_bps = 1048576 }
```
# [[upstreams]]

View File

@@ -254,6 +254,19 @@ docker compose down
> - `docker-compose.yml` maps `./config.toml` to `/app/config.toml` (read-only)
> - By default it publishes `443:443` and runs with dropped capabilities (only `NET_BIND_SERVICE` is added)
> - If you really need host networking (usually only for some IPv6 setups) uncomment `network_mode: host`
> - If you enable mutating Control API endpoints, mount a writable config directory instead of a single `config.toml` file. Telemt persists config changes with atomic `tmp + rename` writes, and a single bind-mounted file can fail with `Device or resource busy`.
Example writable config mount for Control API mutations:
```yaml
services:
telemt:
working_dir: /run/telemt
volumes:
- ./config:/etc/telemt:rw
tmpfs:
- /run/telemt:rw,mode=1777,size=4m
command: /usr/local/bin/telemt /etc/telemt/config.toml
```
**Run without Compose**
```bash

View File

@@ -164,7 +164,7 @@ tls_front_dir = "tlsfront" # Директория кэша для эмуляц
hello = "00000000000000000000000000000000"
```
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
Затем нажмите Ctrl+O -> Ctrl+X, чтобы сохранить
> [!WARNING]
> Замените значение параметра `hello` на значение, которое вы получили в пункте 0.

View File

@@ -27,6 +27,8 @@ ACTION="install"
TARGET_VERSION="${VERSION:-latest}"
LANG_CHOICE="en"
PATH="${PATH}:/usr/sbin:/sbin"
set_language() {
case "$1" in
ru)
@@ -102,6 +104,7 @@ set_language() {
L_OUT_SUCC_H="УСТАНОВКА УСПЕШНО ЗАВЕРШЕНА"
L_OUT_UNINST_H="УДАЛЕНИЕ ЗАВЕРШЕНО"
L_OUT_LINK="Ваша ссылка для подключения к Telegram Proxy:\n"
L_ERR_INCORR_ROOT_LOGIN="Используйте 'su -' или 'sudo -i' для входа под пользователем root"
;;
*)
L_ERR_DOMAIN_REQ="requires a domain argument."
@@ -176,6 +179,7 @@ set_language() {
L_OUT_SUCC_H="INSTALLATION SUCCESS"
L_OUT_UNINST_H="UNINSTALLATION COMPLETE"
L_OUT_LINK="Your Telegram Proxy connection link:\n"
L_ERR_INCORR_ROOT_LOGIN="Use 'su -' or 'sudo -i' to login under root"
;;
esac
}
@@ -388,6 +392,9 @@ verify_common() {
if [ "$(id -u)" -eq 0 ]; then
SUDO=""
if [ "$(id -u)" -ne 0 ]; then
die "$L_ERR_INCORR_ROOT_LOGIN"
fi
else
command -v sudo >/dev/null 2>&1 || die "$L_ERR_ROOT"
SUDO="sudo"

View File

@@ -7,7 +7,7 @@ use hyper::header::IF_MATCH;
use serde::Serialize;
use sha2::{Digest, Sha256};
use crate::config::ProxyConfig;
use crate::config::{ProxyConfig, RateLimitBps};
use super::model::ApiFailure;
@@ -18,6 +18,7 @@ pub(super) enum AccessSection {
UserMaxTcpConns,
UserExpirations,
UserDataQuota,
UserRateLimits,
UserMaxUniqueIps,
}
@@ -29,6 +30,7 @@ impl AccessSection {
Self::UserMaxTcpConns => "access.user_max_tcp_conns",
Self::UserExpirations => "access.user_expirations",
Self::UserDataQuota => "access.user_data_quota",
Self::UserRateLimits => "access.user_rate_limits",
Self::UserMaxUniqueIps => "access.user_max_unique_ips",
}
}
@@ -82,6 +84,7 @@ pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyCon
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
}
#[allow(dead_code)]
pub(super) async fn save_config_to_disk(
config_path: &Path,
cfg: &ProxyConfig,
@@ -106,6 +109,12 @@ pub(super) async fn save_access_sections_to_disk(
if applied.contains(section) {
continue;
}
if find_toml_table_bounds(&content, section.table_name()).is_none()
&& access_section_is_empty(cfg, *section)
{
applied.push(*section);
continue;
}
let rendered = render_access_section(cfg, *section)?;
content = upsert_toml_table(&content, section.table_name(), &rendered);
applied.push(*section);
@@ -162,6 +171,15 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result<St
.collect();
serialize_table_body(&rows)?
}
AccessSection::UserRateLimits => {
let rows: BTreeMap<String, RateLimitBps> = cfg
.access
.user_rate_limits
.iter()
.map(|(key, value)| (key.clone(), *value))
.collect();
serialize_rate_limit_body(&rows)?
}
AccessSection::UserMaxUniqueIps => {
let rows: BTreeMap<String, usize> = cfg
.access
@@ -183,11 +201,45 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result<St
Ok(out)
}
fn access_section_is_empty(cfg: &ProxyConfig, section: AccessSection) -> bool {
match section {
AccessSection::Users => cfg.access.users.is_empty(),
AccessSection::UserAdTags => cfg.access.user_ad_tags.is_empty(),
AccessSection::UserMaxTcpConns => cfg.access.user_max_tcp_conns.is_empty(),
AccessSection::UserExpirations => cfg.access.user_expirations.is_empty(),
AccessSection::UserDataQuota => cfg.access.user_data_quota.is_empty(),
AccessSection::UserRateLimits => cfg.access.user_rate_limits.is_empty(),
AccessSection::UserMaxUniqueIps => cfg.access.user_max_unique_ips.is_empty(),
}
}
fn serialize_table_body<T: Serialize>(value: &T) -> Result<String, ApiFailure> {
toml::to_string(value)
.map_err(|e| ApiFailure::internal(format!("failed to serialize access section: {}", e)))
}
fn serialize_rate_limit_body(rows: &BTreeMap<String, RateLimitBps>) -> Result<String, ApiFailure> {
let mut out = String::new();
for (key, value) in rows {
let key = serialize_toml_key(key)?;
out.push_str(&format!(
"{key} = {{ up_bps = {}, down_bps = {} }}\n",
value.up_bps, value.down_bps
));
}
Ok(out)
}
fn serialize_toml_key(key: &str) -> Result<String, ApiFailure> {
let mut row = BTreeMap::new();
row.insert(key.to_string(), 0_u8);
let rendered = serialize_table_body(&row)?;
rendered
.split_once(" = ")
.map(|(key, _)| key.to_string())
.ok_or_else(|| ApiFailure::internal("failed to serialize TOML key"))
}
fn upsert_toml_table(source: &str, table_name: &str, replacement: &str) -> String {
if let Some((start, end)) = find_toml_table_bounds(source, table_name) {
let mut out = String::with_capacity(source.len() + replacement.len());
@@ -267,3 +319,26 @@ fn write_atomic_sync(path: &Path, contents: &str) -> std::io::Result<()> {
}
write_result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn render_user_rate_limits_section() {
let mut cfg = ProxyConfig::default();
cfg.access.user_rate_limits.insert(
"alice".to_string(),
RateLimitBps {
up_bps: 1024,
down_bps: 2048,
},
);
let rendered = render_access_section(&cfg, AccessSection::UserRateLimits)
.expect("section must render");
assert!(rendered.starts_with("[access.user_rate_limits]\n"));
assert!(rendered.contains("alice = { up_bps = 1024, down_bps = 2048 }"));
}
}

View File

@@ -5,6 +5,7 @@ use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::Duration;
use http_body_util::Full;
use hyper::body::{Bytes, Incoming};
@@ -12,8 +13,10 @@ use hyper::header::AUTHORIZATION;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper::{Method, Request, Response, StatusCode};
use subtle::ConstantTimeEq;
use tokio::net::TcpListener;
use tokio::sync::{Mutex, RwLock, watch};
use tokio::sync::{Mutex, RwLock, Semaphore, watch};
use tokio::time::timeout;
use tracing::{debug, info, warn};
use crate::config::{ApiGrayAction, ProxyConfig};
@@ -43,7 +46,8 @@ use events::ApiEventStore;
use http_utils::{error_response, read_json, read_optional_json, success_response};
use model::{
ApiFailure, ClassCount, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
PatchUserRequest, RotateSecretRequest, SummaryData, UserActiveIps,
PatchUserRequest, ResetUserQuotaResponse, RotateSecretRequest, SummaryData, UserActiveIps,
is_valid_username,
};
use runtime_edge::{
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
@@ -64,7 +68,13 @@ use runtime_zero::{
build_limits_effective_data, build_runtime_gates_data, build_security_posture_data,
build_system_info_data,
};
use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config};
use users::{
build_user_quota_list, create_user, delete_user, patch_user, rotate_secret, users_from_config,
};
const API_MAX_CONTROL_CONNECTIONS: usize = 1024;
const API_HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
const ROUTE_USERNAME_ERROR: &str = "username must match [A-Za-z0-9_.-] and be 1..64 chars";
pub(super) struct ApiRuntimeState {
pub(super) process_started_at_epoch_secs: u64,
@@ -80,6 +90,7 @@ pub(super) struct ApiShared {
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
pub(super) upstream_manager: Arc<UpstreamManager>,
pub(super) config_path: PathBuf,
pub(super) quota_state_path: PathBuf,
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
pub(super) mutation_lock: Arc<Mutex<()>>,
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
@@ -102,6 +113,18 @@ impl ApiShared {
}
}
fn auth_header_matches(actual: &str, expected: &str) -> bool {
actual.as_bytes().ct_eq(expected.as_bytes()).into()
}
fn parse_route_username(user: &str) -> Result<&str, ApiFailure> {
if is_valid_username(user) {
Ok(user)
} else {
Err(ApiFailure::bad_request(ROUTE_USERNAME_ERROR))
}
}
pub async fn serve(
listen: SocketAddr,
stats: Arc<Stats>,
@@ -112,6 +135,7 @@ pub async fn serve(
config_rx: watch::Receiver<Arc<ProxyConfig>>,
admission_rx: watch::Receiver<bool>,
config_path: PathBuf,
quota_state_path: PathBuf,
detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
process_started_at_epoch_secs: u64,
startup_tracker: Arc<StartupTracker>,
@@ -143,6 +167,7 @@ pub async fn serve(
me_pool,
upstream_manager,
config_path,
quota_state_path,
detected_ips_rx,
mutation_lock: Arc::new(Mutex::new(())),
minimal_cache: Arc::new(Mutex::new(None)),
@@ -164,6 +189,8 @@ pub async fn serve(
shared.runtime_events.clone(),
);
let connection_permits = Arc::new(Semaphore::new(API_MAX_CONTROL_CONNECTIONS));
loop {
let (stream, peer) = match listener.accept().await {
Ok(v) => v,
@@ -173,20 +200,45 @@ pub async fn serve(
}
};
let connection_permit = match connection_permits.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
debug!(
peer = %peer,
max_connections = API_MAX_CONTROL_CONNECTIONS,
"Dropping API connection: control-plane connection budget exhausted"
);
continue;
}
};
let shared_conn = shared.clone();
let config_rx_conn = config_rx.clone();
tokio::spawn(async move {
let _connection_permit = connection_permit;
let svc = service_fn(move |req: Request<Incoming>| {
let shared_req = shared_conn.clone();
let config_rx_req = config_rx_conn.clone();
async move { handle(req, peer, shared_req, config_rx_req).await }
});
if let Err(error) = http1::Builder::new()
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
.await
match timeout(
API_HTTP_CONNECTION_TIMEOUT,
http1::Builder::new().serve_connection(hyper_util::rt::TokioIo::new(stream), svc),
)
.await
{
if !error.is_user() {
debug!(error = %error, "API connection error");
Ok(Ok(())) => {}
Ok(Err(error)) => {
if !error.is_user() {
debug!(error = %error, "API connection error");
}
}
Err(_) => {
debug!(
peer = %peer,
timeout_ms = API_HTTP_CONNECTION_TIMEOUT.as_millis() as u64,
"API connection timed out"
);
}
}
});
@@ -242,7 +294,7 @@ async fn handle(
.headers()
.get(AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.map(|v| v == api_cfg.auth_header)
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
.unwrap_or(false);
if !auth_ok {
return Ok(error_response(
@@ -454,6 +506,12 @@ async fn handle(
.await;
Ok(success_response(StatusCode::OK, users, revision))
}
("GET", "/v1/users/quota") => {
let revision = current_revision(&shared.config_path).await?;
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
Ok(success_response(StatusCode::OK, data, revision))
}
("POST", "/v1/users") => {
if api_cfg.read_only {
return Ok(error_response(
@@ -491,10 +549,115 @@ async fn handle(
Ok(success_response(status, data, revision))
}
_ => {
if method == Method::POST
&& let Some(user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/reset-quota"))
&& !user.is_empty()
&& !user.contains('/')
{
let user = parse_route_username(user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let snapshot = match crate::quota_state::reset_user_quota(
&shared.quota_state_path,
shared.stats.as_ref(),
user,
)
.await
{
Ok(snapshot) => snapshot,
Err(error) => {
shared.runtime_events.record(
"api.user.reset_quota.failed",
format!("username={} error={}", user, error),
);
return Err(ApiFailure::internal(format!(
"Failed to reset user quota: {}",
error
)));
}
};
shared
.runtime_events
.record("api.user.reset_quota.ok", format!("username={}", user));
let revision = current_revision(&shared.config_path).await?;
return Ok(success_response(
StatusCode::OK,
ResetUserQuotaResponse {
username: user.to_string(),
used_bytes: snapshot.used_bytes,
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
},
revision,
));
}
if method == Method::POST
&& let Some(base_user) = normalized_path
.strip_prefix("/v1/users/")
.and_then(|path| path.strip_suffix("/rotate-secret"))
&& !base_user.is_empty()
&& !base_user.contains('/')
{
let base_user = parse_route_username(base_user)?;
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body =
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
.await?;
let result = rotate_secret(
base_user,
body.unwrap_or_default(),
expected_revision,
&shared,
)
.await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.rotate_secret.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.user.in_runtime =
runtime_cfg.access.users.contains_key(&data.user.username);
shared.runtime_events.record(
"api.user.rotate_secret.ok",
format!("username={}", base_user),
);
let status = if data.user.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
&& !user.is_empty()
&& !user.contains('/')
{
let user = parse_route_username(user)?;
if method == Method::GET {
let revision = current_revision(&shared.config_path).await?;
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
@@ -595,56 +758,6 @@ async fn handle(
};
return Ok(success_response(status, response, revision));
}
if method == Method::POST
&& let Some(base_user) = user.strip_suffix("/rotate-secret")
&& !base_user.is_empty()
&& !base_user.contains('/')
{
if api_cfg.read_only {
return Ok(error_response(
request_id,
ApiFailure::new(
StatusCode::FORBIDDEN,
"read_only",
"API runs in read-only mode",
),
));
}
let expected_revision = parse_if_match(req.headers());
let body =
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
.await?;
let result = rotate_secret(
base_user,
body.unwrap_or_default(),
expected_revision,
&shared,
)
.await;
let (mut data, revision) = match result {
Ok(ok) => ok,
Err(error) => {
shared.runtime_events.record(
"api.user.rotate_secret.failed",
format!("username={} code={}", base_user, error.code),
);
return Err(error);
}
};
let runtime_cfg = config_rx.borrow().clone();
data.user.in_runtime =
runtime_cfg.access.users.contains_key(&data.user.username);
shared.runtime_events.record(
"api.user.rotate_secret.ok",
format!("username={}", base_user),
);
let status = if data.user.in_runtime {
StatusCode::OK
} else {
StatusCode::ACCEPTED
};
return Ok(success_response(status, data, revision));
}
if method == Method::POST {
return Ok(error_response(
request_id,

View File

@@ -456,6 +456,13 @@ pub(super) struct UserLinks {
pub(super) classic: Vec<String>,
pub(super) secure: Vec<String>,
pub(super) tls: Vec<String>,
pub(super) tls_domains: Vec<TlsDomainLink>,
}
#[derive(Serialize)]
pub(super) struct TlsDomainLink {
pub(super) domain: String,
pub(super) link: String,
}
#[derive(Serialize)]
@@ -466,6 +473,8 @@ pub(super) struct UserInfo {
pub(super) max_tcp_conns: Option<usize>,
pub(super) expiration_rfc3339: Option<String>,
pub(super) data_quota_bytes: Option<u64>,
pub(super) rate_limit_up_bps: Option<u64>,
pub(super) rate_limit_down_bps: Option<u64>,
pub(super) max_unique_ips: Option<usize>,
pub(super) current_connections: u64,
pub(super) active_unique_ips: usize,
@@ -494,6 +503,26 @@ pub(super) struct DeleteUserResponse {
pub(super) in_runtime: bool,
}
#[derive(Serialize)]
pub(super) struct ResetUserQuotaResponse {
pub(super) username: String,
pub(super) used_bytes: u64,
pub(super) last_reset_epoch_secs: u64,
}
#[derive(Serialize)]
pub(super) struct UserQuotaListData {
pub(super) users: Vec<UserQuotaEntry>,
}
#[derive(Serialize)]
pub(super) struct UserQuotaEntry {
pub(super) username: String,
pub(super) data_quota_bytes: u64,
pub(super) used_bytes: u64,
pub(super) last_reset_epoch_secs: u64,
}
#[derive(Deserialize)]
pub(super) struct CreateUserRequest {
pub(super) username: String,
@@ -502,6 +531,8 @@ pub(super) struct CreateUserRequest {
pub(super) max_tcp_conns: Option<usize>,
pub(super) expiration_rfc3339: Option<String>,
pub(super) data_quota_bytes: Option<u64>,
pub(super) rate_limit_up_bps: Option<u64>,
pub(super) rate_limit_down_bps: Option<u64>,
pub(super) max_unique_ips: Option<usize>,
}
@@ -517,6 +548,10 @@ pub(super) struct PatchUserRequest {
#[serde(default, deserialize_with = "patch_field")]
pub(super) data_quota_bytes: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) rate_limit_up_bps: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) rate_limit_down_bps: Patch<u64>,
#[serde(default, deserialize_with = "patch_field")]
pub(super) max_unique_ips: Patch<usize>,
}

View File

@@ -114,7 +114,9 @@ mod tests {
"secret": "00112233445566778899aabbccddeeff",
"max_tcp_conns": 0,
"max_unique_ips": null,
"data_quota_bytes": 1024
"data_quota_bytes": 1024,
"rate_limit_up_bps": 4096,
"rate_limit_down_bps": null
}"#;
let req: PatchUserRequest = serde_json::from_str(raw).expect("valid json");
assert_eq!(
@@ -124,6 +126,8 @@ mod tests {
assert!(matches!(req.max_tcp_conns, Patch::Set(0)));
assert!(matches!(req.max_unique_ips, Patch::Remove));
assert!(matches!(req.data_quota_bytes, Patch::Set(1024)));
assert!(matches!(req.rate_limit_up_bps, Patch::Set(4096)));
assert!(matches!(req.rate_limit_down_bps, Patch::Remove));
assert!(matches!(req.expiration_rfc3339, Patch::Unchanged));
assert!(matches!(req.user_ad_tag, Patch::Unchanged));
}

View File

@@ -3,18 +3,20 @@ use std::net::IpAddr;
use hyper::StatusCode;
use crate::config::ProxyConfig;
use crate::config::RateLimitBps;
use crate::ip_tracker::UserIpTracker;
use crate::stats::Stats;
use super::ApiShared;
use super::config_store::{
AccessSection, ensure_expected_revision, load_config_from_disk, save_access_sections_to_disk,
save_config_to_disk,
AccessSection, current_revision, ensure_expected_revision, load_config_from_disk,
save_access_sections_to_disk,
};
use super::model::{
ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest,
UserInfo, UserLinks, is_valid_ad_tag, is_valid_user_secret, is_valid_username,
parse_optional_expiration, parse_patch_expiration, random_user_secret,
TlsDomainLink, UserInfo, UserLinks, UserQuotaEntry, UserQuotaListData, is_valid_ad_tag,
is_valid_user_secret, is_valid_username, parse_optional_expiration, parse_patch_expiration,
random_user_secret,
};
use super::patch::Patch;
@@ -27,6 +29,8 @@ pub(super) async fn create_user(
let touches_user_max_tcp_conns = body.max_tcp_conns.is_some();
let touches_user_expirations = body.expiration_rfc3339.is_some();
let touches_user_data_quota = body.data_quota_bytes.is_some();
let touches_user_rate_limits =
body.rate_limit_up_bps.is_some() || body.rate_limit_down_bps.is_some();
let touches_user_max_unique_ips = body.max_unique_ips.is_some();
if !is_valid_username(&body.username) {
@@ -91,6 +95,15 @@ pub(super) async fn create_user(
.user_data_quota
.insert(body.username.clone(), quota);
}
if touches_user_rate_limits {
cfg.access.user_rate_limits.insert(
body.username.clone(),
RateLimitBps {
up_bps: body.rate_limit_up_bps.unwrap_or(0),
down_bps: body.rate_limit_down_bps.unwrap_or(0),
},
);
}
let updated_limit = body.max_unique_ips;
if let Some(limit) = updated_limit {
@@ -115,6 +128,9 @@ pub(super) async fn create_user(
if touches_user_data_quota {
touched_sections.push(AccessSection::UserDataQuota);
}
if touches_user_rate_limits {
touched_sections.push(AccessSection::UserRateLimits);
}
if touches_user_max_unique_ips {
touched_sections.push(AccessSection::UserMaxUniqueIps);
}
@@ -157,6 +173,8 @@ pub(super) async fn create_user(
.then_some(cfg.access.user_max_tcp_conns_global_each)),
expiration_rfc3339: None,
data_quota_bytes: None,
rate_limit_up_bps: body.rate_limit_up_bps.filter(|limit| *limit > 0),
rate_limit_down_bps: body.rate_limit_down_bps.filter(|limit| *limit > 0),
max_unique_ips: updated_limit,
current_connections: 0,
active_unique_ips: 0,
@@ -176,6 +194,15 @@ pub(super) async fn patch_user(
expected_revision: Option<String>,
shared: &ApiShared,
) -> Result<(UserInfo, String), ApiFailure> {
let touches_users = body.secret.is_some();
let touches_user_ad_tags = !matches!(&body.user_ad_tag, Patch::Unchanged);
let touches_user_max_tcp_conns = !matches!(&body.max_tcp_conns, Patch::Unchanged);
let touches_user_expirations = !matches!(&body.expiration_rfc3339, Patch::Unchanged);
let touches_user_data_quota = !matches!(&body.data_quota_bytes, Patch::Unchanged);
let touches_user_rate_limits = !matches!(&body.rate_limit_up_bps, Patch::Unchanged)
|| !matches!(&body.rate_limit_down_bps, Patch::Unchanged);
let touches_user_max_unique_ips = !matches!(&body.max_unique_ips, Patch::Unchanged);
if let Some(secret) = body.secret.as_ref()
&& !is_valid_user_secret(secret)
{
@@ -246,6 +273,31 @@ pub(super) async fn patch_user(
cfg.access.user_data_quota.insert(user.to_string(), quota);
}
}
if touches_user_rate_limits {
let mut rate_limit = cfg
.access
.user_rate_limits
.get(user)
.copied()
.unwrap_or_default();
match body.rate_limit_up_bps {
Patch::Unchanged => {}
Patch::Remove => rate_limit.up_bps = 0,
Patch::Set(limit) => rate_limit.up_bps = limit,
}
match body.rate_limit_down_bps {
Patch::Unchanged => {}
Patch::Remove => rate_limit.down_bps = 0,
Patch::Set(limit) => rate_limit.down_bps = limit,
}
if rate_limit.up_bps == 0 && rate_limit.down_bps == 0 {
cfg.access.user_rate_limits.remove(user);
} else {
cfg.access
.user_rate_limits
.insert(user.to_string(), rate_limit);
}
}
// Capture how the per-user IP limit changed, so the in-memory ip_tracker
// can be synced (set or removed) after the config is persisted.
let max_unique_ips_change = match body.max_unique_ips {
@@ -265,7 +317,34 @@ pub(super) async fn patch_user(
cfg.validate()
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
let revision = save_config_to_disk(&shared.config_path, &cfg).await?;
let mut touched_sections = Vec::new();
if touches_users {
touched_sections.push(AccessSection::Users);
}
if touches_user_ad_tags {
touched_sections.push(AccessSection::UserAdTags);
}
if touches_user_max_tcp_conns {
touched_sections.push(AccessSection::UserMaxTcpConns);
}
if touches_user_expirations {
touched_sections.push(AccessSection::UserExpirations);
}
if touches_user_data_quota {
touched_sections.push(AccessSection::UserDataQuota);
}
if touches_user_rate_limits {
touched_sections.push(AccessSection::UserRateLimits);
}
if touches_user_max_unique_ips {
touched_sections.push(AccessSection::UserMaxUniqueIps);
}
let revision = if touched_sections.is_empty() {
current_revision(&shared.config_path).await?
} else {
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?
};
drop(_guard);
match max_unique_ips_change {
Some(Some(limit)) => shared.ip_tracker.set_user_limit(user, limit).await,
@@ -324,6 +403,7 @@ pub(super) async fn rotate_secret(
AccessSection::UserMaxTcpConns,
AccessSection::UserExpirations,
AccessSection::UserDataQuota,
AccessSection::UserRateLimits,
AccessSection::UserMaxUniqueIps,
];
let revision =
@@ -383,6 +463,7 @@ pub(super) async fn delete_user(
cfg.access.user_max_tcp_conns.remove(user);
cfg.access.user_expirations.remove(user);
cfg.access.user_data_quota.remove(user);
cfg.access.user_rate_limits.remove(user);
cfg.access.user_max_unique_ips.remove(user);
cfg.validate()
@@ -393,6 +474,7 @@ pub(super) async fn delete_user(
AccessSection::UserMaxTcpConns,
AccessSection::UserExpirations,
AccessSection::UserDataQuota,
AccessSection::UserRateLimits,
AccessSection::UserMaxUniqueIps,
];
let revision =
@@ -434,11 +516,7 @@ pub(super) async fn users_from_config(
.map(|secret| {
build_user_links(cfg, secret, startup_detected_ip_v4, startup_detected_ip_v6)
})
.unwrap_or(UserLinks {
classic: Vec::new(),
secure: Vec::new(),
tls: Vec::new(),
});
.unwrap_or_else(empty_user_links);
users.push(UserInfo {
in_runtime: runtime_cfg
.map(|runtime| runtime.access.users.contains_key(&username))
@@ -458,6 +536,18 @@ pub(super) async fn users_from_config(
.get(&username)
.map(chrono::DateTime::<chrono::Utc>::to_rfc3339),
data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(),
rate_limit_up_bps: cfg
.access
.user_rate_limits
.get(&username)
.map(|limit| limit.up_bps)
.filter(|limit| *limit > 0),
rate_limit_down_bps: cfg
.access
.user_rate_limits
.get(&username)
.map(|limit| limit.down_bps)
.filter(|limit| *limit > 0),
max_unique_ips: cfg
.access
.user_max_unique_ips
@@ -479,6 +569,42 @@ pub(super) async fn users_from_config(
users
}
pub(super) fn build_user_quota_list(cfg: &ProxyConfig, stats: &Stats) -> UserQuotaListData {
let mut names = cfg.access.users.keys().cloned().collect::<Vec<_>>();
names.sort();
let snapshot = stats.user_quota_snapshot();
let mut users = Vec::with_capacity(names.len());
for username in names {
let Some(&data_quota_bytes) = cfg.access.user_data_quota.get(&username) else {
continue;
};
if data_quota_bytes == 0 {
continue;
}
let (used_bytes, last_reset_epoch_secs) = snapshot
.get(&username)
.map(|entry| (entry.used_bytes, entry.last_reset_epoch_secs))
.unwrap_or((0, 0));
users.push(UserQuotaEntry {
username,
data_quota_bytes,
used_bytes,
last_reset_epoch_secs,
});
}
UserQuotaListData { users }
}
fn empty_user_links() -> UserLinks {
UserLinks {
classic: Vec::new(),
secure: Vec::new(),
tls: Vec::new(),
tls_domains: Vec::new(),
}
}
fn build_user_links(
cfg: &ProxyConfig,
secret: &str,
@@ -492,10 +618,12 @@ fn build_user_links(
.public_port
.unwrap_or(resolve_default_link_port(cfg));
let tls_domains = resolve_tls_domains(cfg);
let extra_tls_domains = resolve_extra_tls_domains(cfg);
let mut classic = Vec::new();
let mut secure = Vec::new();
let mut tls = Vec::new();
let mut tls_domain_links = Vec::new();
for host in &hosts {
if cfg.general.modes.classic {
@@ -518,6 +646,17 @@ fn build_user_links(
host, port, secret, domain_hex
));
}
for domain in &extra_tls_domains {
let domain_hex = hex::encode(domain);
let link = format!(
"tg://proxy?server={}&port={}&secret=ee{}{}",
host, port, secret, domain_hex
);
tls_domain_links.push(TlsDomainLink {
domain: (*domain).to_string(),
link,
});
}
}
}
@@ -525,6 +664,7 @@ fn build_user_links(
classic,
secure,
tls,
tls_domains: tls_domain_links,
}
}
@@ -641,6 +781,19 @@ fn resolve_tls_domains(cfg: &ProxyConfig) -> Vec<&str> {
domains
}
fn resolve_extra_tls_domains(cfg: &ProxyConfig) -> Vec<&str> {
let mut domains = Vec::with_capacity(cfg.censorship.tls_domains.len());
let primary = cfg.censorship.tls_domain.as_str();
for domain in &cfg.censorship.tls_domains {
let value = domain.as_str();
if value.is_empty() || value == primary || domains.contains(&value) {
continue;
}
domains.push(value);
}
domains
}
#[cfg(test)]
mod tests {
use super::*;
@@ -695,6 +848,34 @@ mod tests {
assert_eq!(alice.max_tcp_conns, None);
}
#[tokio::test]
async fn users_from_config_reports_user_rate_limits() {
let mut cfg = ProxyConfig::default();
cfg.access.users.insert(
"alice".to_string(),
"0123456789abcdef0123456789abcdef".to_string(),
);
cfg.access.user_rate_limits.insert(
"alice".to_string(),
RateLimitBps {
up_bps: 1024,
down_bps: 0,
},
);
let stats = Stats::new();
let tracker = UserIpTracker::new();
let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await;
let alice = users
.iter()
.find(|entry| entry.username == "alice")
.expect("alice must be present");
assert_eq!(alice.rate_limit_up_bps, Some(1024));
assert_eq!(alice.rate_limit_down_bps, None);
}
#[tokio::test]
async fn users_from_config_marks_runtime_membership_when_snapshot_is_provided() {
let mut disk_cfg = ProxyConfig::default();
@@ -730,4 +911,144 @@ mod tests {
assert!(alice.in_runtime);
assert!(!bob.in_runtime);
}
#[tokio::test]
async fn users_from_config_returns_tls_link_for_each_tls_domain() {
let mut cfg = ProxyConfig::default();
cfg.access.users.insert(
"alice".to_string(),
"0123456789abcdef0123456789abcdef".to_string(),
);
cfg.general.modes.classic = false;
cfg.general.modes.secure = false;
cfg.general.modes.tls = true;
cfg.general.links.public_host = Some("proxy.example.net".to_string());
cfg.general.links.public_port = Some(443);
cfg.censorship.tls_domain = "front-a.example.com".to_string();
cfg.censorship.tls_domains = vec![
"front-b.example.com".to_string(),
"front-c.example.com".to_string(),
"front-b.example.com".to_string(),
"front-a.example.com".to_string(),
];
let stats = Stats::new();
let tracker = UserIpTracker::new();
let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await;
let alice = users
.iter()
.find(|entry| entry.username == "alice")
.expect("alice must be present");
assert_eq!(alice.links.tls.len(), 3);
assert!(
alice
.links
.tls
.iter()
.any(|link| link.ends_with(&hex::encode("front-a.example.com")))
);
assert!(
alice
.links
.tls
.iter()
.any(|link| link.ends_with(&hex::encode("front-b.example.com")))
);
assert!(
alice
.links
.tls
.iter()
.any(|link| link.ends_with(&hex::encode("front-c.example.com")))
);
assert_eq!(alice.links.tls_domains.len(), 2);
assert!(
alice
.links
.tls_domains
.iter()
.any(|entry| entry.domain == "front-b.example.com"
&& entry.link.ends_with(&hex::encode("front-b.example.com")))
);
assert!(
alice
.links
.tls_domains
.iter()
.any(|entry| entry.domain == "front-c.example.com"
&& entry.link.ends_with(&hex::encode("front-c.example.com")))
);
assert!(
!alice
.links
.tls_domains
.iter()
.any(|entry| entry.domain == "front-a.example.com")
);
}
#[test]
fn build_user_quota_list_skips_users_without_positive_quota_and_sorts_by_username() {
let mut cfg = ProxyConfig::default();
cfg.access.users.insert(
"alice".to_string(),
"0123456789abcdef0123456789abcdef".to_string(),
);
cfg.access.users.insert(
"bob".to_string(),
"fedcba9876543210fedcba9876543210".to_string(),
);
cfg.access.users.insert(
"carol".to_string(),
"aaaabbbbccccddddeeeeffff00001111".to_string(),
);
// alice has a positive quota and should be listed.
cfg.access
.user_data_quota
.insert("alice".to_string(), 1 << 20);
// bob has no quota entry at all (None) — should be skipped.
// carol has an explicit zero quota — should be skipped.
cfg.access.user_data_quota.insert("carol".to_string(), 0);
let stats = Stats::new();
// Charge some traffic against alice; carol gets traffic too but should
// still be filtered out by the quota check.
let alice_stats = stats.get_or_create_user_stats_handle("alice");
stats.quota_charge_post_write(&alice_stats, 4096);
let carol_stats = stats.get_or_create_user_stats_handle("carol");
stats.quota_charge_post_write(&carol_stats, 99);
let data = build_user_quota_list(&cfg, &stats);
assert_eq!(data.users.len(), 1);
let entry = &data.users[0];
assert_eq!(entry.username, "alice");
assert_eq!(entry.data_quota_bytes, 1 << 20);
assert_eq!(entry.used_bytes, 4096);
assert_eq!(entry.last_reset_epoch_secs, 0);
}
#[test]
fn build_user_quota_list_orders_multiple_users_by_username_ascending() {
let mut cfg = ProxyConfig::default();
for name in ["charlie", "alice", "bob"] {
cfg.access.users.insert(
name.to_string(),
"0123456789abcdef0123456789abcdef".to_string(),
);
cfg.access.user_data_quota.insert(name.to_string(), 1 << 30);
}
let stats = Stats::new();
let data = build_user_quota_list(&cfg, &stats);
let names: Vec<&str> = data.users.iter().map(|e| e.username.as_str()).collect();
assert_eq!(names, vec!["alice", "bob", "charlie"]);
for entry in &data.users {
assert_eq!(entry.used_bytes, 0);
assert_eq!(entry.last_reset_epoch_secs, 0);
assert_eq!(entry.data_quota_bytes, 1 << 30);
}
}
}

View File

@@ -102,7 +102,7 @@ pub(crate) fn default_fake_cert_len() -> usize {
}
pub(crate) fn default_tls_front_dir() -> String {
"/etc/telemt/tlsfront".to_string()
"tlsfront".to_string()
}
pub(crate) fn default_replay_check_len() -> usize {
@@ -568,7 +568,7 @@ pub(crate) fn default_beobachten_flush_secs() -> u64 {
}
pub(crate) fn default_beobachten_file() -> String {
"/etc/telemt/beobachten.txt".to_string()
"beobachten.txt".to_string()
}
pub(crate) fn default_tls_new_session_tickets() -> u8 {

View File

@@ -22,6 +22,672 @@ const MAX_ME_ROUTE_CHANNEL_CAPACITY: usize = 8_192;
const MAX_ME_C2ME_CHANNEL_CAPACITY: usize = 8_192;
const MIN_MAX_CLIENT_FRAME_BYTES: usize = 4 * 1024;
const MAX_MAX_CLIENT_FRAME_BYTES: usize = 16 * 1024 * 1024;
const MAX_API_REQUEST_BODY_LIMIT_BYTES: usize = 1024 * 1024;
fn is_valid_tls_domain_name(domain: &str) -> bool {
!domain.is_empty()
&& !domain
.chars()
.any(|ch| ch.is_whitespace() || matches!(ch, '/' | '\\'))
}
const TOP_LEVEL_CONFIG_KEYS: &[&str] = &[
"general",
"network",
"server",
"timeouts",
"censorship",
"access",
"upstreams",
"show_link",
"dc_overrides",
"default_dc",
"beobachten",
"beobachten_minutes",
"beobachten_flush_secs",
"beobachten_file",
"include",
];
const GENERAL_CONFIG_KEYS: &[&str] = &[
"data_path",
"quota_state_path",
"config_strict",
"modes",
"prefer_ipv6",
"fast_mode",
"use_middle_proxy",
"proxy_secret_path",
"proxy_secret_url",
"proxy_config_v4_cache_path",
"proxy_config_v4_url",
"proxy_config_v6_cache_path",
"proxy_config_v6_url",
"ad_tag",
"middle_proxy_nat_ip",
"middle_proxy_nat_probe",
"middle_proxy_nat_stun",
"middle_proxy_nat_stun_servers",
"stun_nat_probe_concurrency",
"middle_proxy_pool_size",
"middle_proxy_warm_standby",
"me_init_retry_attempts",
"me2dc_fallback",
"me2dc_fast",
"me_keepalive_enabled",
"me_keepalive_interval_secs",
"me_keepalive_jitter_secs",
"me_keepalive_payload_random",
"rpc_proxy_req_every",
"me_writer_cmd_channel_capacity",
"me_route_channel_capacity",
"me_c2me_channel_capacity",
"me_c2me_send_timeout_ms",
"me_reader_route_data_wait_ms",
"me_d2c_flush_batch_max_frames",
"me_d2c_flush_batch_max_bytes",
"me_d2c_flush_batch_max_delay_us",
"me_d2c_ack_flush_immediate",
"me_quota_soft_overshoot_bytes",
"me_d2c_frame_buf_shrink_threshold_bytes",
"direct_relay_copy_buf_c2s_bytes",
"direct_relay_copy_buf_s2c_bytes",
"crypto_pending_buffer",
"max_client_frame",
"desync_all_full",
"beobachten",
"beobachten_minutes",
"beobachten_flush_secs",
"beobachten_file",
"hardswap",
"me_warmup_stagger_enabled",
"me_warmup_step_delay_ms",
"me_warmup_step_jitter_ms",
"me_reconnect_max_concurrent_per_dc",
"me_reconnect_backoff_base_ms",
"me_reconnect_backoff_cap_ms",
"me_reconnect_fast_retry_count",
"me_single_endpoint_shadow_writers",
"me_single_endpoint_outage_mode_enabled",
"me_single_endpoint_outage_disable_quarantine",
"me_single_endpoint_outage_backoff_min_ms",
"me_single_endpoint_outage_backoff_max_ms",
"me_single_endpoint_shadow_rotate_every_secs",
"me_floor_mode",
"me_adaptive_floor_idle_secs",
"me_adaptive_floor_min_writers_single_endpoint",
"me_adaptive_floor_min_writers_multi_endpoint",
"me_adaptive_floor_recover_grace_secs",
"me_adaptive_floor_writers_per_core_total",
"me_adaptive_floor_cpu_cores_override",
"me_adaptive_floor_max_extra_writers_single_per_core",
"me_adaptive_floor_max_extra_writers_multi_per_core",
"me_adaptive_floor_max_active_writers_per_core",
"me_adaptive_floor_max_warm_writers_per_core",
"me_adaptive_floor_max_active_writers_global",
"me_adaptive_floor_max_warm_writers_global",
"upstream_connect_retry_attempts",
"upstream_connect_retry_backoff_ms",
"upstream_connect_budget_ms",
"tg_connect",
"upstream_unhealthy_fail_threshold",
"upstream_connect_failfast_hard_errors",
"stun_iface_mismatch_ignore",
"unknown_dc_log_path",
"unknown_dc_file_log_enabled",
"log_level",
"disable_colors",
"telemetry",
"me_socks_kdf_policy",
"me_route_backpressure_enabled",
"me_route_fairshare_enabled",
"me_route_backpressure_base_timeout_ms",
"me_route_backpressure_high_timeout_ms",
"me_route_backpressure_high_watermark_pct",
"me_health_interval_ms_unhealthy",
"me_health_interval_ms_healthy",
"me_admission_poll_ms",
"me_warn_rate_limit_ms",
"me_route_no_writer_mode",
"me_route_no_writer_wait_ms",
"me_route_hybrid_max_wait_ms",
"me_route_blocking_send_timeout_ms",
"me_route_inline_recovery_attempts",
"me_route_inline_recovery_wait_ms",
"links",
"fast_mode_min_tls_record",
"update_every",
"me_reinit_every_secs",
"me_hardswap_warmup_delay_min_ms",
"me_hardswap_warmup_delay_max_ms",
"me_hardswap_warmup_extra_passes",
"me_hardswap_warmup_pass_backoff_base_ms",
"me_config_stable_snapshots",
"me_config_apply_cooldown_secs",
"me_snapshot_require_http_2xx",
"me_snapshot_reject_empty_map",
"me_snapshot_min_proxy_for_lines",
"proxy_secret_stable_snapshots",
"proxy_secret_rotate_runtime",
"me_secret_atomic_snapshot",
"proxy_secret_len_max",
"me_pool_drain_ttl_secs",
"me_instadrain",
"me_pool_drain_threshold",
"me_pool_drain_soft_evict_enabled",
"me_pool_drain_soft_evict_grace_secs",
"me_pool_drain_soft_evict_per_writer",
"me_pool_drain_soft_evict_budget_per_core",
"me_pool_drain_soft_evict_cooldown_ms",
"me_bind_stale_mode",
"me_bind_stale_ttl_secs",
"me_pool_min_fresh_ratio",
"me_reinit_drain_timeout_secs",
"proxy_secret_auto_reload_secs",
"proxy_config_auto_reload_secs",
"me_reinit_singleflight",
"me_reinit_trigger_channel",
"me_reinit_coalesce_window_ms",
"me_deterministic_writer_sort",
"me_writer_pick_mode",
"me_writer_pick_sample_size",
"ntp_check",
"ntp_servers",
"auto_degradation_enabled",
"degradation_min_unavailable_dc_groups",
"rst_on_close",
];
const NETWORK_CONFIG_KEYS: &[&str] = &[
"ipv4",
"ipv6",
"prefer",
"multipath",
"stun_use",
"stun_servers",
"stun_tcp_fallback",
"http_ip_detect_urls",
"cache_public_ip_path",
"dns_overrides",
];
const SERVER_CONFIG_KEYS: &[&str] = &[
"port",
"listen_addr_ipv4",
"listen_addr_ipv6",
"listen_unix_sock",
"listen_unix_sock_perm",
"listen_tcp",
"proxy_protocol",
"proxy_protocol_header_timeout_ms",
"proxy_protocol_trusted_cidrs",
"metrics_port",
"metrics_listen",
"metrics_whitelist",
"api",
"admin_api",
"listeners",
"listen_backlog",
"max_connections",
"accept_permit_timeout_ms",
"conntrack_control",
];
const API_CONFIG_KEYS: &[&str] = &[
"enabled",
"listen",
"whitelist",
"gray_action",
"auth_header",
"request_body_limit_bytes",
"minimal_runtime_enabled",
"minimal_runtime_cache_ttl_ms",
"runtime_edge_enabled",
"runtime_edge_cache_ttl_ms",
"runtime_edge_top_n",
"runtime_edge_events_capacity",
"read_only",
];
const CONNTRACK_CONTROL_CONFIG_KEYS: &[&str] = &[
"inline_conntrack_control",
"mode",
"backend",
"profile",
"hybrid_listener_ips",
"pressure_high_watermark_pct",
"pressure_low_watermark_pct",
"delete_budget_per_sec",
];
const LISTENER_CONFIG_KEYS: &[&str] = &[
"ip",
"port",
"announce",
"announce_ip",
"proxy_protocol",
"reuse_allow",
];
const TIMEOUTS_CONFIG_KEYS: &[&str] = &[
"client_first_byte_idle_secs",
"client_handshake",
"relay_idle_policy_v2_enabled",
"relay_client_idle_soft_secs",
"relay_client_idle_hard_secs",
"relay_idle_grace_after_downstream_activity_secs",
"client_keepalive",
"client_ack",
"me_one_retry",
"me_one_timeout_ms",
];
const CENSORSHIP_CONFIG_KEYS: &[&str] = &[
"tls_domain",
"tls_domains",
"unknown_sni_action",
"tls_fetch_scope",
"tls_fetch",
"mask",
"mask_host",
"mask_port",
"mask_unix_sock",
"fake_cert_len",
"tls_emulation",
"tls_front_dir",
"server_hello_delay_min_ms",
"server_hello_delay_max_ms",
"tls_new_session_tickets",
"serverhello_compact",
"tls_full_cert_ttl_secs",
"alpn_enforce",
"mask_proxy_protocol",
"mask_shape_hardening",
"mask_shape_hardening_aggressive_mode",
"mask_shape_bucket_floor_bytes",
"mask_shape_bucket_cap_bytes",
"mask_shape_above_cap_blur",
"mask_shape_above_cap_blur_max_bytes",
"mask_relay_max_bytes",
"mask_relay_timeout_ms",
"mask_relay_idle_timeout_ms",
"mask_classifier_prefetch_timeout_ms",
"mask_timing_normalization_enabled",
"mask_timing_normalization_floor_ms",
"mask_timing_normalization_ceiling_ms",
];
const TLS_FETCH_CONFIG_KEYS: &[&str] = &[
"profiles",
"strict_route",
"attempt_timeout_ms",
"total_budget_ms",
"grease_enabled",
"deterministic",
"profile_cache_ttl_secs",
];
const ACCESS_CONFIG_KEYS: &[&str] = &[
"users",
"user_ad_tags",
"user_max_tcp_conns",
"user_max_tcp_conns_global_each",
"user_expirations",
"user_data_quota",
"user_rate_limits",
"cidr_rate_limits",
"user_max_unique_ips",
"user_max_unique_ips_global_each",
"user_max_unique_ips_mode",
"user_max_unique_ips_window_secs",
"replay_check_len",
"replay_window_secs",
"ignore_time_skew",
];
const RATE_LIMIT_BPS_CONFIG_KEYS: &[&str] = &["up_bps", "down_bps"];
const UPSTREAM_CONFIG_KEYS: &[&str] = &[
"type",
"interface",
"bind_addresses",
"bindtodevice",
"force_bind",
"address",
"user_id",
"username",
"password",
"url",
"weight",
"enabled",
"scopes",
"ipv4",
"ipv6",
];
const PROXY_MODES_CONFIG_KEYS: &[&str] = &["classic", "secure", "tls"];
const TELEMETRY_CONFIG_KEYS: &[&str] = &["core_enabled", "user_enabled", "me_level"];
const LINKS_CONFIG_KEYS: &[&str] = &["show", "public_host", "public_port"];
#[derive(Debug)]
struct UnknownConfigKey {
path: String,
suggestion: Option<String>,
}
fn table_at<'a>(value: &'a toml::Value, path: &[&str]) -> Option<&'a toml::Table> {
let mut current = value;
for segment in path {
current = current.get(*segment)?;
}
current.as_table()
}
fn is_strict_config(parsed_toml: &toml::Value) -> bool {
table_at(parsed_toml, &["general"])
.and_then(|table| table.get("config_strict"))
.and_then(toml::Value::as_bool)
.unwrap_or(false)
}
fn known_config_keys_for_suggestion() -> Vec<&'static str> {
let mut keys = Vec::new();
for group in [
TOP_LEVEL_CONFIG_KEYS,
GENERAL_CONFIG_KEYS,
NETWORK_CONFIG_KEYS,
SERVER_CONFIG_KEYS,
API_CONFIG_KEYS,
CONNTRACK_CONTROL_CONFIG_KEYS,
LISTENER_CONFIG_KEYS,
TIMEOUTS_CONFIG_KEYS,
CENSORSHIP_CONFIG_KEYS,
TLS_FETCH_CONFIG_KEYS,
ACCESS_CONFIG_KEYS,
RATE_LIMIT_BPS_CONFIG_KEYS,
UPSTREAM_CONFIG_KEYS,
PROXY_MODES_CONFIG_KEYS,
TELEMETRY_CONFIG_KEYS,
LINKS_CONFIG_KEYS,
] {
keys.extend_from_slice(group);
}
keys
}
fn levenshtein_distance(a: &str, b: &str) -> usize {
let b_chars: Vec<char> = b.chars().collect();
let mut prev: Vec<usize> = (0..=b_chars.len()).collect();
let mut curr = vec![0usize; b_chars.len() + 1];
for (i, ca) in a.chars().enumerate() {
curr[0] = i + 1;
for (j, cb) in b_chars.iter().enumerate() {
let replace = if ca == *cb { prev[j] } else { prev[j] + 1 };
curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(replace);
}
std::mem::swap(&mut prev, &mut curr);
}
prev[b_chars.len()]
}
fn unknown_key_suggestion(key: &str, known_keys: &[&'static str]) -> Option<String> {
let normalized = key.to_ascii_lowercase();
let mut best: Option<(&str, usize)> = None;
for known in known_keys {
let distance = levenshtein_distance(&normalized, known);
let is_better = match best {
Some((_, best_distance)) => distance < best_distance,
None => true,
};
if distance <= 4 && is_better {
best = Some((known, distance));
}
}
best.map(|(known, _)| known.to_string())
}
fn push_unknown_keys(
unknown: &mut Vec<UnknownConfigKey>,
known_for_suggestion: &[&'static str],
path: &str,
table: &toml::Table,
allowed: &[&str],
) {
for key in table.keys() {
if !allowed.contains(&key.as_str()) {
let full_path = if path.is_empty() {
key.clone()
} else {
format!("{path}.{key}")
};
unknown.push(UnknownConfigKey {
path: full_path,
suggestion: unknown_key_suggestion(key, known_for_suggestion),
});
}
}
}
fn check_known_table(
parsed_toml: &toml::Value,
unknown: &mut Vec<UnknownConfigKey>,
known_for_suggestion: &[&'static str],
path: &[&str],
allowed: &[&str],
) {
if let Some(table) = table_at(parsed_toml, path) {
push_unknown_keys(
unknown,
known_for_suggestion,
&path.join("."),
table,
allowed,
);
}
}
fn check_nested_table_value(
unknown: &mut Vec<UnknownConfigKey>,
known_for_suggestion: &[&'static str],
path: String,
value: &toml::Value,
allowed: &[&str],
) {
if let Some(table) = value.as_table() {
push_unknown_keys(unknown, known_for_suggestion, &path, table, allowed);
}
}
fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<UnknownConfigKey> {
let known_for_suggestion = known_config_keys_for_suggestion();
let mut unknown = Vec::new();
if let Some(root) = parsed_toml.as_table() {
push_unknown_keys(
&mut unknown,
&known_for_suggestion,
"",
root,
TOP_LEVEL_CONFIG_KEYS,
);
}
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general"],
GENERAL_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general", "modes"],
PROXY_MODES_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general", "telemetry"],
TELEMETRY_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["general", "links"],
LINKS_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["network"],
NETWORK_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server"],
SERVER_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server", "api"],
API_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server", "admin_api"],
API_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["server", "conntrack_control"],
CONNTRACK_CONTROL_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["timeouts"],
TIMEOUTS_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["censorship"],
CENSORSHIP_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["censorship", "tls_fetch"],
TLS_FETCH_CONFIG_KEYS,
);
check_known_table(
parsed_toml,
&mut unknown,
&known_for_suggestion,
&["access"],
ACCESS_CONFIG_KEYS,
);
if let Some(listeners) = table_at(parsed_toml, &["server"])
.and_then(|table| table.get("listeners"))
.and_then(toml::Value::as_array)
{
for (idx, listener) in listeners.iter().enumerate() {
check_nested_table_value(
&mut unknown,
&known_for_suggestion,
format!("server.listeners[{idx}]"),
listener,
LISTENER_CONFIG_KEYS,
);
}
}
if let Some(upstreams) = parsed_toml.get("upstreams").and_then(toml::Value::as_array) {
for (idx, upstream) in upstreams.iter().enumerate() {
check_nested_table_value(
&mut unknown,
&known_for_suggestion,
format!("upstreams[{idx}]"),
upstream,
UPSTREAM_CONFIG_KEYS,
);
}
}
for access_map in ["user_rate_limits", "cidr_rate_limits"] {
if let Some(table) = table_at(parsed_toml, &["access"])
.and_then(|access| access.get(access_map))
.and_then(toml::Value::as_table)
{
for (entry_name, value) in table {
check_nested_table_value(
&mut unknown,
&known_for_suggestion,
format!("access.{access_map}.{entry_name}"),
value,
RATE_LIMIT_BPS_CONFIG_KEYS,
);
}
}
}
unknown
}
fn handle_unknown_config_keys(parsed_toml: &toml::Value) -> Result<()> {
let unknown = collect_unknown_config_keys(parsed_toml);
if unknown.is_empty() {
return Ok(());
}
for item in &unknown {
if let Some(suggestion) = item.suggestion.as_deref() {
warn!(
key = %item.path,
suggestion = %suggestion,
"Unknown config key ignored; did you mean the suggested key?"
);
} else {
warn!(key = %item.path, "Unknown config key ignored");
}
}
if is_strict_config(parsed_toml) {
let mut paths = Vec::with_capacity(unknown.len());
for item in unknown {
if let Some(suggestion) = item.suggestion {
paths.push(format!("{} (did you mean `{}`?)", item.path, suggestion));
} else {
paths.push(item.path);
}
}
return Err(ProxyError::Config(format!(
"unknown config keys are not allowed when general.config_strict=true: {}",
paths.join(", ")
)));
}
Ok(())
}
#[derive(Debug, Clone)]
pub(crate) struct LoadedConfig {
@@ -337,6 +1003,7 @@ impl ProxyConfig {
let parsed_toml: toml::Value =
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
handle_unknown_config_keys(&parsed_toml)?;
let general_table = parsed_toml
.get("general")
.and_then(|value| value.as_table());
@@ -1087,9 +1754,9 @@ impl ProxyConfig {
));
}
if config.general.me_route_blocking_send_timeout_ms > 5000 {
if !(1..=5000).contains(&config.general.me_route_blocking_send_timeout_ms) {
return Err(ProxyError::Config(
"general.me_route_blocking_send_timeout_ms must be within [0, 5000]".to_string(),
"general.me_route_blocking_send_timeout_ms must be within [1, 5000]".to_string(),
));
}
@@ -1111,9 +1778,11 @@ impl ProxyConfig {
));
}
if config.server.api.request_body_limit_bytes == 0 {
if !(1..=MAX_API_REQUEST_BODY_LIMIT_BYTES)
.contains(&config.server.api.request_body_limit_bytes)
{
return Err(ProxyError::Config(
"server.api.request_body_limit_bytes must be > 0".to_string(),
"server.api.request_body_limit_bytes must be within [1, 1048576]".to_string(),
));
}
@@ -1441,13 +2110,22 @@ impl ProxyConfig {
return Err(ProxyError::Config("No modes enabled".to_string()));
}
if self.censorship.tls_domain.contains(' ') || self.censorship.tls_domain.contains('/') {
if !is_valid_tls_domain_name(&self.censorship.tls_domain) {
return Err(ProxyError::Config(format!(
"Invalid tls_domain: '{}'. Must be a valid domain name",
self.censorship.tls_domain
)));
}
for domain in &self.censorship.tls_domains {
if !is_valid_tls_domain_name(domain) {
return Err(ProxyError::Config(format!(
"Invalid tls_domains entry: '{}'. Must be a valid domain name",
domain
)));
}
}
for (user, tag) in &self.access.user_ad_tags {
let zeros = "00000000000000000000000000000000";
if !is_valid_ad_tag(tag) {
@@ -2602,6 +3280,26 @@ mod tests {
let _ = std::fs::remove_file(path);
}
#[test]
fn me_route_blocking_send_timeout_ms_zero_is_rejected() {
let toml = r#"
[general]
me_route_blocking_send_timeout_ms = 0
[censorship]
tls_domain = "example.com"
[access.users]
user = "00000000000000000000000000000000"
"#;
let dir = std::env::temp_dir();
let path = dir.join("telemt_me_route_blocking_send_timeout_zero_test.toml");
std::fs::write(&path, toml).unwrap();
let err = ProxyConfig::load(&path).unwrap_err().to_string();
assert!(err.contains("general.me_route_blocking_send_timeout_ms must be within [1, 5000]"));
let _ = std::fs::remove_file(path);
}
#[test]
fn me_route_no_writer_mode_is_parsed() {
let toml = r#"

View File

@@ -26,6 +26,10 @@ pub enum LogLevel {
Silent,
}
fn default_quota_state_path() -> PathBuf {
PathBuf::from("telemt.limit.json")
}
impl LogLevel {
/// Convert to tracing EnvFilter directive string.
pub fn to_filter_str(&self) -> &'static str {
@@ -375,6 +379,15 @@ pub struct GeneralConfig {
#[serde(default)]
pub data_path: Option<PathBuf>,
/// JSON state file for runtime per-user quota consumption.
#[serde(default = "default_quota_state_path")]
pub quota_state_path: PathBuf,
/// Reject unknown TOML config keys during load.
/// Startup fails fast; hot-reload rejects the new snapshot and keeps the current config.
#[serde(default)]
pub config_strict: bool,
#[serde(default)]
pub modes: ProxyModes,
@@ -530,10 +543,17 @@ pub struct GeneralConfig {
pub me_d2c_frame_buf_shrink_threshold_bytes: usize,
/// Copy buffer size for client->DC direction in direct relay.
///
/// This is also the upper bound for one amortized upload rate-limit burst:
/// upload debt is settled before the next relay read instead of blocking
/// inside the completed read path.
#[serde(default = "default_direct_relay_copy_buf_c2s_bytes")]
pub direct_relay_copy_buf_c2s_bytes: usize,
/// Copy buffer size for DC->client direction in direct relay.
///
/// This bounds one direct download rate-limit grant because writes are
/// clipped to the currently available shaper budget.
#[serde(default = "default_direct_relay_copy_buf_s2c_bytes")]
pub direct_relay_copy_buf_s2c_bytes: usize,
@@ -778,7 +798,7 @@ pub struct GeneralConfig {
pub me_route_hybrid_max_wait_ms: u64,
/// Maximum wait in milliseconds for blocking ME writer channel send fallback.
/// `0` keeps legacy unbounded wait behavior.
/// Must be within [1, 5000].
#[serde(default = "default_me_route_blocking_send_timeout_ms")]
pub me_route_blocking_send_timeout_ms: u64,
@@ -974,6 +994,8 @@ impl Default for GeneralConfig {
fn default() -> Self {
Self {
data_path: None,
quota_state_path: default_quota_state_path(),
config_strict: false,
modes: ProxyModes::default(),
prefer_ipv6: false,
fast_mode: default_true(),
@@ -1876,17 +1898,26 @@ pub struct AccessConfig {
///
/// Each entry supports independent upload (`up_bps`) and download
/// (`down_bps`) ceilings. A value of `0` in one direction means
/// "unlimited" for that direction.
/// "unlimited" for that direction. Limits are amortized: a relay quantum
/// may pass as a bounded burst, and the limiter applies the resulting wait
/// before later traffic in the same direction proceeds.
#[serde(default)]
pub user_rate_limits: HashMap<String, RateLimitBps>,
/// Per-CIDR aggregate transport rate limits in bits-per-second.
///
/// Matching uses longest-prefix-wins semantics. A value of `0` in one
/// direction means "unlimited" for that direction.
/// direction means "unlimited" for that direction. Limits are amortized
/// with the same bounded-burst contract as per-user rate limits.
#[serde(default)]
pub cidr_rate_limits: HashMap<IpNetwork, RateLimitBps>,
/// Per-username client source IP/CIDR deny list. Checked after successful
/// authentication; matching IPs get the same rejection path as invalid auth
/// (handshake fails closed for that connection).
#[serde(default)]
pub user_source_deny: HashMap<String, Vec<IpNetwork>>,
#[serde(default)]
pub user_max_unique_ips: HashMap<String, usize>,
@@ -1922,6 +1953,7 @@ impl Default for AccessConfig {
user_data_quota: HashMap::new(),
user_rate_limits: HashMap::new(),
cidr_rate_limits: HashMap::new(),
user_source_deny: HashMap::new(),
user_max_unique_ips: HashMap::new(),
user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(),
user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(),
@@ -1933,6 +1965,15 @@ impl Default for AccessConfig {
}
}
impl AccessConfig {
/// Returns true if `ip` is contained in any CIDR listed for `username` under `user_source_deny`.
pub fn is_user_source_ip_denied(&self, username: &str, ip: IpAddr) -> bool {
self.user_source_deny
.get(username)
.is_some_and(|nets| nets.iter().any(|n| n.contains(ip)))
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct RateLimitBps {
#[serde(default)]

View File

@@ -222,6 +222,21 @@ pub enum ProxyError {
#[error("Proxy error: {0}")]
Proxy(String),
#[error("ME connection lost")]
MiddleConnectionLost,
#[error("Session terminated")]
RouteSwitched,
#[error("Traffic budget wait cancelled")]
TrafficBudgetWaitCancelled,
#[error("Traffic budget wait deadline exceeded")]
TrafficBudgetWaitDeadlineExceeded,
#[error("ME client writer cancelled")]
MiddleClientWriterCancelled,
// ============= Config Errors =============
#[error("Config error: {0}")]
Config(String),

View File

@@ -9,30 +9,53 @@ use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use tokio::sync::{Mutex as AsyncMutex, RwLock};
use tokio::sync::{Mutex as AsyncMutex, RwLock, RwLockWriteGuard};
use crate::config::UserMaxUniqueIpsMode;
const CLEANUP_DRAIN_BATCH_LIMIT: usize = 1024;
const MAX_ACTIVE_IP_ENTRIES: u64 = 131_072;
const MAX_RECENT_IP_ENTRIES: u64 = 262_144;
/// Tracks active and recent client IPs for per-user admission control.
#[derive(Debug, Clone)]
pub struct UserIpTracker {
active_ips: Arc<RwLock<HashMap<String, HashMap<IpAddr, usize>>>>,
recent_ips: Arc<RwLock<HashMap<String, HashMap<IpAddr, Instant>>>>,
active_entry_count: Arc<AtomicU64>,
recent_entry_count: Arc<AtomicU64>,
active_cap_rejects: Arc<AtomicU64>,
recent_cap_rejects: Arc<AtomicU64>,
cleanup_deferred_releases: Arc<AtomicU64>,
max_ips: Arc<RwLock<HashMap<String, usize>>>,
default_max_ips: Arc<RwLock<usize>>,
limit_mode: Arc<RwLock<UserMaxUniqueIpsMode>>,
limit_window: Arc<RwLock<Duration>>,
last_compact_epoch_secs: Arc<AtomicU64>,
cleanup_queue_len: Arc<AtomicU64>,
cleanup_queue: Arc<Mutex<HashMap<(String, IpAddr), usize>>>,
cleanup_drain_lock: Arc<AsyncMutex<()>>,
}
/// Point-in-time memory counters for user/IP limiter state.
#[derive(Debug, Clone, Copy)]
pub struct UserIpTrackerMemoryStats {
/// Number of users with active IP state.
pub active_users: usize,
/// Number of users with recent IP state.
pub recent_users: usize,
/// Number of active `(user, ip)` entries.
pub active_entries: usize,
/// Number of recent-window `(user, ip)` entries.
pub recent_entries: usize,
/// Number of deferred disconnect cleanups waiting to be drained.
pub cleanup_queue_len: usize,
/// Number of new connections rejected by the global active-entry cap.
pub active_cap_rejects: u64,
/// Number of new connections rejected by the global recent-entry cap.
pub recent_cap_rejects: u64,
/// Number of release cleanups deferred through the cleanup queue.
pub cleanup_deferred_releases: u64,
}
impl UserIpTracker {
@@ -40,26 +63,81 @@ impl UserIpTracker {
Self {
active_ips: Arc::new(RwLock::new(HashMap::new())),
recent_ips: Arc::new(RwLock::new(HashMap::new())),
active_entry_count: Arc::new(AtomicU64::new(0)),
recent_entry_count: Arc::new(AtomicU64::new(0)),
active_cap_rejects: Arc::new(AtomicU64::new(0)),
recent_cap_rejects: Arc::new(AtomicU64::new(0)),
cleanup_deferred_releases: Arc::new(AtomicU64::new(0)),
max_ips: Arc::new(RwLock::new(HashMap::new())),
default_max_ips: Arc::new(RwLock::new(0)),
limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)),
limit_window: Arc::new(RwLock::new(Duration::from_secs(30))),
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
cleanup_queue: Arc::new(Mutex::new(HashMap::new())),
cleanup_drain_lock: Arc::new(AsyncMutex::new(())),
}
}
fn decrement_counter(counter: &AtomicU64, amount: usize) {
if amount == 0 {
return;
}
let amount = amount as u64;
let _ = counter.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| {
Some(current.saturating_sub(amount))
});
}
fn apply_active_cleanup(
active_ips: &mut HashMap<String, HashMap<IpAddr, usize>>,
user: &str,
ip: IpAddr,
pending_count: usize,
) -> usize {
if pending_count == 0 {
return 0;
}
let mut remove_user = false;
let mut removed_active_entries = 0usize;
if let Some(user_ips) = active_ips.get_mut(user) {
if let Some(count) = user_ips.get_mut(&ip) {
if *count > pending_count {
*count -= pending_count;
} else if user_ips.remove(&ip).is_some() {
removed_active_entries = 1;
}
}
remove_user = user_ips.is_empty();
}
if remove_user {
active_ips.remove(user);
}
removed_active_entries
}
/// Queues a deferred active IP cleanup for a later async drain.
pub fn enqueue_cleanup(&self, user: String, ip: IpAddr) {
match self.cleanup_queue.lock() {
Ok(mut queue) => {
let count = queue.entry((user, ip)).or_insert(0);
if *count == 0 {
self.cleanup_queue_len.fetch_add(1, Ordering::Relaxed);
}
*count = count.saturating_add(1);
self.cleanup_deferred_releases
.fetch_add(1, Ordering::Relaxed);
}
Err(poisoned) => {
let mut queue = poisoned.into_inner();
let count = queue.entry((user.clone(), ip)).or_insert(0);
if *count == 0 {
self.cleanup_queue_len.fetch_add(1, Ordering::Relaxed);
}
*count = count.saturating_add(1);
self.cleanup_deferred_releases
.fetch_add(1, Ordering::Relaxed);
self.cleanup_queue.clear_poison();
tracing::warn!(
"UserIpTracker cleanup_queue lock poisoned; recovered and enqueued IP cleanup for {} ({})",
@@ -86,16 +164,31 @@ impl UserIpTracker {
}
pub(crate) async fn drain_cleanup_queue(&self) {
// Serialize queue draining and active-IP mutation so check-and-add cannot
// observe stale active entries that are already queued for removal.
let _drain_guard = self.cleanup_drain_lock.lock().await;
if self.cleanup_queue_len.load(Ordering::Relaxed) == 0 {
return;
}
let Ok(_drain_guard) = self.cleanup_drain_lock.try_lock() else {
return;
};
let to_remove = {
match self.cleanup_queue.lock() {
Ok(mut queue) => {
if queue.is_empty() {
return;
}
std::mem::take(&mut *queue)
let mut drained =
HashMap::with_capacity(queue.len().min(CLEANUP_DRAIN_BATCH_LIMIT));
for _ in 0..CLEANUP_DRAIN_BATCH_LIMIT {
let Some(key) = queue.keys().next().cloned() else {
break;
};
if let Some(count) = queue.remove(&key) {
self.cleanup_queue_len.fetch_sub(1, Ordering::Relaxed);
drained.insert(key, count);
}
}
drained
}
Err(poisoned) => {
let mut queue = poisoned.into_inner();
@@ -103,31 +196,34 @@ impl UserIpTracker {
self.cleanup_queue.clear_poison();
return;
}
let drained = std::mem::take(&mut *queue);
let mut drained =
HashMap::with_capacity(queue.len().min(CLEANUP_DRAIN_BATCH_LIMIT));
for _ in 0..CLEANUP_DRAIN_BATCH_LIMIT {
let Some(key) = queue.keys().next().cloned() else {
break;
};
if let Some(count) = queue.remove(&key) {
self.cleanup_queue_len.fetch_sub(1, Ordering::Relaxed);
drained.insert(key, count);
}
}
self.cleanup_queue.clear_poison();
drained
}
}
};
if to_remove.is_empty() {
return;
}
let mut active_ips = self.active_ips.write().await;
let mut removed_active_entries = 0usize;
for ((user, ip), pending_count) in to_remove {
if pending_count == 0 {
continue;
}
if let Some(user_ips) = active_ips.get_mut(&user) {
if let Some(count) = user_ips.get_mut(&ip) {
if *count > pending_count {
*count -= pending_count;
} else {
user_ips.remove(&ip);
}
}
if user_ips.is_empty() {
active_ips.remove(&user);
}
}
removed_active_entries = removed_active_entries.saturating_add(
Self::apply_active_cleanup(&mut active_ips, &user, ip, pending_count),
);
}
Self::decrement_counter(&self.active_entry_count, removed_active_entries);
}
fn now_epoch_secs() -> u64 {
@@ -137,6 +233,24 @@ impl UserIpTracker {
.as_secs()
}
async fn active_and_recent_write(
&self,
) -> (
RwLockWriteGuard<'_, HashMap<String, HashMap<IpAddr, usize>>>,
RwLockWriteGuard<'_, HashMap<String, HashMap<IpAddr, Instant>>>,
) {
loop {
let active_ips = self.active_ips.write().await;
match self.recent_ips.try_write() {
Ok(recent_ips) => return (active_ips, recent_ips),
Err(_) => {
drop(active_ips);
tokio::task::yield_now().await;
}
}
}
}
async fn maybe_compact_empty_users(&self) {
const COMPACT_INTERVAL_SECS: u64 = 60;
let now_epoch_secs = Self::now_epoch_secs();
@@ -157,14 +271,16 @@ impl UserIpTracker {
return;
}
let mut active_ips = self.active_ips.write().await;
let mut recent_ips = self.recent_ips.write().await;
let window = *self.limit_window.read().await;
let now = Instant::now();
let (mut active_ips, mut recent_ips) = self.active_and_recent_write().await;
let mut pruned_recent_entries = 0usize;
for user_recent in recent_ips.values_mut() {
Self::prune_recent(user_recent, now, window);
pruned_recent_entries =
pruned_recent_entries.saturating_add(Self::prune_recent(user_recent, now, window));
}
Self::decrement_counter(&self.recent_entry_count, pruned_recent_entries);
let mut users =
Vec::<String>::with_capacity(active_ips.len().saturating_add(recent_ips.len()));
@@ -191,12 +307,17 @@ impl UserIpTracker {
}
}
pub async fn run_periodic_maintenance(self: Arc<Self>) {
let mut interval = tokio::time::interval(Duration::from_secs(1));
loop {
interval.tick().await;
self.drain_cleanup_queue().await;
self.maybe_compact_empty_users().await;
}
}
pub async fn memory_stats(&self) -> UserIpTrackerMemoryStats {
let cleanup_queue_len = self
.cleanup_queue
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.len();
let cleanup_queue_len = self.cleanup_queue_len.load(Ordering::Relaxed) as usize;
let active_ips = self.active_ips.read().await;
let recent_ips = self.recent_ips.read().await;
let active_entries = active_ips.values().map(HashMap::len).sum();
@@ -208,6 +329,9 @@ impl UserIpTracker {
active_entries,
recent_entries,
cleanup_queue_len,
active_cap_rejects: self.active_cap_rejects.load(Ordering::Relaxed),
recent_cap_rejects: self.recent_cap_rejects.load(Ordering::Relaxed),
cleanup_deferred_releases: self.cleanup_deferred_releases.load(Ordering::Relaxed),
}
}
@@ -238,11 +362,17 @@ impl UserIpTracker {
max_ips.clone_from(limits);
}
fn prune_recent(user_recent: &mut HashMap<IpAddr, Instant>, now: Instant, window: Duration) {
fn prune_recent(
user_recent: &mut HashMap<IpAddr, Instant>,
now: Instant,
window: Duration,
) -> usize {
if user_recent.is_empty() {
return;
return 0;
}
let before = user_recent.len();
user_recent.retain(|_, seen_at| now.duration_since(*seen_at) <= window);
before.saturating_sub(user_recent.len())
}
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
@@ -261,26 +391,40 @@ impl UserIpTracker {
let window = *self.limit_window.read().await;
let now = Instant::now();
let mut active_ips = self.active_ips.write().await;
let (mut active_ips, mut recent_ips) = self.active_and_recent_write().await;
let user_active = active_ips
.entry(username.to_string())
.or_insert_with(HashMap::new);
let mut recent_ips = self.recent_ips.write().await;
let user_recent = recent_ips
.entry(username.to_string())
.or_insert_with(HashMap::new);
Self::prune_recent(user_recent, now, window);
let pruned_recent_entries = Self::prune_recent(user_recent, now, window);
Self::decrement_counter(&self.recent_entry_count, pruned_recent_entries);
let recent_contains_ip = user_recent.contains_key(&ip);
if let Some(count) = user_active.get_mut(&ip) {
if !recent_contains_ip
&& self.recent_entry_count.load(Ordering::Relaxed) >= MAX_RECENT_IP_ENTRIES
{
self.recent_cap_rejects.fetch_add(1, Ordering::Relaxed);
return Err(format!(
"IP tracker recent entry cap reached: entries={}/{}",
self.recent_entry_count.load(Ordering::Relaxed),
MAX_RECENT_IP_ENTRIES
));
}
*count = count.saturating_add(1);
user_recent.insert(ip, now);
if user_recent.insert(ip, now).is_none() {
self.recent_entry_count.fetch_add(1, Ordering::Relaxed);
}
return Ok(());
}
let is_new_ip = !recent_contains_ip;
if let Some(limit) = limit {
let active_limit_reached = user_active.len() >= limit;
let recent_limit_reached = user_recent.len() >= limit;
let recent_limit_reached = user_recent.len() >= limit && is_new_ip;
let deny = match mode {
UserMaxUniqueIpsMode::ActiveWindow => active_limit_reached,
UserMaxUniqueIpsMode::TimeWindow => recent_limit_reached,
@@ -300,30 +444,62 @@ impl UserIpTracker {
}
}
user_active.insert(ip, 1);
user_recent.insert(ip, now);
if self.active_entry_count.load(Ordering::Relaxed) >= MAX_ACTIVE_IP_ENTRIES {
self.active_cap_rejects.fetch_add(1, Ordering::Relaxed);
return Err(format!(
"IP tracker active entry cap reached: entries={}/{}",
self.active_entry_count.load(Ordering::Relaxed),
MAX_ACTIVE_IP_ENTRIES
));
}
if is_new_ip && self.recent_entry_count.load(Ordering::Relaxed) >= MAX_RECENT_IP_ENTRIES {
self.recent_cap_rejects.fetch_add(1, Ordering::Relaxed);
return Err(format!(
"IP tracker recent entry cap reached: entries={}/{}",
self.recent_entry_count.load(Ordering::Relaxed),
MAX_RECENT_IP_ENTRIES
));
}
if user_active.insert(ip, 1).is_none() {
self.active_entry_count.fetch_add(1, Ordering::Relaxed);
}
if user_recent.insert(ip, now).is_none() {
self.recent_entry_count.fetch_add(1, Ordering::Relaxed);
}
Ok(())
}
pub async fn remove_ip(&self, username: &str, ip: IpAddr) {
self.maybe_compact_empty_users().await;
let mut active_ips = self.active_ips.write().await;
let mut removed_active_entries = 0usize;
if let Some(user_ips) = active_ips.get_mut(username) {
if let Some(count) = user_ips.get_mut(&ip) {
if *count > 1 {
*count -= 1;
} else {
user_ips.remove(&ip);
if user_ips.remove(&ip).is_some() {
removed_active_entries = 1;
}
}
}
if user_ips.is_empty() {
active_ips.remove(username);
}
}
Self::decrement_counter(&self.active_entry_count, removed_active_entries);
}
pub async fn get_recent_counts_for_users(&self, users: &[String]) -> HashMap<String, usize> {
self.drain_cleanup_queue().await;
self.get_recent_counts_for_users_snapshot(users).await
}
pub(crate) async fn get_recent_counts_for_users_snapshot(
&self,
users: &[String],
) -> HashMap<String, usize> {
let window = *self.limit_window.read().await;
let now = Instant::now();
let recent_ips = self.recent_ips.read().await;
@@ -398,19 +574,29 @@ impl UserIpTracker {
pub async fn get_stats(&self) -> Vec<(String, usize, usize)> {
self.drain_cleanup_queue().await;
self.get_stats_snapshot().await
}
pub(crate) async fn get_stats_snapshot(&self) -> Vec<(String, usize, usize)> {
let active_ips = self.active_ips.read().await;
let active_counts = active_ips
.iter()
.map(|(username, user_ips)| (username.clone(), user_ips.len()))
.collect::<Vec<_>>();
drop(active_ips);
let max_ips = self.max_ips.read().await;
let default_max_ips = *self.default_max_ips.read().await;
let mut stats = Vec::new();
for (username, user_ips) in active_ips.iter() {
let mut stats = Vec::with_capacity(active_counts.len());
for (username, active_count) in active_counts {
let limit = max_ips
.get(username)
.get(&username)
.copied()
.filter(|limit| *limit > 0)
.or((default_max_ips > 0).then_some(default_max_ips))
.unwrap_or(0);
stats.push((username.clone(), user_ips.len(), limit));
stats.push((username, active_count, limit));
}
stats.sort_by(|a, b| a.0.cmp(&b.0));
@@ -419,20 +605,30 @@ impl UserIpTracker {
pub async fn clear_user_ips(&self, username: &str) {
let mut active_ips = self.active_ips.write().await;
active_ips.remove(username);
let removed_active_entries = active_ips
.remove(username)
.map(|ips| ips.len())
.unwrap_or(0);
drop(active_ips);
Self::decrement_counter(&self.active_entry_count, removed_active_entries);
let mut recent_ips = self.recent_ips.write().await;
recent_ips.remove(username);
let removed_recent_entries = recent_ips
.remove(username)
.map(|ips| ips.len())
.unwrap_or(0);
Self::decrement_counter(&self.recent_entry_count, removed_recent_entries);
}
pub async fn clear_all(&self) {
let mut active_ips = self.active_ips.write().await;
active_ips.clear();
drop(active_ips);
self.active_entry_count.store(0, Ordering::Relaxed);
let mut recent_ips = self.recent_ips.write().await;
recent_ips.clear();
self.recent_entry_count.store(0, Ordering::Relaxed);
}
pub async fn is_ip_active(&self, username: &str, ip: IpAddr) -> bool {
@@ -860,4 +1056,19 @@ mod tests {
.unwrap_or(false);
assert!(!stale_exists);
}
#[tokio::test]
async fn test_time_window_allows_same_ip_reconnect() {
let tracker = UserIpTracker::new();
tracker.set_user_limit("test_user", 1).await;
tracker
.set_limit_policy(UserMaxUniqueIpsMode::TimeWindow, 1)
.await;
let ip1 = test_ipv4(10, 4, 0, 1);
assert!(tracker.check_and_add("test_user", ip1).await.is_ok());
tracker.remove_ip("test_user", ip1).await;
assert!(tracker.check_and_add("test_user", ip1).await.is_ok());
}
}

View File

@@ -17,6 +17,7 @@ pub(crate) async fn configure_admission_gate(
route_runtime: Arc<RouteRuntimeController>,
admission_tx: &watch::Sender<bool>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
me_ready_rx: watch::Receiver<u64>,
) {
if config.general.use_middle_proxy {
if let Some(pool) = me_pool.as_ref() {
@@ -52,6 +53,7 @@ pub(crate) async fn configure_admission_gate(
let admission_tx_gate = admission_tx.clone();
let route_runtime_gate = route_runtime.clone();
let mut config_rx_gate = config_rx.clone();
let mut me_ready_rx_gate = me_ready_rx;
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
tokio::spawn(async move {
let mut gate_open = initial_gate_open;
@@ -74,6 +76,11 @@ pub(crate) async fn configure_admission_gate(
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
continue;
}
changed = me_ready_rx_gate.changed() => {
if changed.is_err() {
break;
}
}
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
}
let ready = pool_for_gate.admission_ready_conditional_cast().await;

View File

@@ -1,6 +1,6 @@
#![allow(clippy::items_after_test_module)]
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::sync::watch;
@@ -17,7 +17,7 @@ use crate::transport::middle_proxy::{
pub(crate) fn resolve_runtime_config_path(
config_path_cli: &str,
startup_cwd: &std::path::Path,
startup_cwd: &Path,
config_path_explicit: bool,
) -> PathBuf {
if config_path_explicit {
@@ -46,6 +46,39 @@ pub(crate) fn resolve_runtime_config_path(
startup_cwd.join("config.toml")
}
pub(crate) fn resolve_runtime_base_dir(
config_path: &Path,
startup_cwd: &Path,
config_path_explicit: bool,
data_path: Option<&Path>,
) -> PathBuf {
if let Some(path) = data_path {
return normalize_runtime_dir(path, startup_cwd);
}
if startup_cwd != Path::new("/") {
return normalize_runtime_dir(startup_cwd, startup_cwd);
}
if config_path_explicit
&& let Some(parent) = config_path.parent()
&& !parent.as_os_str().is_empty()
{
return normalize_runtime_dir(parent, startup_cwd);
}
PathBuf::from("/etc/telemt")
}
fn normalize_runtime_dir(path: &Path, startup_cwd: &Path) -> PathBuf {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
startup_cwd.join(path)
};
absolute.canonicalize().unwrap_or(absolute)
}
/// Parsed CLI arguments.
pub(crate) struct CliArgs {
pub config_path: String,
@@ -231,9 +264,11 @@ fn print_help() {
#[cfg(test)]
mod tests {
use std::path::{Path, PathBuf};
use super::{
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
resolve_runtime_config_path,
resolve_runtime_base_dir, resolve_runtime_config_path,
};
use crate::error::{ProxyError, StreamError};
@@ -304,6 +339,91 @@ mod tests {
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_prefers_cli_data_path() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let startup_cwd = std::env::temp_dir().join(format!("telemt_runtime_base_cwd_{nonce}"));
let data_path = std::env::temp_dir().join(format!("telemt_runtime_base_data_{nonce}"));
std::fs::create_dir_all(&startup_cwd).unwrap();
std::fs::create_dir_all(&data_path).unwrap();
let resolved = resolve_runtime_base_dir(
&startup_cwd.join("config.toml"),
&startup_cwd,
true,
Some(&data_path),
);
assert_eq!(resolved, data_path.canonicalize().unwrap());
let _ = std::fs::remove_dir(&data_path);
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_uses_working_directory_before_explicit_config_parent() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let startup_cwd = std::env::temp_dir().join(format!("telemt_runtime_base_start_{nonce}"));
let config_dir = std::env::temp_dir().join(format!("telemt_runtime_base_cfg_{nonce}"));
std::fs::create_dir_all(&startup_cwd).unwrap();
std::fs::create_dir_all(&config_dir).unwrap();
let resolved =
resolve_runtime_base_dir(&config_dir.join("telemt.toml"), &startup_cwd, true, None);
assert_eq!(resolved, startup_cwd.canonicalize().unwrap());
let _ = std::fs::remove_dir(&config_dir);
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_uses_explicit_config_parent_from_root() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let config_dir = std::env::temp_dir().join(format!("telemt_runtime_base_root_cfg_{nonce}"));
std::fs::create_dir_all(&config_dir).unwrap();
let resolved =
resolve_runtime_base_dir(&config_dir.join("telemt.toml"), Path::new("/"), true, None);
assert_eq!(resolved, config_dir.canonicalize().unwrap());
let _ = std::fs::remove_dir(&config_dir);
}
#[test]
fn resolve_runtime_base_dir_uses_systemd_working_directory_before_etc() {
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos();
let startup_cwd = std::env::temp_dir().join(format!("telemt_runtime_base_systemd_{nonce}"));
std::fs::create_dir_all(&startup_cwd).unwrap();
let resolved =
resolve_runtime_base_dir(&startup_cwd.join("config.toml"), &startup_cwd, false, None);
assert_eq!(resolved, startup_cwd.canonicalize().unwrap());
let _ = std::fs::remove_dir(&startup_cwd);
}
#[test]
fn resolve_runtime_base_dir_falls_back_to_etc_from_root() {
let resolved = resolve_runtime_base_dir(
Path::new("/etc/telemt/config.toml"),
Path::new("/"),
false,
None,
);
assert_eq!(resolved, PathBuf::from("/etc/telemt"));
}
#[test]
fn expected_handshake_eof_matches_connection_reset() {
let err = ProxyError::Io(std::io::Error::from(std::io::ErrorKind::ConnectionReset));

View File

@@ -13,7 +13,7 @@ use crate::config::{ProxyConfig, RstOnCloseMode};
use crate::crypto::SecureRandom;
use crate::ip_tracker::UserIpTracker;
use crate::proxy::ClientHandler;
use crate::proxy::route_mode::{ROUTE_SWITCH_ERROR_MSG, RouteRuntimeController};
use crate::proxy::route_mode::RouteRuntimeController;
use crate::proxy::shared_state::ProxySharedState;
use crate::startup::{COMPONENT_LISTENERS_BIND, StartupTracker};
use crate::stats::beobachten::BeobachtenStore;
@@ -492,14 +492,10 @@ pub(crate) fn spawn_tcp_accept_loops(
let handshake_close_reason =
expected_handshake_close_description(&e);
let me_closed = matches!(
&e,
crate::error::ProxyError::Proxy(msg) if msg == "ME connection lost"
);
let route_switched = matches!(
&e,
crate::error::ProxyError::Proxy(msg) if msg == ROUTE_SWITCH_ERROR_MSG
);
let me_closed =
matches!(&e, crate::error::ProxyError::MiddleConnectionLost);
let route_switched =
matches!(&e, crate::error::ProxyError::RouteSwitched);
match (peer_close_reason, me_closed) {
(Some(reason), _) => {

View File

@@ -3,7 +3,7 @@
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
use tokio::sync::{RwLock, watch};
use tracing::{error, info, warn};
use crate::config::ProxyConfig;
@@ -29,6 +29,7 @@ pub(crate) async fn initialize_me_pool(
rng: Arc<SecureRandom>,
stats: Arc<Stats>,
api_me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
me_ready_tx: watch::Sender<u64>,
) -> Option<Arc<MePool>> {
if !use_middle_proxy {
return None;
@@ -314,6 +315,7 @@ pub(crate) async fn initialize_me_pool(
let pool_bg = pool.clone();
let rng_bg = rng.clone();
let startup_tracker_bg = startup_tracker.clone();
let me_ready_tx_bg = me_ready_tx.clone();
let retry_limit = if me_init_retry_attempts == 0 {
String::from("unlimited")
} else {
@@ -347,6 +349,9 @@ pub(crate) async fn initialize_me_pool(
startup_tracker_bg
.set_me_status(StartupMeStatus::Ready, "ready")
.await;
me_ready_tx_bg.send_modify(|version| {
*version = version.saturating_add(1);
});
info!(
attempt = init_attempt,
"Middle-End pool initialized successfully"
@@ -474,6 +479,9 @@ pub(crate) async fn initialize_me_pool(
startup_tracker
.set_me_status(StartupMeStatus::Ready, "ready")
.await;
me_ready_tx.send_modify(|version| {
*version = version.saturating_add(1);
});
info!(
attempt = init_attempt,
"Middle-End pool initialized successfully"

View File

@@ -47,7 +47,7 @@ use crate::stats::{ReplayChecker, Stats};
use crate::stream::BufferPool;
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::MePool;
use helpers::{parse_cli, resolve_runtime_config_path};
use helpers::{parse_cli, resolve_runtime_base_dir, resolve_runtime_config_path};
#[cfg(unix)]
use crate::daemon::{DaemonOptions, PidFile, drop_privileges};
@@ -112,8 +112,51 @@ async fn run_telemt_core(
std::process::exit(1);
}
};
if let Some(ref data_path) = data_path
&& !data_path.is_absolute()
{
eprintln!(
"[telemt] data_path must be absolute: {}",
data_path.display()
);
std::process::exit(1);
}
let mut config_path =
resolve_runtime_config_path(&config_path_cli, &startup_cwd, config_path_explicit);
let runtime_base_dir = resolve_runtime_base_dir(
&config_path,
&startup_cwd,
config_path_explicit,
data_path.as_deref(),
);
if !runtime_base_dir.exists()
&& let Err(e) = std::fs::create_dir_all(&runtime_base_dir)
{
eprintln!(
"[telemt] Can't create runtime directory {}: {}",
runtime_base_dir.display(),
e
);
std::process::exit(1);
}
if !runtime_base_dir.is_dir() {
eprintln!(
"[telemt] Runtime path exists but is not a directory: {}",
runtime_base_dir.display()
);
std::process::exit(1);
}
if let Err(e) = std::env::set_current_dir(&runtime_base_dir) {
eprintln!(
"[telemt] Can't use runtime directory {}: {}",
runtime_base_dir.display(),
e
);
std::process::exit(1);
}
let mut config = match ProxyConfig::load(&config_path) {
Ok(c) => c,
@@ -156,16 +199,15 @@ async fn run_telemt_core(
);
}
} else {
let system_dir = std::path::Path::new("/etc/telemt");
let system_config_path = system_dir.join("telemt.toml");
let startup_config_path = startup_cwd.join("config.toml");
let runtime_config_path = runtime_base_dir.join("telemt.toml");
let fallback_config_path = runtime_base_dir.join("config.toml");
let mut persisted = false;
if let Some(serialized) = serialized.as_ref() {
match std::fs::create_dir_all(system_dir) {
Ok(()) => match std::fs::write(&system_config_path, serialized) {
match std::fs::create_dir_all(&runtime_base_dir) {
Ok(()) => match std::fs::write(&runtime_config_path, serialized) {
Ok(()) => {
config_path = system_config_path;
config_path = runtime_config_path;
eprintln!(
"[telemt] Created default config at {}",
config_path.display()
@@ -175,7 +217,7 @@ async fn run_telemt_core(
Err(write_error) => {
eprintln!(
"[telemt] Warning: failed to write default config at {}: {}",
system_config_path.display(),
runtime_config_path.display(),
write_error
);
}
@@ -183,16 +225,16 @@ async fn run_telemt_core(
Err(create_error) => {
eprintln!(
"[telemt] Warning: failed to create {}: {}",
system_dir.display(),
runtime_base_dir.display(),
create_error
);
}
}
if !persisted {
match std::fs::write(&startup_config_path, serialized) {
match std::fs::write(&fallback_config_path, serialized) {
Ok(()) => {
config_path = startup_config_path;
config_path = fallback_config_path;
eprintln!(
"[telemt] Created default config at {}",
config_path.display()
@@ -202,7 +244,7 @@ async fn run_telemt_core(
Err(write_error) => {
eprintln!(
"[telemt] Warning: failed to write default config at {}: {}",
startup_config_path.display(),
fallback_config_path.display(),
write_error
);
}
@@ -375,6 +417,8 @@ async fn run_telemt_core(
let stats = Arc::new(Stats::new());
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
let quota_state_path = config.general.quota_state_path.clone();
crate::quota_state::load_quota_state(&quota_state_path, stats.as_ref()).await;
let upstream_manager = Arc::new(UpstreamManager::new(
config.upstreams.clone(),
@@ -454,6 +498,7 @@ async fn run_telemt_core(
let config_rx_api = api_config_rx.clone();
let admission_rx_api = admission_rx.clone();
let config_path_api = config_path.clone();
let quota_state_path_api = quota_state_path.clone();
let startup_tracker_api = startup_tracker.clone();
let detected_ips_rx_api = detected_ips_rx.clone();
tokio::spawn(async move {
@@ -467,6 +512,7 @@ async fn run_telemt_core(
config_rx_api,
admission_rx_api,
config_path_api,
quota_state_path_api,
detected_ips_rx_api,
process_started_at_epoch_secs,
startup_tracker_api,
@@ -618,6 +664,8 @@ async fn run_telemt_core(
.await;
}
let (me_ready_tx, me_ready_rx) = watch::channel(0_u64);
let me_pool: Option<Arc<MePool>> = me_startup::initialize_me_pool(
use_middle_proxy,
&config,
@@ -628,6 +676,7 @@ async fn run_telemt_core(
rng.clone(),
stats.clone(),
api_me_pool.clone(),
me_ready_tx.clone(),
)
.await;
@@ -701,6 +750,7 @@ async fn run_telemt_core(
api_config_tx.clone(),
me_pool.clone(),
shared_state.clone(),
me_ready_tx.clone(),
)
.await;
let config_rx = runtime_watches.config_rx;
@@ -714,6 +764,7 @@ async fn run_telemt_core(
route_runtime.clone(),
&admission_tx,
config_rx.clone(),
me_ready_rx,
)
.await;
let _admission_tx_hold = admission_tx;
@@ -772,6 +823,7 @@ async fn run_telemt_core(
beobachten.clone(),
shared_state.clone(),
ip_tracker.clone(),
tls_cache.clone(),
config_rx.clone(),
)
.await;
@@ -799,7 +851,7 @@ async fn run_telemt_core(
max_connections.clone(),
);
shutdown::wait_for_shutdown(process_started_at, me_pool, stats).await;
shutdown::wait_for_shutdown(process_started_at, me_pool, stats, quota_state_path).await;
Ok(())
}

View File

@@ -21,6 +21,7 @@ use crate::startup::{
use crate::stats::beobachten::BeobachtenStore;
use crate::stats::telemetry::TelemetryPolicy;
use crate::stats::{ReplayChecker, Stats};
use crate::tls_front::TlsFrontCache;
use crate::transport::UpstreamManager;
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
@@ -52,6 +53,7 @@ pub(crate) async fn spawn_runtime_tasks(
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
me_pool_for_policy: Option<Arc<MePool>>,
shared_state: Arc<ProxySharedState>,
me_ready_tx: watch::Sender<u64>,
) -> RuntimeWatches {
let um_clone = upstream_manager.clone();
let dc_overrides_for_health = config.dc_overrides.clone();
@@ -71,6 +73,18 @@ pub(crate) async fn spawn_runtime_tasks(
rc_clone.run_periodic_cleanup().await;
});
let stats_maintenance = stats.clone();
tokio::spawn(async move {
stats_maintenance
.run_periodic_user_stats_maintenance()
.await;
});
let ip_tracker_maintenance = ip_tracker.clone();
tokio::spawn(async move {
ip_tracker_maintenance.run_periodic_maintenance().await;
});
let detected_ip_v4: Option<IpAddr> = probe.detected_ipv4.map(IpAddr::V4);
let detected_ip_v6: Option<IpAddr> = probe.detected_ipv6.map(IpAddr::V6);
debug!(
@@ -249,12 +263,14 @@ pub(crate) async fn spawn_runtime_tasks(
let pool_clone_sched = pool.clone();
let rng_clone_sched = rng.clone();
let config_rx_clone_sched = config_rx.clone();
let me_ready_tx_sched = me_ready_tx.clone();
tokio::spawn(async move {
crate::transport::middle_proxy::me_reinit_scheduler(
pool_clone_sched,
rng_clone_sched,
config_rx_clone_sched,
reinit_rx,
me_ready_tx_sched,
)
.await;
});
@@ -328,6 +344,7 @@ pub(crate) async fn spawn_metrics_if_configured(
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
) {
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
@@ -363,6 +380,7 @@ pub(crate) async fn spawn_metrics_if_configured(
let shared_state = shared_state.clone();
let config_rx_metrics = config_rx.clone();
let ip_tracker_metrics = ip_tracker.clone();
let tls_cache_metrics = tls_cache.clone();
let whitelist = config.server.metrics_whitelist.clone();
let listen_backlog = config.server.listen_backlog;
tokio::spawn(async move {
@@ -374,6 +392,7 @@ pub(crate) async fn spawn_metrics_if_configured(
beobachten,
shared_state,
ip_tracker_metrics,
tls_cache_metrics,
config_rx_metrics,
whitelist,
)

View File

@@ -8,6 +8,7 @@
//!
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -48,9 +49,17 @@ pub(crate) async fn wait_for_shutdown(
process_started_at: Instant,
me_pool: Option<Arc<MePool>>,
stats: Arc<Stats>,
quota_state_path: PathBuf,
) {
let signal = wait_for_shutdown_signal().await;
perform_shutdown(signal, process_started_at, me_pool, &stats).await;
perform_shutdown(
signal,
process_started_at,
me_pool,
&stats,
quota_state_path,
)
.await;
}
/// Waits for any shutdown signal (SIGINT, SIGTERM, SIGQUIT).
@@ -79,6 +88,7 @@ async fn perform_shutdown(
process_started_at: Instant,
me_pool: Option<Arc<MePool>>,
stats: &Stats,
quota_state_path: PathBuf,
) {
let shutdown_started_at = Instant::now();
info!(signal = %signal, "Received shutdown signal");
@@ -109,6 +119,22 @@ async fn perform_shutdown(
}
}
match crate::quota_state::save_quota_state(&quota_state_path, stats).await {
Ok(()) => {
info!(
path = %quota_state_path.display(),
"Persisted per-user quota state"
);
}
Err(error) => {
warn!(
error = %error,
path = %quota_state_path.display(),
"Failed to persist per-user quota state"
);
}
}
let shutdown_secs = shutdown_started_at.elapsed().as_secs();
info!(
"Shutdown completed successfully in {} {}.",

View File

@@ -10,6 +10,14 @@ use crate::tls_front::TlsFrontCache;
use crate::tls_front::fetcher::TlsFetchStrategy;
use crate::transport::UpstreamManager;
fn tls_fetch_host_for_domain(mask_host: &str, primary_tls_domain: &str, domain: &str) -> String {
if mask_host.eq_ignore_ascii_case(primary_tls_domain) {
domain.to_string()
} else {
mask_host.to_string()
}
}
pub(crate) async fn bootstrap_tls_front(
config: &ProxyConfig,
tls_domains: &[String],
@@ -56,6 +64,7 @@ pub(crate) async fn bootstrap_tls_front(
let cache_initial = cache.clone();
let domains_initial = tls_domains.to_vec();
let host_initial = mask_host.clone();
let primary_initial = config.censorship.tls_domain.clone();
let unix_sock_initial = mask_unix_sock.clone();
let scope_initial = tls_fetch_scope.clone();
let upstream_initial = upstream_manager.clone();
@@ -64,7 +73,8 @@ pub(crate) async fn bootstrap_tls_front(
let mut join = tokio::task::JoinSet::new();
for domain in domains_initial {
let cache_domain = cache_initial.clone();
let host_domain = host_initial.clone();
let host_domain =
tls_fetch_host_for_domain(&host_initial, &primary_initial, &domain);
let unix_sock_domain = unix_sock_initial.clone();
let scope_domain = scope_initial.clone();
let upstream_domain = upstream_initial.clone();
@@ -117,6 +127,7 @@ pub(crate) async fn bootstrap_tls_front(
let cache_refresh = cache.clone();
let domains_refresh = tls_domains.to_vec();
let host_refresh = mask_host.clone();
let primary_refresh = config.censorship.tls_domain.clone();
let unix_sock_refresh = mask_unix_sock.clone();
let scope_refresh = tls_fetch_scope.clone();
let upstream_refresh = upstream_manager.clone();
@@ -130,7 +141,8 @@ pub(crate) async fn bootstrap_tls_front(
let mut join = tokio::task::JoinSet::new();
for domain in domains_refresh.clone() {
let cache_domain = cache_refresh.clone();
let host_domain = host_refresh.clone();
let host_domain =
tls_fetch_host_for_domain(&host_refresh, &primary_refresh, &domain);
let unix_sock_domain = unix_sock_refresh.clone();
let scope_domain = scope_refresh.clone();
let upstream_domain = upstream_refresh.clone();
@@ -186,3 +198,24 @@ pub(crate) async fn bootstrap_tls_front(
tls_cache
}
#[cfg(test)]
mod tests {
use super::tls_fetch_host_for_domain;
#[test]
fn tls_fetch_host_uses_each_domain_when_mask_host_is_primary_default() {
assert_eq!(
tls_fetch_host_for_domain("a.com", "a.com", "b.com"),
"b.com"
);
}
#[test]
fn tls_fetch_host_preserves_explicit_non_primary_mask_host() {
assert_eq!(
tls_fetch_host_for_domain("origin.example", "a.com", "b.com"),
"origin.example"
);
}
}

View File

@@ -25,6 +25,7 @@ mod metrics;
mod network;
mod protocol;
mod proxy;
mod quota_state;
mod service;
mod startup;
mod stats;

View File

@@ -11,6 +11,8 @@ use hyper::service::service_fn;
use hyper::{Request, Response, StatusCode};
use ipnetwork::IpNetwork;
use tokio::net::TcpListener;
use tokio::sync::Semaphore;
use tokio::time::timeout;
use tracing::{debug, info, warn};
use crate::config::ProxyConfig;
@@ -18,8 +20,18 @@ use crate::ip_tracker::UserIpTracker;
use crate::proxy::shared_state::ProxySharedState;
use crate::stats::Stats;
use crate::stats::beobachten::BeobachtenStore;
use crate::tls_front::TlsFrontCache;
use crate::tls_front::cache;
use crate::tls_front::fetcher;
use crate::transport::{ListenOptions, create_listener};
// Keeps `/metrics` response size bounded when per-user telemetry is enabled.
const USER_LABELED_METRICS_MAX_USERS: usize = 4096;
// Keeps TLS-front per-domain health series bounded for large generated configs.
const TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS: usize = 256;
const METRICS_MAX_CONTROL_CONNECTIONS: usize = 512;
const METRICS_HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
pub async fn serve(
port: u16,
listen: Option<String>,
@@ -28,6 +40,7 @@ pub async fn serve(
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Vec<IpNetwork>,
) {
@@ -52,6 +65,7 @@ pub async fn serve(
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
@@ -64,11 +78,11 @@ pub async fn serve(
return;
}
// Fallback: bind on 0.0.0.0 and [::] using metrics_port.
// Fallback: keep metrics local unless an explicit metrics_listen is configured.
let mut listener_v4 = None;
let mut listener_v6 = None;
let addr_v4 = SocketAddr::from(([0, 0, 0, 0], port));
let addr_v4 = SocketAddr::from(([127, 0, 0, 1], port));
match bind_metrics_listener(addr_v4, false, listen_backlog) {
Ok(listener) => {
info!(
@@ -82,11 +96,11 @@ pub async fn serve(
}
}
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], port));
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 1], port));
match bind_metrics_listener(addr_v6, true, listen_backlog) {
Ok(listener) => {
info!(
"Metrics endpoint: http://[::]:{}/metrics and /beobachten",
"Metrics endpoint: http://[::1]:{}/metrics and /beobachten",
port
);
listener_v6 = Some(listener);
@@ -107,6 +121,7 @@ pub async fn serve(
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
@@ -117,6 +132,7 @@ pub async fn serve(
let beobachten_v6 = beobachten.clone();
let shared_state_v6 = shared_state.clone();
let ip_tracker_v6 = ip_tracker.clone();
let tls_cache_v6 = tls_cache.clone();
let config_rx_v6 = config_rx.clone();
let whitelist_v6 = whitelist.clone();
tokio::spawn(async move {
@@ -126,6 +142,7 @@ pub async fn serve(
beobachten_v6,
shared_state_v6,
ip_tracker_v6,
tls_cache_v6,
config_rx_v6,
whitelist_v6,
)
@@ -137,6 +154,7 @@ pub async fn serve(
beobachten,
shared_state,
ip_tracker,
tls_cache,
config_rx,
whitelist,
)
@@ -166,9 +184,12 @@ async fn serve_listener(
beobachten: Arc<BeobachtenStore>,
shared_state: Arc<ProxySharedState>,
ip_tracker: Arc<UserIpTracker>,
tls_cache: Option<Arc<TlsFrontCache>>,
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
whitelist: Arc<Vec<IpNetwork>>,
) {
let connection_permits = Arc::new(Semaphore::new(METRICS_MAX_CONTROL_CONNECTIONS));
loop {
let (stream, peer) = match listener.accept().await {
Ok(v) => v,
@@ -183,17 +204,32 @@ async fn serve_listener(
continue;
}
let connection_permit = match connection_permits.clone().try_acquire_owned() {
Ok(permit) => permit,
Err(_) => {
debug!(
peer = %peer,
max_connections = METRICS_MAX_CONTROL_CONNECTIONS,
"Dropping metrics connection: control-plane connection budget exhausted"
);
continue;
}
};
let stats = stats.clone();
let beobachten = beobachten.clone();
let shared_state = shared_state.clone();
let ip_tracker = ip_tracker.clone();
let tls_cache = tls_cache.clone();
let config_rx_conn = config_rx.clone();
tokio::spawn(async move {
let _connection_permit = connection_permit;
let svc = service_fn(move |req| {
let stats = stats.clone();
let beobachten = beobachten.clone();
let shared_state = shared_state.clone();
let ip_tracker = ip_tracker.clone();
let tls_cache = tls_cache.clone();
let config = config_rx_conn.borrow().clone();
async move {
handle(
@@ -202,16 +238,29 @@ async fn serve_listener(
&beobachten,
&shared_state,
&ip_tracker,
tls_cache.as_deref(),
&config,
)
.await
}
});
if let Err(e) = http1::Builder::new()
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
.await
match timeout(
METRICS_HTTP_CONNECTION_TIMEOUT,
http1::Builder::new().serve_connection(hyper_util::rt::TokioIo::new(stream), svc),
)
.await
{
debug!(error = %e, "Metrics connection error");
Ok(Ok(())) => {}
Ok(Err(e)) => {
debug!(error = %e, "Metrics connection error");
}
Err(_) => {
debug!(
peer = %peer,
timeout_ms = METRICS_HTTP_CONNECTION_TIMEOUT.as_millis() as u64,
"Metrics connection timed out"
);
}
}
});
}
@@ -223,10 +272,11 @@ async fn handle<B>(
beobachten: &BeobachtenStore,
shared_state: &ProxySharedState,
ip_tracker: &UserIpTracker,
tls_cache: Option<&TlsFrontCache>,
config: &ProxyConfig,
) -> Result<Response<Full<Bytes>>, Infallible> {
if req.uri().path() == "/metrics" {
let body = render_metrics(stats, shared_state, config, ip_tracker).await;
let body = render_metrics(stats, shared_state, config, ip_tracker, tls_cache).await;
let resp = Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/plain; version=0.0.4; charset=utf-8")
@@ -261,11 +311,138 @@ fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> Stri
beobachten.snapshot_text(ttl)
}
fn tls_front_domains(config: &ProxyConfig) -> Vec<String> {
let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
if !config.censorship.tls_domain.is_empty() {
domains.push(config.censorship.tls_domain.clone());
}
for domain in &config.censorship.tls_domains {
if !domain.is_empty() && !domains.contains(domain) {
domains.push(domain.clone());
}
}
domains
}
fn prometheus_label_value(value: &str) -> String {
value.replace('\\', "\\\\").replace('"', "\\\"")
}
async fn render_tls_front_profile_health(
out: &mut String,
config: &ProxyConfig,
tls_cache: Option<&TlsFrontCache>,
) {
use std::fmt::Write;
let domains = tls_front_domains(config);
let (health, suppressed) = match (config.censorship.tls_emulation, tls_cache) {
(true, Some(cache)) => {
cache
.profile_health_snapshot(&domains, TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS)
.await
}
_ => (Vec::new(), domains.len()),
};
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_domains TLS front configured profile domains by export status"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_domains gauge");
let _ = writeln!(
out,
"telemt_tls_front_profile_domains{{status=\"configured\"}} {}",
domains.len()
);
let _ = writeln!(
out,
"telemt_tls_front_profile_domains{{status=\"emitted\"}} {}",
health.len()
);
let _ = writeln!(
out,
"telemt_tls_front_profile_domains{{status=\"suppressed\"}} {}",
suppressed
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_info TLS front profile source and feature flags per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_info gauge");
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_age_seconds Age of cached TLS front profile data per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_age_seconds gauge");
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_app_data_records TLS front cached app-data record count per configured domain"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_front_profile_app_data_records gauge"
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_ticket_records TLS front cached ticket-like tail record count per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_ticket_records gauge");
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_change_cipher_spec_records TLS front cached ChangeCipherSpec record count per configured domain"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge"
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_profile_app_data_bytes TLS front cached total app-data bytes per configured domain"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_app_data_bytes gauge");
for item in health {
let domain = prometheus_label_value(&item.domain);
let _ = writeln!(
out,
"telemt_tls_front_profile_info{{domain=\"{}\",source=\"{}\",is_default=\"{}\",has_cert_info=\"{}\",has_cert_payload=\"{}\"}} 1",
domain, item.source, item.is_default, item.has_cert_info, item.has_cert_payload
);
let _ = writeln!(
out,
"telemt_tls_front_profile_age_seconds{{domain=\"{}\"}} {}",
domain, item.age_seconds
);
let _ = writeln!(
out,
"telemt_tls_front_profile_app_data_records{{domain=\"{}\"}} {}",
domain, item.app_data_records
);
let _ = writeln!(
out,
"telemt_tls_front_profile_ticket_records{{domain=\"{}\"}} {}",
domain, item.ticket_records
);
let _ = writeln!(
out,
"telemt_tls_front_profile_change_cipher_spec_records{{domain=\"{}\"}} {}",
domain, item.change_cipher_spec_count
);
let _ = writeln!(
out,
"telemt_tls_front_profile_app_data_bytes{{domain=\"{}\"}} {}",
domain, item.total_app_data_len
);
}
}
async fn render_metrics(
stats: &Stats,
shared_state: &ProxySharedState,
config: &ProxyConfig,
ip_tracker: &UserIpTracker,
tls_cache: Option<&TlsFrontCache>,
) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(4096);
@@ -311,6 +488,12 @@ async fn render_metrics(
"telemt_telemetry_user_enabled {}",
if user_enabled { 1 } else { 0 }
);
let _ = writeln!(
out,
"# HELP telemt_stats_user_entries Retained per-user stats entries"
);
let _ = writeln!(out, "# TYPE telemt_stats_user_entries gauge");
let _ = writeln!(out, "telemt_stats_user_entries {}", stats.user_stats_len());
let _ = writeln!(
out,
@@ -366,6 +549,54 @@ async fn render_metrics(
stats.get_buffer_pool_in_use_gauge()
);
let _ = writeln!(
out,
"# HELP telemt_tls_fetch_profile_cache_entries Current adaptive TLS fetch profile-cache entries"
);
let _ = writeln!(out, "# TYPE telemt_tls_fetch_profile_cache_entries gauge");
let _ = writeln!(
out,
"telemt_tls_fetch_profile_cache_entries {}",
fetcher::profile_cache_entries_for_metrics()
);
let _ = writeln!(
out,
"# HELP telemt_tls_fetch_profile_cache_cap_drops_total Profile-cache winner inserts skipped because the cache cap was reached"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"
);
let _ = writeln!(
out,
"telemt_tls_fetch_profile_cache_cap_drops_total {}",
fetcher::profile_cache_cap_drops_for_metrics()
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_full_cert_budget_ips Current IP entries tracked by TLS full-cert budget"
);
let _ = writeln!(out, "# TYPE telemt_tls_front_full_cert_budget_ips gauge");
let _ = writeln!(
out,
"telemt_tls_front_full_cert_budget_ips {}",
cache::full_cert_sent_ips_for_metrics()
);
let _ = writeln!(
out,
"# HELP telemt_tls_front_full_cert_budget_cap_drops_total New IPs denied full-cert budget tracking because the cap was reached"
);
let _ = writeln!(
out,
"# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter"
);
let _ = writeln!(
out,
"telemt_tls_front_full_cert_budget_cap_drops_total {}",
cache::full_cert_sent_cap_drops_for_metrics()
);
render_tls_front_profile_health(&mut out, config, tls_cache).await;
let _ = writeln!(
out,
"# HELP telemt_connections_total Total accepted connections"
@@ -396,6 +627,21 @@ async fn render_metrics(
}
);
let _ = writeln!(
out,
"# HELP telemt_connections_bad_by_class_total Bad/rejected connections by class"
);
let _ = writeln!(out, "# TYPE telemt_connections_bad_by_class_total counter");
if core_enabled {
for (class, total) in stats.get_connects_bad_class_counts() {
let _ = writeln!(
out,
"telemt_connections_bad_by_class_total{{class=\"{}\"}} {}",
class, total
);
}
}
let _ = writeln!(
out,
"# HELP telemt_handshake_timeouts_total Handshake timeouts"
@@ -411,6 +657,24 @@ async fn render_metrics(
}
);
let _ = writeln!(
out,
"# HELP telemt_handshake_failures_by_class_total Handshake failures by class"
);
let _ = writeln!(
out,
"# TYPE telemt_handshake_failures_by_class_total counter"
);
if core_enabled {
for (class, total) in stats.get_handshake_failure_class_counts() {
let _ = writeln!(
out,
"telemt_handshake_failures_by_class_total{{class=\"{}\"}} {}",
class, total
);
}
}
let _ = writeln!(
out,
"# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation"
@@ -462,6 +726,63 @@ async fn render_metrics(
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_refund_bytes_total Reserved quota bytes returned before commit"
);
let _ = writeln!(out, "# TYPE telemt_quota_refund_bytes_total counter");
let _ = writeln!(
out,
"telemt_quota_refund_bytes_total {}",
if core_enabled {
stats.get_quota_refund_bytes_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_contention_total Quota reservation CAS contention events"
);
let _ = writeln!(out, "# TYPE telemt_quota_contention_total counter");
let _ = writeln!(
out,
"telemt_quota_contention_total {}",
if core_enabled {
stats.get_quota_contention_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_contention_timeout_total Quota reservations that hit the bounded contention budget"
);
let _ = writeln!(out, "# TYPE telemt_quota_contention_timeout_total counter");
let _ = writeln!(
out,
"telemt_quota_contention_timeout_total {}",
if core_enabled {
stats.get_quota_contention_timeout_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_quota_acquire_cancelled_total Quota acquisitions cancelled before reservation completed"
);
let _ = writeln!(out, "# TYPE telemt_quota_acquire_cancelled_total counter");
let _ = writeln!(
out,
"telemt_quota_acquire_cancelled_total {}",
if core_enabled {
stats.get_quota_acquire_cancelled_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_conntrack_control_state Runtime conntrack control state flags"
@@ -576,6 +897,29 @@ async fn render_metrics(
);
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
let _ = writeln!(
out,
"# HELP telemt_rate_limiter_burst_bound_bytes Configured upper bound for one direct relay rate-limit burst"
);
let _ = writeln!(out, "# TYPE telemt_rate_limiter_burst_bound_bytes gauge");
let _ = writeln!(
out,
"telemt_rate_limiter_burst_bound_bytes{{direction=\"up\"}} {}",
if core_enabled {
config.general.direct_relay_copy_buf_c2s_bytes
} else {
0
}
);
let _ = writeln!(
out,
"telemt_rate_limiter_burst_bound_bytes{{direction=\"down\"}} {}",
if core_enabled {
config.general.direct_relay_copy_buf_s2c_bytes
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
@@ -1678,6 +2022,85 @@ async fn render_metrics(
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_child_join_timeout_total Middle relay child tasks that did not join before cleanup deadline"
);
let _ = writeln!(out, "# TYPE telemt_me_child_join_timeout_total counter");
let _ = writeln!(
out,
"telemt_me_child_join_timeout_total {}",
if core_enabled {
stats.get_me_child_join_timeout_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_me_child_abort_total Middle relay child tasks aborted after bounded cleanup timeout"
);
let _ = writeln!(out, "# TYPE telemt_me_child_abort_total counter");
let _ = writeln!(
out,
"telemt_me_child_abort_total {}",
if core_enabled {
stats.get_me_child_abort_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_flow_wait_events_total Flow wait events by reason, direction, and outcome"
);
let _ = writeln!(out, "# TYPE telemt_flow_wait_events_total counter");
let _ = writeln!(
out,
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"waited\"}} {}",
if core_enabled {
stats.get_flow_wait_middle_rate_limit_total()
} else {
0
}
);
let _ = writeln!(
out,
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"cancelled\"}} {}",
if core_enabled {
stats.get_flow_wait_middle_rate_limit_cancelled_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_flow_wait_ms_total Flow wait time in milliseconds by reason and direction"
);
let _ = writeln!(out, "# TYPE telemt_flow_wait_ms_total counter");
let _ = writeln!(
out,
"telemt_flow_wait_ms_total{{reason=\"middle_rate_limit\",direction=\"down\"}} {}",
if core_enabled {
stats.get_flow_wait_middle_rate_limit_ms_total()
} else {
0
}
);
let _ = writeln!(
out,
"# HELP telemt_session_drop_fallback_total Session reservations cleaned by Drop instead of explicit async release"
);
let _ = writeln!(out, "# TYPE telemt_session_drop_fallback_total counter");
let _ = writeln!(
out,
"telemt_session_drop_fallback_total {}",
if core_enabled {
stats.get_session_drop_fallback_total()
} else {
0
}
);
let _ = writeln!(
out,
@@ -3019,17 +3442,6 @@ async fn render_metrics(
0
}
);
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_suppressed {}",
if user_enabled { 0 } else { 1 }
);
let ip_memory = ip_tracker.memory_stats().await;
let _ = writeln!(
out,
@@ -3071,11 +3483,46 @@ async fn render_metrics(
"telemt_ip_tracker_cleanup_queue_len {}",
ip_memory.cleanup_queue_len
);
let _ = writeln!(
out,
"# HELP telemt_ip_tracker_cleanup_total Release cleanups deferred through the cleanup queue"
);
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cleanup_total counter");
let _ = writeln!(
out,
"telemt_ip_tracker_cleanup_total{{path=\"deferred\"}} {}",
ip_memory.cleanup_deferred_releases
);
let _ = writeln!(
out,
"# HELP telemt_ip_tracker_cap_rejects_total New connection rejects caused by global IP tracker caps"
);
let _ = writeln!(out, "# TYPE telemt_ip_tracker_cap_rejects_total counter");
let _ = writeln!(
out,
"telemt_ip_tracker_cap_rejects_total{{scope=\"active\"}} {}",
ip_memory.active_cap_rejects
);
let _ = writeln!(
out,
"telemt_ip_tracker_cap_rejects_total{{scope=\"recent\"}} {}",
ip_memory.recent_cap_rejects
);
let mut user_stats_emitted = 0usize;
let mut user_stats_suppressed = 0usize;
let mut unique_ip_emitted = 0usize;
let mut unique_ip_suppressed = 0usize;
if user_enabled {
for entry in stats.iter_user_stats() {
if user_stats_emitted >= USER_LABELED_METRICS_MAX_USERS {
user_stats_suppressed = user_stats_suppressed.saturating_add(1);
continue;
}
let user = entry.key();
let s = entry.value();
user_stats_emitted = user_stats_emitted.saturating_add(1);
let _ = writeln!(
out,
"telemt_user_connections_total{{user=\"{}\"}} {}",
@@ -3117,7 +3564,7 @@ async fn render_metrics(
);
}
let ip_stats = ip_tracker.get_stats().await;
let ip_stats = ip_tracker.get_stats_snapshot().await;
let ip_counts: HashMap<String, usize> = ip_stats
.into_iter()
.map(|(user, count, _)| (user, count))
@@ -3129,7 +3576,7 @@ async fn render_metrics(
unique_users.extend(ip_counts.keys().cloned());
let unique_users_vec: Vec<String> = unique_users.iter().cloned().collect();
let recent_counts = ip_tracker
.get_recent_counts_for_users(&unique_users_vec)
.get_recent_counts_for_users_snapshot(&unique_users_vec)
.await;
let _ = writeln!(
@@ -3154,6 +3601,11 @@ async fn render_metrics(
let _ = writeln!(out, "# TYPE telemt_user_unique_ips_utilization gauge");
for user in unique_users {
if unique_ip_emitted >= USER_LABELED_METRICS_MAX_USERS {
unique_ip_suppressed = unique_ip_suppressed.saturating_add(1);
continue;
}
unique_ip_emitted = unique_ip_emitted.saturating_add(1);
let current = ip_counts.get(&user).copied().unwrap_or(0);
let limit = config
.access
@@ -3193,6 +3645,46 @@ async fn render_metrics(
}
}
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_suppressed gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_suppressed {}",
if user_enabled && user_stats_suppressed == 0 && unique_ip_suppressed == 0 {
0
} else {
1
}
);
let _ = writeln!(
out,
"# HELP telemt_telemetry_user_series_users User-labeled metric users by export status"
);
let _ = writeln!(out, "# TYPE telemt_telemetry_user_series_users gauge");
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"emitted\"}} {}",
user_stats_emitted
);
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"stats\",status=\"suppressed\"}} {}",
user_stats_suppressed
);
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"emitted\"}} {}",
unique_ip_emitted
);
let _ = writeln!(
out,
"telemt_telemetry_user_series_users{{family=\"unique_ip\",status=\"suppressed\"}} {}",
unique_ip_suppressed
);
out
}
@@ -3201,6 +3693,11 @@ mod tests {
use super::*;
use http_body_util::BodyExt;
use std::net::IpAddr;
use std::time::SystemTime;
use crate::tls_front::types::{
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
};
#[tokio::test]
async fn test_render_metrics_format() {
@@ -3215,8 +3712,9 @@ mod tests {
stats.increment_connects_all();
stats.increment_connects_all();
stats.increment_connects_bad();
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
stats.increment_handshake_timeouts();
stats.increment_handshake_failure_class("timeout");
shared_state
.handshake
.auth_expensive_checks_total
@@ -3268,7 +3766,7 @@ mod tests {
.await
.unwrap();
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker).await;
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker, None).await;
assert!(output.contains(&format!(
"telemt_build_info{{version=\"{}\"}} 1",
@@ -3276,7 +3774,11 @@ mod tests {
)));
assert!(output.contains("telemt_connections_total 2"));
assert!(output.contains("telemt_connections_bad_total 1"));
assert!(output.contains(
"telemt_connections_bad_by_class_total{class=\"tls_handshake_bad_client\"} 1"
));
assert!(output.contains("telemt_handshake_timeouts_total 1"));
assert!(output.contains("telemt_handshake_failures_by_class_total{class=\"timeout\"} 1"));
assert!(output.contains("telemt_auth_expensive_checks_total 9"));
assert!(output.contains("telemt_auth_budget_exhausted_total 2"));
assert!(output.contains("telemt_upstream_connect_attempt_total 2"));
@@ -3330,13 +3832,91 @@ mod tests {
assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0"));
}
#[tokio::test]
async fn test_render_tls_front_profile_health() {
let stats = Stats::new();
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let mut config = ProxyConfig::default();
config.censorship.tls_domain = "primary.example".to_string();
config.censorship.tls_domains = vec!["fallback.example".to_string()];
let cache = TlsFrontCache::new(
&[
"primary.example".to_string(),
"fallback.example".to_string(),
],
1024,
"tlsfront-profile-health-test",
);
cache
.set(
"primary.example",
CachedTlsData {
server_hello_template: ParsedServerHello {
version: [0x03, 0x03],
random: [0u8; 32],
session_id: Vec::new(),
cipher_suite: [0x13, 0x01],
compression: 0,
extensions: Vec::new(),
},
cert_info: None,
cert_payload: Some(TlsCertPayload {
cert_chain_der: vec![vec![0x30, 0x01]],
certificate_message: vec![0x0b, 0x00, 0x00, 0x00],
}),
app_data_records_sizes: vec![1024, 512],
total_app_data_len: 1536,
behavior_profile: TlsBehaviorProfile {
change_cipher_spec_count: 1,
app_data_record_sizes: vec![1024, 512],
ticket_record_sizes: vec![69],
source: TlsProfileSource::Merged,
},
fetched_at: SystemTime::now(),
domain: "primary.example".to_string(),
},
)
.await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, Some(&cache)).await;
assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2"));
assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2"));
assert!(output.contains("telemt_tls_front_profile_domains{status=\"suppressed\"} 0"));
assert!(
output.contains("telemt_tls_front_profile_info{domain=\"primary.example\",source=\"merged\",is_default=\"false\",has_cert_info=\"false\",has_cert_payload=\"true\"} 1")
);
assert!(
output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1")
);
assert!(
output.contains(
"telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2"
)
);
assert!(
output
.contains("telemt_tls_front_profile_ticket_records{domain=\"primary.example\"} 1")
);
assert!(output.contains(
"telemt_tls_front_profile_change_cipher_spec_records{domain=\"primary.example\"} 1"
));
assert!(
output.contains(
"telemt_tls_front_profile_app_data_bytes{domain=\"primary.example\"} 1536"
)
);
}
#[tokio::test]
async fn test_render_empty_stats() {
let stats = Stats::new();
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
assert!(output.contains("telemt_connections_total 0"));
assert!(output.contains("telemt_connections_bad_total 0"));
assert!(output.contains("telemt_handshake_timeouts_total 0"));
@@ -3360,7 +3940,7 @@ mod tests {
let mut config = ProxyConfig::default();
config.access.user_max_unique_ips_global_each = 2;
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
@@ -3372,11 +3952,13 @@ mod tests {
let shared_state = ProxySharedState::new();
let tracker = UserIpTracker::new();
let config = ProxyConfig::default();
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
assert!(output.contains("# TYPE telemt_connections_total counter"));
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
assert!(output.contains("# TYPE telemt_connections_bad_by_class_total counter"));
assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter"));
assert!(output.contains("# TYPE telemt_handshake_failures_by_class_total counter"));
assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter"));
assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter"));
assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter"));
@@ -3406,9 +3988,28 @@ mod tests {
assert!(output.contains("# TYPE telemt_user_unique_ips_recent_window gauge"));
assert!(output.contains("# TYPE telemt_user_unique_ips_limit gauge"));
assert!(output.contains("# TYPE telemt_user_unique_ips_utilization gauge"));
assert!(output.contains("# TYPE telemt_stats_user_entries gauge"));
assert!(output.contains("# TYPE telemt_telemetry_user_series_users gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_users gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_entries gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_queue_len gauge"));
assert!(output.contains("# TYPE telemt_ip_tracker_cleanup_total counter"));
assert!(output.contains("# TYPE telemt_ip_tracker_cap_rejects_total counter"));
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_entries gauge"));
assert!(output.contains("# TYPE telemt_tls_fetch_profile_cache_cap_drops_total counter"));
assert!(output.contains("# TYPE telemt_tls_front_full_cert_budget_ips gauge"));
assert!(
output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter")
);
assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge"));
assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge"));
assert!(
output.contains("# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge")
);
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_bytes gauge"));
}
#[tokio::test]
@@ -3429,6 +4030,7 @@ mod tests {
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
@@ -3463,6 +4065,7 @@ mod tests {
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await
@@ -3480,6 +4083,7 @@ mod tests {
&beobachten,
shared_state.as_ref(),
&tracker,
None,
&config,
)
.await

View File

@@ -32,7 +32,13 @@ struct UserConnectionReservation {
user: String,
ip: IpAddr,
tracks_ip: bool,
active: bool,
state: SessionReservationState,
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum SessionReservationState {
Active,
Released,
}
impl UserConnectionReservation {
@@ -49,28 +55,35 @@ impl UserConnectionReservation {
user,
ip,
tracks_ip,
active: true,
state: SessionReservationState::Active,
}
}
fn mark_released(&mut self) -> bool {
if self.state != SessionReservationState::Active {
return false;
}
self.state = SessionReservationState::Released;
true
}
async fn release(mut self) {
if !self.active {
if !self.mark_released() {
return;
}
if self.tracks_ip {
self.ip_tracker.remove_ip(&self.user, self.ip).await;
}
self.active = false;
self.stats.decrement_user_curr_connects(&self.user);
}
}
impl Drop for UserConnectionReservation {
fn drop(&mut self) {
if !self.active {
if !self.mark_released() {
return;
}
self.active = false;
self.stats.increment_session_drop_fallback_total();
self.stats.decrement_user_curr_connects(&self.user);
if self.tracks_ip {
self.ip_tracker.enqueue_cleanup(self.user.clone(), self.ip);
@@ -466,6 +479,17 @@ where
let mut local_addr = synthetic_local_addr(config.server.port);
if proxy_protocol_enabled {
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs) {
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
warn!(
peer = %peer,
trusted = ?config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
}
let proxy_header_timeout =
Duration::from_millis(config.server.proxy_protocol_header_timeout_ms.max(1));
match timeout(
@@ -475,17 +499,6 @@ where
.await
{
Ok(Ok(info)) => {
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs)
{
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
warn!(
peer = %peer,
trusted = ?config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
}
debug!(
peer = %peer,
client = %info.src_addr,
@@ -978,6 +991,21 @@ impl RunningClientHandler {
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
if self.proxy_protocol_enabled {
if !is_trusted_proxy_source(
self.peer.ip(),
&self.config.server.proxy_protocol_trusted_cidrs,
) {
self.stats
.increment_connects_bad_with_class("proxy_protocol_untrusted");
warn!(
peer = %self.peer,
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(&self.beobachten, &self.config, self.peer.ip(), "other");
return Err(ProxyError::InvalidProxyProtocol);
}
let proxy_header_timeout =
Duration::from_millis(self.config.server.proxy_protocol_header_timeout_ms.max(1));
match timeout(
@@ -987,25 +1015,6 @@ impl RunningClientHandler {
.await
{
Ok(Ok(info)) => {
if !is_trusted_proxy_source(
self.peer.ip(),
&self.config.server.proxy_protocol_trusted_cidrs,
) {
self.stats
.increment_connects_bad_with_class("proxy_protocol_untrusted");
warn!(
peer = %self.peer,
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
"Rejecting PROXY protocol header from untrusted source"
);
record_beobachten_class(
&self.beobachten,
&self.config,
self.peer.ip(),
"other",
);
return Err(ProxyError::InvalidProxyProtocol);
}
debug!(
peer = %self.peer,
client = %info.src_addr,
@@ -1431,8 +1440,8 @@ impl RunningClientHandler {
/// Main dispatch after successful handshake.
/// Two modes:
/// - Direct: TCP relay to TG DC (existing behavior)
/// - Middle Proxy: RPC multiplex through ME pool (new — supports CDN DCs)
/// - Direct: TCP relay to TG DC (existing behavior)
/// - Middle Proxy: RPC multiplex through ME pool (supports CDN DCs)
#[cfg(test)]
async fn handle_authenticated_static<R, W>(
client_reader: CryptoReader<R>,

View File

@@ -18,8 +18,7 @@ use crate::error::{ProxyError, Result};
use crate::protocol::constants::*;
use crate::proxy::handshake::{HandshakeSuccess, encrypt_tg_nonce_with_ciphers, generate_tg_nonce};
use crate::proxy::route_mode::{
ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state,
cutover_stagger_delay,
RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay,
};
use crate::proxy::shared_state::{
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
@@ -360,7 +359,7 @@ where
"Cutover affected direct session, closing client connection"
);
tokio::time::sleep(delay).await;
break Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
break Err(ProxyError::RouteSwitched);
}
tokio::select! {
result = &mut relay_result => {

View File

@@ -1450,6 +1450,20 @@ where
validated_secret.copy_from_slice(secret);
}
if config
.access
.is_user_source_ip_denied(validated_user.as_str(), peer.ip())
{
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
warn!(
peer = %peer,
user = %validated_user,
"TLS handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
);
return HandshakeResult::BadClient { reader, writer };
}
// Reject known replay digests before expensive cache/domain/ALPN policy work.
let digest_half = &validation_digest[..tls::TLS_DIGEST_HALF_LEN];
if replay_checker.check_tls_digest(digest_half) {
@@ -1795,6 +1809,20 @@ where
let validation = matched_validation.expect("validation must exist when matched");
if config
.access
.is_user_source_ip_denied(matched_user.as_str(), peer.ip())
{
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
warn!(
peer = %peer,
user = %matched_user,
"MTProto handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
);
return HandshakeResult::BadClient { reader, writer };
}
// Apply replay tracking only after successful authentication.
//
// This ordering prevents an attacker from producing invalid handshakes that
@@ -1873,6 +1901,20 @@ where
.auth_expensive_checks_total
.fetch_add(validation_checks as u64, Ordering::Relaxed);
if config
.access
.is_user_source_ip_denied(user.as_str(), peer.ip())
{
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
maybe_apply_server_hello_delay(config).await;
warn!(
peer = %peer,
user = %user,
"MTProto handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
);
return HandshakeResult::BadClient { reader, writer };
}
// Apply replay tracking only after successful authentication.
//
// This ordering prevents an attacker from producing invalid handshakes that

View File

@@ -2,6 +2,7 @@
use crate::config::ProxyConfig;
use crate::network::dns_overrides::resolve_socket_addr;
use crate::protocol::tls;
use crate::stats::beobachten::BeobachtenStore;
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
#[cfg(unix)]
@@ -328,6 +329,89 @@ async fn wait_mask_outcome_budget(started: Instant, config: &ProxyConfig) {
}
}
#[cfg(test)]
mod tls_domain_mask_host_tests {
use super::{mask_host_for_initial_data, matching_tls_domain_for_sni};
use crate::config::ProxyConfig;
fn client_hello_with_sni(sni_host: &str) -> Vec<u8> {
let mut body = Vec::new();
body.extend_from_slice(&[0x03, 0x03]);
body.extend_from_slice(&[0u8; 32]);
body.push(32);
body.extend_from_slice(&[0x42u8; 32]);
body.extend_from_slice(&2u16.to_be_bytes());
body.extend_from_slice(&[0x13, 0x01]);
body.push(1);
body.push(0);
let host_bytes = sni_host.as_bytes();
let mut sni_payload = Vec::new();
sni_payload.extend_from_slice(&((host_bytes.len() + 3) as u16).to_be_bytes());
sni_payload.push(0);
sni_payload.extend_from_slice(&(host_bytes.len() as u16).to_be_bytes());
sni_payload.extend_from_slice(host_bytes);
let mut extensions = Vec::new();
extensions.extend_from_slice(&0x0000u16.to_be_bytes());
extensions.extend_from_slice(&(sni_payload.len() as u16).to_be_bytes());
extensions.extend_from_slice(&sni_payload);
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
body.extend_from_slice(&extensions);
let mut handshake = Vec::new();
handshake.push(0x01);
let body_len = (body.len() as u32).to_be_bytes();
handshake.extend_from_slice(&body_len[1..4]);
handshake.extend_from_slice(&body);
let mut record = Vec::new();
record.push(0x16);
record.extend_from_slice(&[0x03, 0x01]);
record.extend_from_slice(&(handshake.len() as u16).to_be_bytes());
record.extend_from_slice(&handshake);
record
}
fn config_with_tls_domains() -> ProxyConfig {
let mut config = ProxyConfig::default();
config.censorship.tls_domain = "a.com".to_string();
config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()];
config.censorship.mask_host = Some("a.com".to_string());
config
}
#[test]
fn matching_tls_domain_accepts_primary_and_extra_domains_case_insensitively() {
let config = config_with_tls_domains();
assert_eq!(matching_tls_domain_for_sni(&config, "A.COM"), Some("a.com"));
assert_eq!(matching_tls_domain_for_sni(&config, "B.COM"), Some("b.com"));
assert_eq!(matching_tls_domain_for_sni(&config, "unknown.com"), None);
}
#[test]
fn mask_host_preserves_explicit_non_primary_origin() {
let mut config = config_with_tls_domains();
config.censorship.mask_host = Some("origin.example".to_string());
let initial_data = client_hello_with_sni("b.com");
assert_eq!(
mask_host_for_initial_data(&config, &initial_data),
"origin.example"
);
}
#[test]
fn mask_host_uses_matching_tls_domain_when_mask_host_is_primary_default() {
let config = config_with_tls_domains();
let initial_data = client_hello_with_sni("b.com");
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com");
}
}
/// Detect client type based on initial data
fn detect_client_type(data: &[u8]) -> &'static str {
// Check for HTTP request
@@ -360,6 +444,37 @@ fn parse_mask_host_ip_literal(host: &str) -> Option<IpAddr> {
host.parse::<IpAddr>().ok()
}
fn matching_tls_domain_for_sni<'a>(config: &'a ProxyConfig, sni: &str) -> Option<&'a str> {
if config.censorship.tls_domain.eq_ignore_ascii_case(sni) {
return Some(config.censorship.tls_domain.as_str());
}
for domain in &config.censorship.tls_domains {
if domain.eq_ignore_ascii_case(sni) {
return Some(domain.as_str());
}
}
None
}
fn mask_host_for_initial_data<'a>(config: &'a ProxyConfig, initial_data: &[u8]) -> &'a str {
let configured_mask_host = config
.censorship
.mask_host
.as_deref()
.unwrap_or(&config.censorship.tls_domain);
if !configured_mask_host.eq_ignore_ascii_case(&config.censorship.tls_domain) {
return configured_mask_host;
}
tls::extract_sni_from_client_hello(initial_data)
.as_deref()
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
.unwrap_or(configured_mask_host)
}
fn canonical_ip(ip: IpAddr) -> IpAddr {
match ip {
IpAddr::V6(v6) => v6
@@ -734,11 +849,7 @@ pub async fn handle_bad_client<R, W>(
return;
}
let mask_host = config
.censorship
.mask_host
.as_deref()
.unwrap_or(&config.censorship.tls_domain);
let mask_host = mask_host_for_initial_data(config, initial_data);
let mask_port = config.censorship.mask_port;
// Fail closed when fallback points at our own listener endpoint.

View File

@@ -14,6 +14,7 @@ use std::time::{Duration, Instant};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch};
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, trace, warn};
use crate::config::{ConntrackPressureProfile, ProxyConfig};
@@ -22,8 +23,7 @@ use crate::error::{ProxyError, Result};
use crate::protocol::constants::{secure_padding_len, *};
use crate::proxy::handshake::HandshakeSuccess;
use crate::proxy::route_mode::{
ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state,
cutover_stagger_delay,
RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay,
};
use crate::proxy::shared_state::{
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
@@ -65,6 +65,15 @@ const ME_D2C_SINGLE_WRITE_COALESCE_MAX_BYTES: usize = 128 * 1024;
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
const QUOTA_RESERVE_BACKOFF_MIN_MS: u64 = 1;
const QUOTA_RESERVE_BACKOFF_MAX_MS: u64 = 16;
const QUOTA_RESERVE_MAX_BACKOFF_ROUNDS: usize = 16;
const ME_CHILD_JOIN_TIMEOUT: Duration = Duration::from_secs(2);
enum MiddleQuotaReserveError {
LimitExceeded,
Contended,
Cancelled,
DeadlineExceeded,
}
#[derive(Default)]
pub(crate) struct DesyncDedupRotationState {
@@ -622,21 +631,43 @@ async fn reserve_user_quota_with_yield(
user_stats: &UserStats,
bytes: u64,
limit: u64,
) -> std::result::Result<u64, QuotaReserveError> {
stats: &Stats,
cancel: &CancellationToken,
deadline: Option<Instant>,
) -> std::result::Result<u64, MiddleQuotaReserveError> {
let mut backoff_ms = QUOTA_RESERVE_BACKOFF_MIN_MS;
let mut backoff_rounds = 0usize;
loop {
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match user_stats.quota_try_reserve(bytes, limit) {
Ok(total) => return Ok(total),
Err(QuotaReserveError::LimitExceeded) => {
return Err(QuotaReserveError::LimitExceeded);
return Err(MiddleQuotaReserveError::LimitExceeded);
}
Err(QuotaReserveError::Contended) => {
stats.increment_quota_contention_total();
std::hint::spin_loop();
}
Err(QuotaReserveError::Contended) => std::hint::spin_loop(),
}
}
tokio::task::yield_now().await;
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
stats.increment_quota_contention_timeout_total();
return Err(MiddleQuotaReserveError::DeadlineExceeded);
}
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {}
_ = cancel.cancelled() => {
stats.increment_quota_acquire_cancelled_total();
return Err(MiddleQuotaReserveError::Cancelled);
}
}
backoff_rounds = backoff_rounds.saturating_add(1);
if backoff_rounds >= QUOTA_RESERVE_MAX_BACKOFF_ROUNDS {
stats.increment_quota_contention_timeout_total();
return Err(MiddleQuotaReserveError::Contended);
}
backoff_ms = backoff_ms
.saturating_mul(2)
.min(QUOTA_RESERVE_BACKOFF_MAX_MS);
@@ -647,12 +678,13 @@ async fn wait_for_traffic_budget(
lease: Option<&Arc<TrafficLease>>,
direction: RateDirection,
bytes: u64,
) {
deadline: Option<Instant>,
) -> Result<()> {
if bytes == 0 {
return;
return Ok(());
}
let Some(lease) = lease else {
return;
return Ok(());
};
let mut remaining = bytes;
@@ -664,6 +696,9 @@ async fn wait_for_traffic_budget(
}
let wait_started_at = Instant::now();
if deadline.is_some_and(|deadline| wait_started_at >= deadline) {
return Err(ProxyError::TrafficBudgetWaitDeadlineExceeded);
}
tokio::time::sleep(next_refill_delay()).await;
let wait_ms = wait_started_at
.elapsed()
@@ -676,6 +711,59 @@ async fn wait_for_traffic_budget(
wait_ms,
);
}
Ok(())
}
async fn wait_for_traffic_budget_or_cancel(
lease: Option<&Arc<TrafficLease>>,
direction: RateDirection,
bytes: u64,
cancel: &CancellationToken,
stats: &Stats,
deadline: Option<Instant>,
) -> Result<()> {
if bytes == 0 {
return Ok(());
}
let Some(lease) = lease else {
return Ok(());
};
let mut remaining = bytes;
while remaining > 0 {
let consume = lease.try_consume(direction, remaining);
if consume.granted > 0 {
remaining = remaining.saturating_sub(consume.granted);
continue;
}
let wait_started_at = Instant::now();
if deadline.is_some_and(|deadline| wait_started_at >= deadline) {
stats.increment_flow_wait_middle_rate_limit_cancelled_total();
return Err(ProxyError::TrafficBudgetWaitDeadlineExceeded);
}
tokio::select! {
_ = tokio::time::sleep(next_refill_delay()) => {}
_ = cancel.cancelled() => {
stats.increment_flow_wait_middle_rate_limit_cancelled_total();
return Err(ProxyError::TrafficBudgetWaitCancelled);
}
}
let wait_ms = wait_started_at
.elapsed()
.as_millis()
.min(u128::from(u64::MAX)) as u64;
lease.observe_wait_ms(
direction,
consume.blocked_user,
consume.blocked_cidr,
wait_ms,
);
stats.observe_flow_wait_middle_rate_limit_ms(wait_ms);
}
Ok(())
}
fn classify_me_d2c_flush_reason(
@@ -1114,7 +1202,7 @@ where
tokio::time::sleep(delay).await;
let _ = me_pool.send_close(conn_id).await;
me_pool.registry().unregister(conn_id).await;
return Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
return Err(ProxyError::RouteSwitched);
}
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
@@ -1169,7 +1257,7 @@ where
let c2me_byte_semaphore = Arc::new(Semaphore::new(c2me_byte_budget));
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
let me_pool_c2me = me_pool.clone();
let c2me_sender = tokio::spawn(async move {
let mut c2me_sender = tokio::spawn(async move {
let mut sent_since_yield = 0usize;
while let Some(cmd) = c2me_rx.recv().await {
match cmd {
@@ -1205,16 +1293,18 @@ where
});
let (stop_tx, mut stop_rx) = oneshot::channel::<()>();
let flow_cancel = CancellationToken::new();
let mut me_rx_task = me_rx;
let stats_clone = stats.clone();
let rng_clone = rng.clone();
let user_clone = user.clone();
let quota_user_stats_me_writer = quota_user_stats.clone();
let traffic_lease_me_writer = traffic_lease.clone();
let flow_cancel_me_writer = flow_cancel.clone();
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
let bytes_me2c_clone = bytes_me2c.clone();
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
let me_writer = tokio::spawn(async move {
let mut me_writer = tokio::spawn(async move {
let mut writer = crypto_writer;
let mut frame_buf = Vec::with_capacity(16 * 1024);
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
@@ -1234,7 +1324,7 @@ where
let Some(first) = msg else {
debug!(conn_id, "ME channel closed");
shrink_session_vec(&mut frame_buf, shrink_threshold);
return Err(ProxyError::Proxy("ME connection lost".into()));
return Err(ProxyError::MiddleConnectionLost);
};
let mut batch_frames = 0usize;
@@ -1256,6 +1346,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1276,7 +1367,7 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1317,6 +1408,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1338,7 +1430,8 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ =
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1381,6 +1474,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1405,7 +1499,11 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ = flush_client_or_cancel(
&mut writer,
&flow_cancel_me_writer,
)
.await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1447,6 +1545,7 @@ where
quota_limit,
d2c_flush_policy.quota_soft_overshoot_bytes,
traffic_lease_me_writer.as_ref(),
&flow_cancel_me_writer,
bytes_me2c_clone.as_ref(),
conn_id,
d2c_flush_policy.ack_flush_immediate,
@@ -1471,7 +1570,11 @@ where
} else {
None
};
let _ = writer.flush().await;
let _ = flush_client_or_cancel(
&mut writer,
&flow_cancel_me_writer,
)
.await;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1495,7 +1598,7 @@ where
Ok(None) => {
debug!(conn_id, "ME channel closed");
shrink_session_vec(&mut frame_buf, shrink_threshold);
return Err(ProxyError::Proxy("ME connection lost".into()));
return Err(ProxyError::MiddleConnectionLost);
}
Err(_) => {
max_delay_fired = true;
@@ -1517,7 +1620,7 @@ where
} else {
None
};
writer.flush().await.map_err(ProxyError::Io)?;
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
let flush_duration_us = flush_started_at.map(|started| {
started
.elapsed()
@@ -1610,7 +1713,7 @@ where
stats.as_ref(),
)
.await;
main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
main_result = Err(ProxyError::RouteSwitched);
break;
}
@@ -1641,26 +1744,50 @@ where
traffic_lease.as_ref(),
RateDirection::Up,
payload.len() as u64,
None,
)
.await;
.await?;
forensics.bytes_c2me = forensics
.bytes_c2me
.saturating_add(payload.len() as u64);
if let (Some(limit), Some(user_stats)) =
(quota_limit, quota_user_stats.as_deref())
{
if reserve_user_quota_with_yield(
match reserve_user_quota_with_yield(
user_stats,
payload.len() as u64,
limit,
stats.as_ref(),
&flow_cancel,
None,
)
.await
.is_err()
{
main_result = Err(ProxyError::DataQuotaExceeded {
user: user.clone(),
});
break;
Ok(_) => {}
Err(MiddleQuotaReserveError::LimitExceeded) => {
main_result = Err(ProxyError::DataQuotaExceeded {
user: user.clone(),
});
break;
}
Err(MiddleQuotaReserveError::Contended) => {
main_result = Err(ProxyError::Proxy(
"ME C->ME quota reservation contended".into(),
));
break;
}
Err(MiddleQuotaReserveError::Cancelled) => {
main_result = Err(ProxyError::Proxy(
"ME C->ME quota reservation cancelled".into(),
));
break;
}
Err(MiddleQuotaReserveError::DeadlineExceeded) => {
main_result = Err(ProxyError::Proxy(
"ME C->ME quota reservation deadline exceeded".into(),
));
break;
}
}
stats.add_user_octets_from_handle(user_stats, payload.len() as u64);
} else {
@@ -1729,22 +1856,34 @@ where
}
drop(c2me_tx);
let c2me_result = c2me_sender
.await
.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME sender join error: {e}"))));
let c2me_result = match timeout(ME_CHILD_JOIN_TIMEOUT, &mut c2me_sender).await {
Ok(joined) => {
joined.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME sender join error: {e}"))))
}
Err(_) => {
stats.increment_me_child_join_timeout_total();
stats.increment_me_child_abort_total();
c2me_sender.abort();
Err(ProxyError::Proxy("ME sender join timeout".into()))
}
};
flow_cancel.cancel();
let _ = stop_tx.send(());
let mut writer_result = me_writer
.await
.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME writer join error: {e}"))));
let mut writer_result = match timeout(ME_CHILD_JOIN_TIMEOUT, &mut me_writer).await {
Ok(joined) => {
joined.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME writer join error: {e}"))))
}
Err(_) => {
stats.increment_me_child_join_timeout_total();
stats.increment_me_child_abort_total();
me_writer.abort();
Err(ProxyError::Proxy("ME writer join timeout".into()))
}
};
// When client closes, but ME channel stopped as unregistered - it isnt error
if client_closed
&& matches!(
writer_result,
Err(ProxyError::Proxy(ref msg)) if msg == "ME connection lost"
)
{
if client_closed && matches!(writer_result, Err(ProxyError::MiddleConnectionLost)) {
writer_result = Ok(());
}
@@ -2300,6 +2439,7 @@ where
quota_limit,
quota_soft_overshoot_bytes,
None,
&CancellationToken::new(),
bytes_me2c,
conn_id,
ack_flush_immediate,
@@ -2320,6 +2460,7 @@ async fn process_me_writer_response_with_traffic_lease<W>(
quota_limit: Option<u64>,
quota_soft_overshoot_bytes: u64,
traffic_lease: Option<&Arc<TrafficLease>>,
cancel: &CancellationToken,
bytes_me2c: &AtomicU64,
conn_id: u64,
ack_flush_immediate: bool,
@@ -2338,31 +2479,65 @@ where
let data_len = data.len() as u64;
if let (Some(limit), Some(user_stats)) = (quota_limit, quota_user_stats) {
let soft_limit = quota_soft_cap(limit, quota_soft_overshoot_bytes);
if reserve_user_quota_with_yield(user_stats, data_len, soft_limit)
.await
.is_err()
match reserve_user_quota_with_yield(
user_stats, data_len, soft_limit, stats, cancel, None,
)
.await
{
stats.increment_me_d2c_quota_reject_total(MeD2cQuotaRejectStage::PreWrite);
return Err(ProxyError::DataQuotaExceeded {
user: user.to_string(),
});
Ok(_) => {}
Err(MiddleQuotaReserveError::LimitExceeded) => {
stats.increment_me_d2c_quota_reject_total(MeD2cQuotaRejectStage::PreWrite);
return Err(ProxyError::DataQuotaExceeded {
user: user.to_string(),
});
}
Err(MiddleQuotaReserveError::Contended) => {
return Err(ProxyError::Proxy(
"ME D->C quota reservation contended".into(),
));
}
Err(MiddleQuotaReserveError::Cancelled) => {
return Err(ProxyError::Proxy(
"ME D->C quota reservation cancelled".into(),
));
}
Err(MiddleQuotaReserveError::DeadlineExceeded) => {
return Err(ProxyError::Proxy(
"ME D->C quota reservation deadline exceeded".into(),
));
}
}
}
wait_for_traffic_budget(traffic_lease, RateDirection::Down, data_len).await;
wait_for_traffic_budget_or_cancel(
traffic_lease,
RateDirection::Down,
data_len,
cancel,
stats,
None,
)
.await?;
let write_mode =
match write_client_payload(client_writer, proto_tag, flags, &data, rng, frame_buf)
.await
{
Ok(mode) => mode,
Err(err) => {
if quota_limit.is_some() {
stats.add_quota_write_fail_bytes_total(data_len);
stats.increment_quota_write_fail_events_total();
}
return Err(err);
let write_mode = match write_client_payload(
client_writer,
proto_tag,
flags,
&data,
rng,
frame_buf,
cancel,
)
.await
{
Ok(mode) => mode,
Err(err) => {
if quota_limit.is_some() {
stats.add_quota_write_fail_bytes_total(data_len);
stats.increment_quota_write_fail_events_total();
}
};
return Err(err);
}
};
bytes_me2c.fetch_add(data_len, Ordering::Relaxed);
if let Some(user_stats) = quota_user_stats {
@@ -2386,8 +2561,16 @@ where
} else {
trace!(conn_id, confirm, "ME->C quickack");
}
wait_for_traffic_budget(traffic_lease, RateDirection::Down, 4).await;
write_client_ack(client_writer, proto_tag, confirm).await?;
wait_for_traffic_budget_or_cancel(
traffic_lease,
RateDirection::Down,
4,
cancel,
stats,
None,
)
.await?;
write_client_ack(client_writer, proto_tag, confirm, cancel).await?;
stats.increment_me_d2c_ack_frames_total();
Ok(MeWriterResponseOutcome::Continue {
@@ -2439,6 +2622,7 @@ async fn write_client_payload<W>(
data: &[u8],
rng: &SecureRandom,
frame_buf: &mut Vec<u8>,
cancel: &CancellationToken,
) -> Result<MeD2cWriteMode>
where
W: AsyncWrite + Unpin + Send + 'static,
@@ -2466,21 +2650,12 @@ where
frame_buf.reserve(wire_len);
frame_buf.push(first);
frame_buf.extend_from_slice(data);
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
MeD2cWriteMode::Coalesced
} else {
let header = [first];
client_writer
.write_all(&header)
.await
.map_err(ProxyError::Io)?;
client_writer
.write_all(data)
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, &header, cancel).await?;
write_all_client_or_cancel(client_writer, data, cancel).await?;
MeD2cWriteMode::Split
}
} else if len_words < (1 << 24) {
@@ -2495,21 +2670,12 @@ where
frame_buf.reserve(wire_len);
frame_buf.extend_from_slice(&[first, lw[0], lw[1], lw[2]]);
frame_buf.extend_from_slice(data);
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
MeD2cWriteMode::Coalesced
} else {
let header = [first, lw[0], lw[1], lw[2]];
client_writer
.write_all(&header)
.await
.map_err(ProxyError::Io)?;
client_writer
.write_all(data)
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, &header, cancel).await?;
write_all_client_or_cancel(client_writer, data, cancel).await?;
MeD2cWriteMode::Split
}
} else {
@@ -2544,21 +2710,12 @@ where
frame_buf.resize(start + padding_len, 0);
rng.fill(&mut frame_buf[start..]);
}
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
MeD2cWriteMode::Coalesced
} else {
let header = len_val.to_le_bytes();
client_writer
.write_all(&header)
.await
.map_err(ProxyError::Io)?;
client_writer
.write_all(data)
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, &header, cancel).await?;
write_all_client_or_cancel(client_writer, data, cancel).await?;
if padding_len > 0 {
frame_buf.clear();
if frame_buf.capacity() < padding_len {
@@ -2566,10 +2723,7 @@ where
}
frame_buf.resize(padding_len, 0);
rng.fill(frame_buf.as_mut_slice());
client_writer
.write_all(frame_buf.as_slice())
.await
.map_err(ProxyError::Io)?;
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
}
MeD2cWriteMode::Split
}
@@ -2583,6 +2737,7 @@ async fn write_client_ack<W>(
client_writer: &mut CryptoWriter<W>,
proto_tag: ProtoTag,
confirm: u32,
cancel: &CancellationToken,
) -> Result<()>
where
W: AsyncWrite + Unpin + Send + 'static,
@@ -2592,10 +2747,34 @@ where
} else {
confirm.to_le_bytes()
};
client_writer
.write_all(&bytes)
.await
.map_err(ProxyError::Io)
write_all_client_or_cancel(client_writer, &bytes, cancel).await
}
async fn write_all_client_or_cancel<W>(
client_writer: &mut CryptoWriter<W>,
bytes: &[u8],
cancel: &CancellationToken,
) -> Result<()>
where
W: AsyncWrite + Unpin + Send + 'static,
{
tokio::select! {
result = client_writer.write_all(bytes) => result.map_err(ProxyError::Io),
_ = cancel.cancelled() => Err(ProxyError::MiddleClientWriterCancelled),
}
}
async fn flush_client_or_cancel<W>(
client_writer: &mut CryptoWriter<W>,
cancel: &CancellationToken,
) -> Result<()>
where
W: AsyncWrite + Unpin + Send + 'static,
{
tokio::select! {
result = client_writer.flush() => result.map_err(ProxyError::Io),
_ = cancel.cancelled() => Err(ProxyError::MiddleClientWriterCancelled),
}
}
#[cfg(test)]

View File

@@ -215,6 +215,7 @@ struct StatsIo<S> {
c2s_rate_debt_bytes: u64,
c2s_wait: RateWaitState,
s2c_wait: RateWaitState,
quota_wait: RateWaitState,
quota_limit: Option<u64>,
quota_exceeded: Arc<AtomicBool>,
quota_bytes_since_check: u64,
@@ -275,6 +276,7 @@ impl<S> StatsIo<S> {
c2s_rate_debt_bytes: 0,
c2s_wait: RateWaitState::default(),
s2c_wait: RateWaitState::default(),
quota_wait: RateWaitState::default(),
quota_limit,
quota_exceeded,
quota_bytes_since_check: 0,
@@ -353,6 +355,11 @@ impl<S> StatsIo<S> {
Poll::Ready(())
}
fn arm_quota_wait(&mut self, cx: &mut Context<'_>) -> Poll<()> {
Self::arm_wait(&mut self.quota_wait, false, false);
Self::poll_wait(&mut self.quota_wait, cx, None, RateDirection::Up)
}
}
#[derive(Debug)]
@@ -430,8 +437,13 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
if this.settle_c2s_rate_debt(cx).is_pending() {
return Poll::Pending;
}
if buf.remaining() == 0 {
return Pin::new(&mut this.inner).poll_read(cx, buf);
}
let mut remaining_before = None;
let mut reserved_read_bytes = 0u64;
let mut read_limit = buf.remaining();
if let Some(limit) = this.quota_limit {
let used_before = this.user_stats.quota_used();
let remaining = limit.saturating_sub(used_before);
@@ -440,50 +452,79 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
return Poll::Ready(Err(quota_io_error()));
}
remaining_before = Some(remaining);
read_limit = read_limit.min(remaining as usize);
if read_limit == 0 {
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
let desired = read_limit as u64;
let mut reserve_rounds = 0usize;
while reserved_read_bytes == 0 {
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match this.user_stats.quota_try_reserve(desired, limit) {
Ok(_) => {
reserved_read_bytes = desired;
break;
}
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
Err(crate::stats::QuotaReserveError::Contended) => {
this.stats.increment_quota_contention_total();
}
}
}
if reserved_read_bytes == 0 {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.stats.increment_quota_contention_timeout_total();
if this.arm_quota_wait(cx).is_pending() {
return Poll::Pending;
}
reserve_rounds = 0;
}
}
}
}
let before = buf.filled().len();
let limited_read = read_limit < buf.remaining();
let read_result = if limited_read {
let mut limited_buf = ReadBuf::new(buf.initialize_unfilled_to(read_limit));
match Pin::new(&mut this.inner).poll_read(cx, &mut limited_buf) {
Poll::Ready(Ok(())) => {
let n = limited_buf.filled().len();
buf.advance(n);
Poll::Ready(Ok(n))
}
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Pending => Poll::Pending,
}
} else {
let before = buf.filled().len();
match Pin::new(&mut this.inner).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
let n = buf.filled().len() - before;
Poll::Ready(Ok(n))
}
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Pending => Poll::Pending,
}
};
match Pin::new(&mut this.inner).poll_read(cx, buf) {
Poll::Ready(Ok(())) => {
let n = buf.filled().len() - before;
match read_result {
Poll::Ready(Ok(n)) => {
if reserved_read_bytes > n as u64 {
let refund_bytes = reserved_read_bytes - n as u64;
refund_reserved_quota_bytes(this.user_stats.as_ref(), refund_bytes);
this.stats.add_quota_refund_bytes_total(refund_bytes);
}
if n > 0 {
let n_to_charge = n as u64;
if let (Some(limit), Some(remaining)) = (this.quota_limit, remaining_before) {
let mut reserved_total = None;
let mut reserve_rounds = 0usize;
while reserved_total.is_none() {
let mut saw_contention = false;
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
match this.user_stats.quota_try_reserve(n_to_charge, limit) {
Ok(total) => {
reserved_total = Some(total);
break;
}
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
this.quota_exceeded.store(true, Ordering::Release);
buf.set_filled(before);
return Poll::Ready(Err(quota_io_error()));
}
Err(crate::stats::QuotaReserveError::Contended) => {
saw_contention = true;
}
}
}
if reserved_total.is_none() {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.quota_exceeded.store(true, Ordering::Release);
buf.set_filled(before);
return Poll::Ready(Err(quota_io_error()));
}
if saw_contention {
std::thread::yield_now();
}
}
}
if let Some(remaining) = remaining_before {
if should_immediate_quota_check(remaining, n_to_charge) {
this.quota_bytes_since_check = 0;
} else {
@@ -494,10 +535,11 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
this.quota_bytes_since_check = 0;
}
}
if reserved_total.unwrap_or(0) >= limit {
this.quota_exceeded.store(true, Ordering::Release);
}
}
if let Some(limit) = this.quota_limit
&& this.user_stats.quota_used() >= limit
{
this.quota_exceeded.store(true, Ordering::Release);
}
// C→S: client sent data
@@ -521,7 +563,20 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
}
Poll::Ready(Ok(()))
}
other => other,
Poll::Pending => {
if reserved_read_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_read_bytes);
this.stats.add_quota_refund_bytes_total(reserved_read_bytes);
}
Poll::Pending
}
Poll::Ready(Err(err)) => {
if reserved_read_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_read_bytes);
this.stats.add_quota_refund_bytes_total(reserved_read_bytes);
}
Poll::Ready(Err(err))
}
}
}
}
@@ -603,6 +658,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
break;
}
Err(crate::stats::QuotaReserveError::Contended) => {
this.stats.increment_quota_contention_total();
saw_contention = true;
}
}
@@ -611,14 +667,14 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
if reserved_bytes == 0 {
reserve_rounds = reserve_rounds.saturating_add(1);
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
this.stats.increment_quota_contention_timeout_total();
if let Some(lease) = this.traffic_lease.as_ref() {
lease.refund(RateDirection::Down, shaper_reserved_bytes);
}
this.quota_exceeded.store(true, Ordering::Release);
return Poll::Ready(Err(quota_io_error()));
}
if saw_contention {
std::thread::yield_now();
let _ = this.arm_quota_wait(cx);
return Poll::Pending;
} else if saw_contention {
std::hint::spin_loop();
}
}
}
@@ -639,10 +695,9 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
match Pin::new(&mut this.inner).poll_write(cx, write_buf) {
Poll::Ready(Ok(n)) => {
if reserved_bytes > n as u64 {
refund_reserved_quota_bytes(
this.user_stats.as_ref(),
reserved_bytes - n as u64,
);
let refund_bytes = reserved_bytes - n as u64;
refund_reserved_quota_bytes(this.user_stats.as_ref(), refund_bytes);
this.stats.add_quota_refund_bytes_total(refund_bytes);
}
if shaper_reserved_bytes > n as u64
&& let Some(lease) = this.traffic_lease.as_ref()
@@ -693,6 +748,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
Poll::Ready(Err(err)) => {
if reserved_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
this.stats.add_quota_refund_bytes_total(reserved_bytes);
}
if shaper_reserved_bytes > 0
&& let Some(lease) = this.traffic_lease.as_ref()
@@ -704,6 +760,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
Poll::Pending => {
if reserved_bytes > 0 {
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
this.stats.add_quota_refund_bytes_total(reserved_bytes);
}
if shaper_reserved_bytes > 0
&& let Some(lease) = this.traffic_lease.as_ref()

View File

@@ -4,8 +4,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::sync::watch;
pub(crate) const ROUTE_SWITCH_ERROR_MSG: &str = "Session terminated";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub(crate) enum RelayRouteMode {

View File

@@ -661,7 +661,7 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s
assert!(
matches!(relay_result, Err(ProxyError::DataQuotaExceeded { .. }))
|| matches!(relay_result, Err(ProxyError::Proxy(ref msg)) if msg == crate::proxy::route_mode::ROUTE_SWITCH_ERROR_MSG),
|| matches!(relay_result, Err(ProxyError::RouteSwitched)),
"overlap race must fail closed via quota enforcement or generic cutover termination"
);

View File

@@ -637,6 +637,22 @@ fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
"telemt-unknown-dc-parent-swap-{}",
std::process::id()
));
if let Ok(meta) = fs::symlink_metadata(&parent) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&parent).expect("stale parent-swap path must be removable");
} else {
fs::remove_dir_all(&parent).expect("stale parent-swap directory must be removable");
}
}
let moved = parent.with_extension("bak");
if let Ok(meta) = fs::symlink_metadata(&moved) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&moved).expect("stale parent-swap backup path must be removable");
} else {
fs::remove_dir_all(&moved)
.expect("stale parent-swap backup directory must be removable");
}
}
fs::create_dir_all(&parent).expect("parent-swap test parent must be creatable");
let rel_candidate = format!(
@@ -646,8 +662,6 @@ fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
.expect("candidate must sanitize before parent swap");
let moved = parent.with_extension("bak");
let _ = fs::remove_dir_all(&moved);
fs::rename(&parent, &moved).expect("parent must be movable for swap simulation");
symlink("/tmp", &parent).expect("symlink replacement for parent must be creatable");
@@ -669,6 +683,13 @@ fn adversarial_check_then_symlink_flip_is_blocked_by_nofollow_open() {
"telemt-unknown-dc-check-open-race-{}",
std::process::id()
));
if let Ok(meta) = fs::symlink_metadata(&parent) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&parent).expect("stale check-open-race path must be removable");
} else {
fs::remove_dir_all(&parent).expect("stale check-open-race parent must be removable");
}
}
fs::create_dir_all(&parent).expect("check-open-race parent must be creatable");
let target = parent.join("unknown-dc.log");
@@ -713,6 +734,24 @@ fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
"telemt-unknown-dc-parent-swap-openat-{}",
std::process::id()
));
if let Ok(meta) = fs::symlink_metadata(&base) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&base).expect("stale parent-swap-openat path must be removable");
} else {
fs::remove_dir_all(&base)
.expect("stale parent-swap-openat directory must be removable");
}
}
let moved = base.with_extension("bak");
if let Ok(meta) = fs::symlink_metadata(&moved) {
if meta.file_type().is_symlink() || meta.is_file() {
fs::remove_file(&moved)
.expect("stale parent-swap-openat backup path must be removable");
} else {
fs::remove_dir_all(&moved)
.expect("stale parent-swap-openat backup directory must be removable");
}
}
fs::create_dir_all(&base).expect("parent-swap-openat base must be creatable");
let rel_candidate = format!(
@@ -736,8 +775,6 @@ fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
let outside_target = outside_parent.join("unknown-dc.log");
let _ = fs::remove_file(&outside_target);
let moved = base.with_extension("bak");
let _ = fs::remove_dir_all(&moved);
fs::rename(&base, &moved).expect("base parent must be movable for swap simulation");
symlink(&outside_parent, &base).expect("base parent symlink replacement must be creatable");
@@ -1482,10 +1519,7 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() {
"cutover should terminate direct relay session"
);
assert!(
matches!(
relay_result,
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
),
matches!(relay_result, Err(ProxyError::RouteSwitched)),
"client-visible cutover error must stay generic and avoid route-internal metadata"
);
@@ -1622,10 +1656,7 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea
.expect("direct relay task must not panic");
assert!(
matches!(
relay_result,
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
),
matches!(relay_result, Err(ProxyError::RouteSwitched)),
"storm-cutover termination must remain generic for all direct sessions"
);
}
@@ -1928,10 +1959,7 @@ async fn adversarial_direct_relay_cutover_integrity() {
.expect("Session must not panic");
assert!(
matches!(
result,
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
),
matches!(result, Err(ProxyError::RouteSwitched)),
"Session must terminate with route switch error on cutover"
);
}

View File

@@ -13,6 +13,8 @@ struct CountedWriter {
fail_writes: bool,
}
struct StalledWriter;
impl CountedWriter {
fn new(write_calls: Arc<AtomicUsize>, fail_writes: bool) -> Self {
Self {
@@ -49,12 +51,36 @@ impl AsyncWrite for CountedWriter {
}
}
impl AsyncWrite for StalledWriter {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
_buf: &[u8],
) -> Poll<io::Result<usize>> {
Poll::Pending
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Pending
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Pending
}
}
fn make_crypto_writer(inner: CountedWriter) -> CryptoWriter<CountedWriter> {
let key = [0u8; 32];
let iv = 0u128;
CryptoWriter::new(inner, AesCtr::new(&key, iv), 8 * 1024)
}
fn make_stalled_crypto_writer() -> CryptoWriter<StalledWriter> {
let key = [0u8; 32];
let iv = 0u128;
CryptoWriter::new(StalledWriter, AesCtr::new(&key, iv), 8 * 1024)
}
#[tokio::test]
async fn me_writer_write_fail_keeps_reserved_quota_and_tracks_fail_metrics() {
let stats = Stats::new();
@@ -189,3 +215,53 @@ async fn me_writer_pre_write_quota_reject_happens_before_writer_poll() {
);
assert_eq!(bytes_me2c.load(Ordering::Relaxed), 0);
}
#[tokio::test]
async fn me_writer_data_write_obeys_flow_cancellation() {
let stats = Stats::new();
let user = "middle-me-writer-cancel-user";
let mut writer = make_stalled_crypto_writer();
let mut frame_buf = Vec::new();
let bytes_me2c = AtomicU64::new(0);
let cancel = CancellationToken::new();
cancel.cancel();
let result = process_me_writer_response_with_traffic_lease(
MeResponse::Data {
flags: 0,
data: Bytes::from_static(&[0x31, 0x32, 0x33, 0x34]),
route_permit: None,
},
&mut writer,
ProtoTag::Intermediate,
&SecureRandom::new(),
&mut frame_buf,
&stats,
user,
None,
None,
0,
None,
&cancel,
&bytes_me2c,
13,
true,
false,
)
.await;
assert!(
matches!(result, Err(ProxyError::MiddleClientWriterCancelled)),
"cancelled middle writer must return a bounded cancellation error"
);
assert_eq!(
bytes_me2c.load(Ordering::Relaxed),
0,
"cancelled write must not advance committed ME->C bytes"
);
assert_eq!(
stats.get_user_total_octets(user),
0,
"cancelled write must not advance user output telemetry"
);
}

View File

@@ -4,10 +4,67 @@ use std::io;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use tokio::io::{AsyncWrite, AsyncWriteExt};
use std::task::{Context, Poll, Wake};
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
use tokio::time::Instant;
enum ReadStep {
Data(Vec<u8>),
Pending,
Eof,
Error,
}
struct ScriptedReader {
scripted_reads: Arc<Mutex<VecDeque<ReadStep>>>,
read_calls: Arc<AtomicUsize>,
}
impl ScriptedReader {
fn new(script: Vec<ReadStep>, read_calls: Arc<AtomicUsize>) -> Self {
Self {
scripted_reads: Arc::new(Mutex::new(script.into())),
read_calls,
}
}
}
impl AsyncRead for ScriptedReader {
fn poll_read(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
let this = self.get_mut();
this.read_calls.fetch_add(1, Ordering::Relaxed);
let step = this
.scripted_reads
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.pop_front()
.unwrap_or(ReadStep::Eof);
match step {
ReadStep::Data(data) => {
let n = data.len().min(buf.remaining());
buf.put_slice(&data[..n]);
Poll::Ready(Ok(()))
}
ReadStep::Pending => Poll::Pending,
ReadStep::Eof => Poll::Ready(Ok(())),
ReadStep::Error => Poll::Ready(Err(io::Error::new(
io::ErrorKind::BrokenPipe,
"forced read failure",
))),
}
}
}
struct NoopWake;
impl Wake for NoopWake {
fn wake(self: Arc<Self>) {}
}
struct ScriptedWriter {
scripted_writes: Arc<Mutex<VecDeque<usize>>>,
write_calls: Arc<AtomicUsize>,
@@ -80,6 +137,127 @@ fn make_stats_io_with_script(
(io, stats, write_calls, quota_exceeded)
}
fn make_stats_io_with_read_script(
user: &str,
quota_limit: u64,
precharged_quota: u64,
script: Vec<ReadStep>,
) -> (
StatsIo<ScriptedReader>,
Arc<Stats>,
Arc<AtomicUsize>,
Arc<AtomicBool>,
) {
let stats = Arc::new(Stats::new());
if precharged_quota > 0 {
let user_stats = stats.get_or_create_user_stats_handle(user);
stats.quota_charge_post_write(user_stats.as_ref(), precharged_quota);
}
let read_calls = Arc::new(AtomicUsize::new(0));
let quota_exceeded = Arc::new(AtomicBool::new(false));
let io = StatsIo::new(
ScriptedReader::new(script, read_calls.clone()),
Arc::new(SharedCounters::new()),
stats.clone(),
user.to_string(),
Some(quota_limit),
quota_exceeded.clone(),
Instant::now(),
);
(io, stats, read_calls, quota_exceeded)
}
fn poll_read_once<R: AsyncRead + Unpin>(
io: &mut StatsIo<R>,
storage: &mut [u8],
) -> Poll<io::Result<usize>> {
let waker = Arc::new(NoopWake).into();
let mut cx = Context::from_waker(&waker);
let mut read_buf = ReadBuf::new(storage);
let before = read_buf.filled().len();
match Pin::new(io).poll_read(&mut cx, &mut read_buf) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(read_buf.filled().len() - before)),
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
Poll::Pending => Poll::Pending,
}
}
#[test]
fn direct_c2s_quota_refunds_unused_on_short_read() {
let user = "direct-c2s-short-read-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Data(vec![0x11; 5])]);
let mut storage = [0u8; 16];
let n = match poll_read_once(&mut io, &mut storage) {
Poll::Ready(Ok(n)) => n,
other => panic!("short read must complete, got {other:?}"),
};
assert_eq!(n, 5);
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 5);
assert_eq!(stats.get_quota_refund_bytes_total(), 11);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[test]
fn direct_c2s_quota_refunds_full_reservation_on_pending() {
let user = "direct-c2s-pending-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Pending]);
let mut storage = [0u8; 16];
assert!(matches!(
poll_read_once(&mut io, &mut storage),
Poll::Pending
));
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 0);
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[test]
fn direct_c2s_quota_refunds_full_reservation_on_eof() {
let user = "direct-c2s-eof-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Eof]);
let mut storage = [0u8; 16];
let n = match poll_read_once(&mut io, &mut storage) {
Poll::Ready(Ok(n)) => n,
other => panic!("EOF read must complete with zero bytes, got {other:?}"),
};
assert_eq!(n, 0);
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 0);
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[test]
fn direct_c2s_quota_refunds_full_reservation_on_error() {
let user = "direct-c2s-error-refund-user";
let (mut io, stats, read_calls, quota_exceeded) =
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Error]);
let mut storage = [0u8; 16];
let error = match poll_read_once(&mut io, &mut storage) {
Poll::Ready(Err(error)) => error,
other => panic!("error read must return error, got {other:?}"),
};
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
assert_eq!(stats.get_user_quota_used(user), 0);
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
assert!(!quota_exceeded.load(Ordering::Acquire));
}
#[tokio::test]
async fn direct_partial_write_charges_only_committed_bytes_without_double_charge() {
let user = "direct-partial-charge-user";

114
src/quota_state.rs Normal file
View File

@@ -0,0 +1,114 @@
use std::collections::BTreeMap;
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tracing::{info, warn};
use crate::stats::{Stats, UserQuotaSnapshot};
#[derive(Debug, Default, Serialize, Deserialize)]
pub(crate) struct QuotaStateFile {
pub(crate) last_reset_epoch_secs: u64,
pub(crate) users: BTreeMap<String, QuotaUserState>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub(crate) struct QuotaUserState {
pub(crate) used_bytes: u64,
pub(crate) last_reset_epoch_secs: u64,
}
fn now_epoch_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
pub(crate) async fn load_quota_state(path: &Path, stats: &Stats) {
let bytes = match tokio::fs::read(path).await {
Ok(bytes) => bytes,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return,
Err(error) => {
warn!(
error = %error,
path = %path.display(),
"Failed to read quota state file"
);
return;
}
};
let state = match serde_json::from_slice::<QuotaStateFile>(&bytes) {
Ok(state) => state,
Err(error) => {
warn!(
error = %error,
path = %path.display(),
"Failed to parse quota state file"
);
return;
}
};
let loaded_users = state.users.len();
for (user, quota) in state.users {
stats.load_user_quota_state(&user, quota.used_bytes, quota.last_reset_epoch_secs);
}
info!(
path = %path.display(),
loaded_users,
"Loaded per-user quota state"
);
}
pub(crate) async fn save_quota_state(path: &Path, stats: &Stats) -> std::io::Result<()> {
let mut users = BTreeMap::new();
let mut last_reset_epoch_secs = 0;
for (user, quota) in stats.user_quota_snapshot() {
last_reset_epoch_secs = last_reset_epoch_secs.max(quota.last_reset_epoch_secs);
users.insert(user, quota_user_state(quota));
}
let state = QuotaStateFile {
last_reset_epoch_secs,
users,
};
write_state_file(path, &state).await
}
pub(crate) async fn reset_user_quota(
path: &Path,
stats: &Stats,
user: &str,
) -> std::io::Result<UserQuotaSnapshot> {
let snapshot = stats.reset_user_quota(user);
save_quota_state(path, stats).await?;
Ok(snapshot)
}
async fn write_state_file(path: &Path, state: &QuotaStateFile) -> std::io::Result<()> {
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
tokio::fs::create_dir_all(parent).await?;
}
let tmp_path = path.with_extension(format!("tmp.{}", now_epoch_secs()));
let payload = serde_json::to_vec_pretty(state)?;
let mut file = tokio::fs::File::create(&tmp_path).await?;
file.write_all(&payload).await?;
file.write_all(b"\n").await?;
file.sync_all().await?;
drop(file);
tokio::fs::rename(&tmp_path, path).await
}
fn quota_user_state(quota: UserQuotaSnapshot) -> QuotaUserState {
QuotaUserState {
used_bytes: quota.used_bytes,
last_reset_epoch_secs: quota.last_reset_epoch_secs,
}
}

View File

@@ -74,16 +74,21 @@ impl BeobachtenStore {
}
let now = Instant::now();
let mut guard = self.inner.lock();
Self::cleanup(&mut guard, now, ttl);
guard.last_cleanup = Some(now);
let entries = {
let mut guard = self.inner.lock();
Self::cleanup(&mut guard, now, ttl);
guard.last_cleanup = Some(now);
guard
.entries
.iter()
.map(|((class, ip), entry)| (class.clone(), *ip, entry.tries))
.collect::<Vec<_>>()
};
let mut grouped = BTreeMap::<String, Vec<(IpAddr, u64)>>::new();
for ((class, ip), entry) in &guard.entries {
grouped
.entry(class.clone())
.or_default()
.push((*ip, entry.tries));
for (class, ip, tries) in entries {
grouped.entry(class).or_default().push((ip, tries));
}
if grouped.is_empty() {

View File

@@ -8,8 +8,8 @@ pub mod telemetry;
use dashmap::DashMap;
use lru::LruCache;
use parking_lot::Mutex;
use std::collections::VecDeque;
use std::collections::hash_map::DefaultHasher;
use std::collections::{HashMap, VecDeque};
use std::hash::{Hash, Hasher};
use std::num::NonZeroUsize;
use std::sync::Arc;
@@ -274,11 +274,22 @@ pub struct Stats {
me_inline_recovery_total: AtomicU64,
ip_reservation_rollback_tcp_limit_total: AtomicU64,
ip_reservation_rollback_quota_limit_total: AtomicU64,
quota_refund_bytes_total: AtomicU64,
quota_contention_total: AtomicU64,
quota_contention_timeout_total: AtomicU64,
quota_acquire_cancelled_total: AtomicU64,
quota_write_fail_bytes_total: AtomicU64,
quota_write_fail_events_total: AtomicU64,
me_child_join_timeout_total: AtomicU64,
me_child_abort_total: AtomicU64,
flow_wait_middle_rate_limit_total: AtomicU64,
flow_wait_middle_rate_limit_cancelled_total: AtomicU64,
flow_wait_middle_rate_limit_ms_total: AtomicU64,
session_drop_fallback_total: AtomicU64,
telemetry_core_enabled: AtomicBool,
telemetry_user_enabled: AtomicBool,
telemetry_me_level: AtomicU8,
cached_epoch_secs: AtomicU64,
user_stats: DashMap<String, Arc<UserStats>>,
user_stats_last_cleanup_epoch_secs: AtomicU64,
start_time: parking_lot::RwLock<Option<Instant>>,
@@ -297,9 +308,16 @@ pub struct UserStats {
/// This counter is the single source of truth for quota enforcement and
/// intentionally tracks attempted traffic, not guaranteed delivery.
pub quota_used: AtomicU64,
pub quota_last_reset_epoch_secs: AtomicU64,
pub last_seen_epoch_secs: AtomicU64,
}
#[derive(Debug, Clone)]
pub struct UserQuotaSnapshot {
pub used_bytes: u64,
pub last_reset_epoch_secs: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuotaReserveError {
LimitExceeded,
@@ -341,6 +359,7 @@ impl Stats {
pub fn new() -> Self {
let stats = Self::default();
stats.apply_telemetry_policy(TelemetryPolicy::default());
stats.refresh_cached_epoch_secs();
*stats.start_time.write() = Some(Instant::now());
stats
}
@@ -390,33 +409,55 @@ impl Stats {
.as_secs()
}
fn touch_user_stats(stats: &UserStats) {
fn refresh_cached_epoch_secs(&self) -> u64 {
let now_epoch_secs = Self::now_epoch_secs();
self.cached_epoch_secs
.store(now_epoch_secs, Ordering::Relaxed);
now_epoch_secs
}
fn cached_epoch_secs(&self) -> u64 {
let cached = self.cached_epoch_secs.load(Ordering::Relaxed);
if cached != 0 {
return cached;
}
self.refresh_cached_epoch_secs()
}
fn touch_user_stats(&self, stats: &UserStats) {
stats
.last_seen_epoch_secs
.store(Self::now_epoch_secs(), Ordering::Relaxed);
.store(self.cached_epoch_secs(), Ordering::Relaxed);
}
pub(crate) fn get_or_create_user_stats_handle(&self, user: &str) -> Arc<UserStats> {
self.maybe_cleanup_user_stats();
if let Some(existing) = self.user_stats.get(user) {
let handle = Arc::clone(existing.value());
Self::touch_user_stats(handle.as_ref());
self.touch_user_stats(handle.as_ref());
return handle;
}
let entry = self.user_stats.entry(user.to_string()).or_default();
if entry.last_seen_epoch_secs.load(Ordering::Relaxed) == 0 {
Self::touch_user_stats(entry.value().as_ref());
self.touch_user_stats(entry.value().as_ref());
}
Arc::clone(entry.value())
}
pub(crate) async fn run_periodic_user_stats_maintenance(self: Arc<Self>) {
let mut interval = tokio::time::interval(Duration::from_secs(60));
loop {
interval.tick().await;
self.maybe_cleanup_user_stats();
}
}
#[inline]
pub(crate) fn add_user_octets_from_handle(&self, user_stats: &UserStats, bytes: u64) {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats
.octets_from_client
.fetch_add(bytes, Ordering::Relaxed);
@@ -427,7 +468,7 @@ impl Stats {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats
.octets_to_client
.fetch_add(bytes, Ordering::Relaxed);
@@ -438,7 +479,7 @@ impl Stats {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
}
@@ -447,7 +488,7 @@ impl Stats {
if !self.telemetry_user_enabled() {
return;
}
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
}
@@ -457,7 +498,7 @@ impl Stats {
/// mixing reserve and post-charge on a single I/O event.
#[inline]
pub(crate) fn quota_charge_post_write(&self, user_stats: &UserStats, bytes: u64) -> u64 {
Self::touch_user_stats(user_stats);
self.touch_user_stats(user_stats);
user_stats
.quota_used
.fetch_add(bytes, Ordering::Relaxed)
@@ -468,7 +509,7 @@ impl Stats {
const USER_STATS_CLEANUP_INTERVAL_SECS: u64 = 60;
const USER_STATS_IDLE_TTL_SECS: u64 = 24 * 60 * 60;
let now_epoch_secs = Self::now_epoch_secs();
let now_epoch_secs = self.refresh_cached_epoch_secs();
let last_cleanup_epoch_secs = self
.user_stats_last_cleanup_epoch_secs
.load(Ordering::Relaxed);
@@ -1430,6 +1471,29 @@ impl Stats {
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn add_quota_refund_bytes_total(&self, bytes: u64) {
if self.telemetry_core_enabled() {
self.quota_refund_bytes_total
.fetch_add(bytes, Ordering::Relaxed);
}
}
pub fn increment_quota_contention_total(&self) {
if self.telemetry_core_enabled() {
self.quota_contention_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_quota_contention_timeout_total(&self) {
if self.telemetry_core_enabled() {
self.quota_contention_timeout_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_quota_acquire_cancelled_total(&self) {
if self.telemetry_core_enabled() {
self.quota_acquire_cancelled_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn add_quota_write_fail_bytes_total(&self, bytes: u64) {
if self.telemetry_core_enabled() {
self.quota_write_fail_bytes_total
@@ -1442,6 +1506,37 @@ impl Stats {
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_child_join_timeout_total(&self) {
if self.telemetry_core_enabled() {
self.me_child_join_timeout_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_child_abort_total(&self) {
if self.telemetry_core_enabled() {
self.me_child_abort_total.fetch_add(1, Ordering::Relaxed);
}
}
pub fn observe_flow_wait_middle_rate_limit_ms(&self, wait_ms: u64) {
if self.telemetry_core_enabled() {
self.flow_wait_middle_rate_limit_total
.fetch_add(1, Ordering::Relaxed);
self.flow_wait_middle_rate_limit_ms_total
.fetch_add(wait_ms, Ordering::Relaxed);
}
}
pub fn increment_flow_wait_middle_rate_limit_cancelled_total(&self) {
if self.telemetry_core_enabled() {
self.flow_wait_middle_rate_limit_cancelled_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_session_drop_fallback_total(&self) {
if self.telemetry_core_enabled() {
self.session_drop_fallback_total
.fetch_add(1, Ordering::Relaxed);
}
}
pub fn increment_me_endpoint_quarantine_total(&self) {
if self.telemetry_me_allows_normal() {
self.me_endpoint_quarantine_total
@@ -2276,19 +2371,52 @@ impl Stats {
self.ip_reservation_rollback_quota_limit_total
.load(Ordering::Relaxed)
}
pub fn get_quota_refund_bytes_total(&self) -> u64 {
self.quota_refund_bytes_total.load(Ordering::Relaxed)
}
pub fn get_quota_contention_total(&self) -> u64 {
self.quota_contention_total.load(Ordering::Relaxed)
}
pub fn get_quota_contention_timeout_total(&self) -> u64 {
self.quota_contention_timeout_total.load(Ordering::Relaxed)
}
pub fn get_quota_acquire_cancelled_total(&self) -> u64 {
self.quota_acquire_cancelled_total.load(Ordering::Relaxed)
}
pub fn get_quota_write_fail_bytes_total(&self) -> u64 {
self.quota_write_fail_bytes_total.load(Ordering::Relaxed)
}
pub fn get_quota_write_fail_events_total(&self) -> u64 {
self.quota_write_fail_events_total.load(Ordering::Relaxed)
}
pub fn get_me_child_join_timeout_total(&self) -> u64 {
self.me_child_join_timeout_total.load(Ordering::Relaxed)
}
pub fn get_me_child_abort_total(&self) -> u64 {
self.me_child_abort_total.load(Ordering::Relaxed)
}
pub fn get_flow_wait_middle_rate_limit_total(&self) -> u64 {
self.flow_wait_middle_rate_limit_total
.load(Ordering::Relaxed)
}
pub fn get_flow_wait_middle_rate_limit_cancelled_total(&self) -> u64 {
self.flow_wait_middle_rate_limit_cancelled_total
.load(Ordering::Relaxed)
}
pub fn get_flow_wait_middle_rate_limit_ms_total(&self) -> u64 {
self.flow_wait_middle_rate_limit_ms_total
.load(Ordering::Relaxed)
}
pub fn get_session_drop_fallback_total(&self) -> u64 {
self.session_drop_fallback_total.load(Ordering::Relaxed)
}
pub fn increment_user_connects(&self, user: &str) {
if !self.telemetry_user_enabled() {
return;
}
let stats = self.get_or_create_user_stats_handle(user);
Self::touch_user_stats(stats.as_ref());
self.touch_user_stats(stats.as_ref());
stats.connects.fetch_add(1, Ordering::Relaxed);
}
@@ -2297,7 +2425,7 @@ impl Stats {
return;
}
let stats = self.get_or_create_user_stats_handle(user);
Self::touch_user_stats(stats.as_ref());
self.touch_user_stats(stats.as_ref());
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
}
@@ -2307,7 +2435,7 @@ impl Stats {
}
let stats = self.get_or_create_user_stats_handle(user);
Self::touch_user_stats(stats.as_ref());
self.touch_user_stats(stats.as_ref());
let counter = &stats.curr_connects;
let mut current = counter.load(Ordering::Relaxed);
@@ -2330,9 +2458,8 @@ impl Stats {
}
pub fn decrement_user_curr_connects(&self, user: &str) {
self.maybe_cleanup_user_stats();
if let Some(stats) = self.user_stats.get(user) {
Self::touch_user_stats(stats.value().as_ref());
self.touch_user_stats(stats.value().as_ref());
let counter = &stats.curr_connects;
let mut current = counter.load(Ordering::Relaxed);
loop {
@@ -2408,6 +2535,47 @@ impl Stats {
.unwrap_or(0)
}
pub fn load_user_quota_state(&self, user: &str, used_bytes: u64, last_reset_epoch_secs: u64) {
let stats = self.get_or_create_user_stats_handle(user);
stats.quota_used.store(used_bytes, Ordering::Relaxed);
stats
.quota_last_reset_epoch_secs
.store(last_reset_epoch_secs, Ordering::Relaxed);
}
pub fn reset_user_quota(&self, user: &str) -> UserQuotaSnapshot {
let stats = self.get_or_create_user_stats_handle(user);
let last_reset_epoch_secs = Self::now_epoch_secs();
stats.quota_used.store(0, Ordering::Relaxed);
stats
.quota_last_reset_epoch_secs
.store(last_reset_epoch_secs, Ordering::Relaxed);
UserQuotaSnapshot {
used_bytes: 0,
last_reset_epoch_secs,
}
}
pub fn user_quota_snapshot(&self) -> HashMap<String, UserQuotaSnapshot> {
let mut out = HashMap::new();
for entry in self.user_stats.iter() {
let stats = entry.value();
let used_bytes = stats.quota_used.load(Ordering::Relaxed);
let last_reset_epoch_secs = stats.quota_last_reset_epoch_secs.load(Ordering::Relaxed);
if used_bytes == 0 && last_reset_epoch_secs == 0 {
continue;
}
out.insert(
entry.key().clone(),
UserQuotaSnapshot {
used_bytes,
last_reset_epoch_secs,
},
);
}
out
}
pub fn get_handshake_timeouts(&self) -> u64 {
self.handshake_timeouts.load(Ordering::Relaxed)
}
@@ -2477,6 +2645,11 @@ impl Stats {
self.user_stats.iter()
}
/// Current number of retained per-user stats entries.
pub fn user_stats_len(&self) -> usize {
self.user_stats.len()
}
pub fn uptime_secs(&self) -> f64 {
self.start_time
.read()

View File

@@ -277,6 +277,7 @@ impl StreamState for TlsReaderState {
pub struct FakeTlsReader<R> {
upstream: R,
state: TlsReaderState,
body_scratch: Vec<u8>,
}
impl<R> FakeTlsReader<R> {
@@ -284,6 +285,7 @@ impl<R> FakeTlsReader<R> {
Self {
upstream,
state: TlsReaderState::Idle,
body_scratch: Vec::new(),
}
}
@@ -439,7 +441,13 @@ impl<R: AsyncRead + Unpin> AsyncRead for FakeTlsReader<R> {
length,
mut buffer,
} => {
let result = poll_read_body(&mut this.upstream, cx, &mut buffer, length);
let result = poll_read_body(
&mut this.upstream,
cx,
&mut buffer,
length,
&mut this.body_scratch,
);
match result {
BodyPollResult::Pending => {
@@ -558,34 +566,36 @@ fn poll_read_body<R: AsyncRead + Unpin>(
cx: &mut Context<'_>,
buffer: &mut BytesMut,
target_len: usize,
scratch: &mut Vec<u8>,
) -> BodyPollResult {
// NOTE: This implementation uses a temporary Vec to avoid tricky borrow/lifetime
// issues with BytesMut spare capacity and ReadBuf across polls.
// It's safe and correct; optimization is possible if needed.
while buffer.len() < target_len {
let remaining = target_len - buffer.len();
let chunk_len = remaining.min(8192);
let mut temp = vec![0u8; remaining.min(8192)];
let mut read_buf = ReadBuf::new(&mut temp);
match Pin::new(&mut *upstream).poll_read(cx, &mut read_buf) {
Poll::Pending => return BodyPollResult::Pending,
Poll::Ready(Err(e)) => return BodyPollResult::Error(e),
Poll::Ready(Ok(())) => {
let n = read_buf.filled().len();
if n == 0 {
return BodyPollResult::Error(Error::new(
ErrorKind::UnexpectedEof,
format!(
"unexpected EOF in TLS body (got {} of {} bytes)",
buffer.len(),
target_len
),
));
}
buffer.extend_from_slice(&temp[..n]);
}
if scratch.len() < chunk_len {
scratch.resize(chunk_len, 0);
}
let n = {
let mut read_buf = ReadBuf::new(&mut scratch[..chunk_len]);
match Pin::new(&mut *upstream).poll_read(cx, &mut read_buf) {
Poll::Pending => return BodyPollResult::Pending,
Poll::Ready(Err(e)) => return BodyPollResult::Error(e),
Poll::Ready(Ok(())) => read_buf.filled().len(),
}
};
if n == 0 {
return BodyPollResult::Error(Error::new(
ErrorKind::UnexpectedEof,
format!(
"unexpected EOF in TLS body (got {} of {} bytes)",
buffer.len(),
target_len
),
));
}
buffer.extend_from_slice(&scratch[..n]);
}
BodyPollResult::Complete(buffer.split().freeze())

View File

@@ -559,9 +559,7 @@ async fn mass_reconnect_sync_cleanup_prevents_temporary_reservation_bloat() {
}
#[tokio::test]
async fn adversarial_drain_cleanup_queue_race_does_not_cause_false_rejections() {
// Regression guard: concurrent cleanup draining must not produce false
// limit denials for a new IP when the previous IP is already queued.
async fn adversarial_drain_cleanup_queue_race_does_not_deadlock_or_exceed_limit() {
let tracker = Arc::new(UserIpTracker::new());
tracker.set_user_limit("racer", 1).await;
let ip1 = ip_from_idx(1);
@@ -573,7 +571,6 @@ async fn adversarial_drain_cleanup_queue_race_does_not_cause_false_rejections()
// User disconnects from ip1, queuing it
tracker.enqueue_cleanup("racer".to_string(), ip1);
let mut saw_false_rejection = false;
for _ in 0..100 {
// Queue cleanup then race explicit drain and check-and-add on the alternative IP.
tracker.enqueue_cleanup("racer".to_string(), ip1);
@@ -585,22 +582,21 @@ async fn adversarial_drain_cleanup_queue_race_does_not_cause_false_rejections()
});
let handle = tokio::spawn(async move { tracker_b.check_and_add("racer", ip2).await });
drain_handle.await.unwrap();
let res = handle.await.unwrap();
if res.is_err() {
saw_false_rejection = true;
break;
}
tokio::time::timeout(Duration::from_secs(1), drain_handle)
.await
.expect("cleanup drain must not deadlock")
.unwrap();
let _ = tokio::time::timeout(Duration::from_secs(1), handle)
.await
.expect("admission must not deadlock")
.unwrap();
// Restore baseline for next iteration.
assert!(tracker.get_active_ip_count("racer").await <= 1);
tracker.drain_cleanup_queue().await;
tracker.remove_ip("racer", ip2).await;
tracker.remove_ip("racer", ip1).await;
tracker.check_and_add("racer", ip1).await.unwrap();
}
assert!(
!saw_false_rejection,
"Concurrent cleanup draining must not cause false-positive IP denials"
);
}
#[tokio::test]

View File

@@ -1,26 +1,71 @@
use std::collections::HashMap;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::net::IpAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::sync::RwLock;
use tokio::time::sleep;
use tracing::{debug, info, warn};
use crate::tls_front::types::{
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult,
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, TlsProfileSource,
};
const FULL_CERT_SENT_SWEEP_INTERVAL_SECS: u64 = 30;
const FULL_CERT_SENT_MAX_IPS: usize = 65_536;
const FULL_CERT_SENT_SHARDS: usize = 64;
static FULL_CERT_SENT_IPS_GAUGE: AtomicU64 = AtomicU64::new(0);
static FULL_CERT_SENT_CAP_DROPS: AtomicU64 = AtomicU64::new(0);
/// Current number of IPs tracked by the TLS full-cert budget gate.
pub(crate) fn full_cert_sent_ips_for_metrics() -> u64 {
FULL_CERT_SENT_IPS_GAUGE.load(Ordering::Relaxed)
}
/// Number of new IPs denied a full-cert budget slot because the cap was reached.
pub(crate) fn full_cert_sent_cap_drops_for_metrics() -> u64 {
FULL_CERT_SENT_CAP_DROPS.load(Ordering::Relaxed)
}
/// Lightweight in-memory + optional on-disk cache for TLS fronting data.
#[derive(Debug)]
pub struct TlsFrontCache {
memory: RwLock<HashMap<String, Arc<CachedTlsData>>>,
default: Arc<CachedTlsData>,
full_cert_sent: RwLock<HashMap<IpAddr, Instant>>,
full_cert_sent_shards: Vec<RwLock<HashMap<IpAddr, Instant>>>,
full_cert_sent_last_sweep_epoch_secs: AtomicU64,
disk_path: PathBuf,
}
/// Read-only health view for one configured TLS front domain.
#[derive(Debug, Clone)]
pub(crate) struct TlsFrontProfileHealth {
pub(crate) domain: String,
pub(crate) source: &'static str,
pub(crate) age_seconds: u64,
pub(crate) is_default: bool,
pub(crate) has_cert_info: bool,
pub(crate) has_cert_payload: bool,
pub(crate) app_data_records: usize,
pub(crate) ticket_records: usize,
pub(crate) change_cipher_spec_count: u8,
pub(crate) total_app_data_len: usize,
}
fn profile_source_label(source: TlsProfileSource) -> &'static str {
match source {
TlsProfileSource::Default => "default",
TlsProfileSource::Raw => "raw",
TlsProfileSource::Rustls => "rustls",
TlsProfileSource::Merged => "merged",
}
}
#[allow(dead_code)]
impl TlsFrontCache {
pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef<Path>) -> Self {
@@ -52,7 +97,10 @@ impl TlsFrontCache {
Self {
memory: RwLock::new(map),
default,
full_cert_sent: RwLock::new(HashMap::new()),
full_cert_sent_shards: (0..FULL_CERT_SENT_SHARDS)
.map(|_| RwLock::new(HashMap::new()))
.collect(),
full_cert_sent_last_sweep_epoch_secs: AtomicU64::new(0),
disk_path: disk_path.as_ref().to_path_buf(),
}
}
@@ -69,22 +117,129 @@ impl TlsFrontCache {
self.memory.read().await.contains_key(domain)
}
pub(crate) async fn profile_health_snapshot(
&self,
domains: &[String],
max_domains: usize,
) -> (Vec<TlsFrontProfileHealth>, usize) {
let guard = self.memory.read().await;
let now = SystemTime::now();
let mut snapshot = Vec::with_capacity(domains.len().min(max_domains));
let mut suppressed = 0usize;
for domain in domains {
if snapshot.len() >= max_domains {
suppressed = suppressed.saturating_add(1);
continue;
}
let cached = guard
.get(domain)
.cloned()
.unwrap_or_else(|| self.default.clone());
let behavior = &cached.behavior_profile;
let age_seconds = now
.duration_since(cached.fetched_at)
.map(|duration| duration.as_secs())
.unwrap_or(0);
snapshot.push(TlsFrontProfileHealth {
domain: domain.clone(),
source: profile_source_label(behavior.source),
age_seconds,
is_default: cached.domain == "default",
has_cert_info: cached.cert_info.is_some(),
has_cert_payload: cached.cert_payload.is_some(),
app_data_records: cached
.app_data_records_sizes
.len()
.max(behavior.app_data_record_sizes.len()),
ticket_records: behavior.ticket_record_sizes.len(),
change_cipher_spec_count: behavior.change_cipher_spec_count,
total_app_data_len: cached.total_app_data_len,
});
}
(snapshot, suppressed)
}
fn full_cert_sent_shard_index(client_ip: IpAddr) -> usize {
let mut hasher = DefaultHasher::new();
client_ip.hash(&mut hasher);
(hasher.finish() as usize) % FULL_CERT_SENT_SHARDS
}
fn full_cert_sent_shard(&self, client_ip: IpAddr) -> &RwLock<HashMap<IpAddr, Instant>> {
&self.full_cert_sent_shards[Self::full_cert_sent_shard_index(client_ip)]
}
fn decrement_full_cert_sent_entries(amount: usize) {
if amount == 0 {
return;
}
let amount = amount as u64;
let _ =
FULL_CERT_SENT_IPS_GAUGE.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |current| {
Some(current.saturating_sub(amount))
});
}
fn try_reserve_full_cert_sent_entry() -> bool {
let mut current = FULL_CERT_SENT_IPS_GAUGE.load(Ordering::Relaxed);
loop {
if current >= FULL_CERT_SENT_MAX_IPS as u64 {
return false;
}
match FULL_CERT_SENT_IPS_GAUGE.compare_exchange_weak(
current,
current.saturating_add(1),
Ordering::AcqRel,
Ordering::Relaxed,
) {
Ok(_) => return true,
Err(actual) => current = actual,
}
}
}
async fn sweep_full_cert_sent_shards(&self, now: Instant, ttl: Duration) {
for shard in &self.full_cert_sent_shards {
let mut guard = shard.write().await;
let before = guard.len();
guard.retain(|_, seen_at| now.duration_since(*seen_at) < ttl);
Self::decrement_full_cert_sent_entries(before.saturating_sub(guard.len()));
}
}
/// Returns true when full cert payload should be sent for client_ip
/// according to TTL policy.
pub async fn take_full_cert_budget_for_ip(&self, client_ip: IpAddr, ttl: Duration) -> bool {
if ttl.is_zero() {
self.full_cert_sent
.write()
.await
.insert(client_ip, Instant::now());
return true;
}
let now = Instant::now();
let mut guard = self.full_cert_sent.write().await;
guard.retain(|_, seen_at| now.duration_since(*seen_at) < ttl);
let now_epoch_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let should_sweep = self
.full_cert_sent_last_sweep_epoch_secs
.fetch_update(Ordering::AcqRel, Ordering::Relaxed, |last_sweep| {
if now_epoch_secs.saturating_sub(last_sweep) >= FULL_CERT_SENT_SWEEP_INTERVAL_SECS {
Some(now_epoch_secs)
} else {
None
}
})
.is_ok();
match guard.get_mut(&client_ip) {
if should_sweep {
self.sweep_full_cert_sent_shards(now, ttl).await;
}
let mut guard = self.full_cert_sent_shard(client_ip).write().await;
let allowed = match guard.get_mut(&client_ip) {
Some(seen_at) => {
if now.duration_since(*seen_at) >= ttl {
*seen_at = now;
@@ -94,12 +249,43 @@ impl TlsFrontCache {
}
}
None => {
if !Self::try_reserve_full_cert_sent_entry() {
FULL_CERT_SENT_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return false;
}
guard.insert(client_ip, now);
true
}
};
allowed
}
#[cfg(test)]
async fn insert_full_cert_sent_for_tests(&self, client_ip: IpAddr, seen_at: Instant) {
let mut guard = self.full_cert_sent_shard(client_ip).write().await;
if guard.insert(client_ip, seen_at).is_none() {
FULL_CERT_SENT_IPS_GAUGE.fetch_add(1, Ordering::Relaxed);
}
}
#[cfg(test)]
async fn full_cert_sent_is_empty_for_tests(&self) -> bool {
for shard in &self.full_cert_sent_shards {
if !shard.read().await.is_empty() {
return false;
}
}
true
}
#[cfg(test)]
async fn full_cert_sent_contains_for_tests(&self, client_ip: IpAddr) -> bool {
self.full_cert_sent_shard(client_ip)
.read()
.await
.contains_key(&client_ip)
}
pub async fn set(&self, domain: &str, data: CachedTlsData) {
let mut guard = self.memory.write().await;
guard.insert(domain.to_string(), Arc::new(data));
@@ -130,6 +316,14 @@ impl TlsFrontCache {
warn!(file = %name, "Skipping TLS cache entry with invalid domain");
continue;
}
if !cert_info_matches_domain(&cached) {
warn!(
file = %name,
domain = %cached.domain,
"Skipping TLS cache entry with mismatched certificate metadata"
);
continue;
}
// fetched_at is skipped during deserialization; approximate with file mtime if available.
if let Ok(meta) = entry.metadata().await
&& let Ok(modified) = meta.modified()
@@ -209,10 +403,100 @@ impl TlsFrontCache {
}
}
fn cert_info_matches_domain(cached: &CachedTlsData) -> bool {
let Some(cert_info) = cached.cert_info.as_ref() else {
return true;
};
if !cert_info.san_names.is_empty() {
return cert_info
.san_names
.iter()
.any(|name| dns_name_matches_domain(name, &cached.domain));
}
cert_info
.subject_cn
.as_deref()
.map_or(true, |name| dns_name_matches_domain(name, &cached.domain))
}
fn dns_name_matches_domain(pattern: &str, domain: &str) -> bool {
let pattern = normalize_dns_name(pattern);
let domain = normalize_dns_name(domain);
if pattern == domain {
return true;
}
let Some(suffix) = pattern.strip_prefix("*.") else {
return false;
};
let Some(prefix) = domain.strip_suffix(suffix) else {
return false;
};
prefix.ends_with('.') && !prefix[..prefix.len() - 1].contains('.')
}
fn normalize_dns_name(value: &str) -> String {
value.trim().trim_end_matches('.').to_ascii_lowercase()
}
#[cfg(test)]
mod tests {
use super::*;
fn cached_with_cert_info(
domain: &str,
subject_cn: Option<&str>,
san_names: Vec<&str>,
) -> CachedTlsData {
CachedTlsData {
server_hello_template: ParsedServerHello {
version: [0x03, 0x03],
random: [0u8; 32],
session_id: Vec::new(),
cipher_suite: [0x13, 0x01],
compression: 0,
extensions: Vec::new(),
},
cert_info: Some(crate::tls_front::types::ParsedCertificateInfo {
not_after_unix: None,
not_before_unix: None,
issuer_cn: None,
subject_cn: subject_cn.map(str::to_string),
san_names: san_names.into_iter().map(str::to_string).collect(),
}),
cert_payload: None,
app_data_records_sizes: vec![1024],
total_app_data_len: 1024,
behavior_profile: TlsBehaviorProfile::default(),
fetched_at: SystemTime::now(),
domain: domain.to_string(),
}
}
#[test]
fn cert_info_domain_match_accepts_exact_san() {
let cached = cached_with_cert_info("b.com", Some("a.com"), vec!["b.com"]);
assert!(cert_info_matches_domain(&cached));
}
#[test]
fn cert_info_domain_match_rejects_wrong_san() {
let cached = cached_with_cert_info("b.com", Some("b.com"), vec!["a.com"]);
assert!(!cert_info_matches_domain(&cached));
}
#[test]
fn cert_info_domain_match_accepts_single_label_wildcard_san() {
let cached = cached_with_cert_info("api.b.com", None, vec!["*.b.com"]);
assert!(cert_info_matches_domain(&cached));
}
#[test]
fn cert_info_domain_match_rejects_multi_label_wildcard_san() {
let cached = cached_with_cert_info("deep.api.b.com", None, vec!["*.b.com"]);
assert!(!cert_info_matches_domain(&cached));
}
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_uses_ttl() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
@@ -230,10 +514,68 @@ mod tests {
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_zero_ttl_always_allows_full_payload() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
let ip: IpAddr = "127.0.0.1".parse().expect("ip");
let ttl = Duration::ZERO;
assert!(cache.take_full_cert_budget_for_ip(ip, ttl).await);
assert!(cache.take_full_cert_budget_for_ip(ip, ttl).await);
for idx in 0..100_000u32 {
let ip = IpAddr::V4(std::net::Ipv4Addr::new(
10,
((idx >> 16) & 0xff) as u8,
((idx >> 8) & 0xff) as u8,
(idx & 0xff) as u8,
));
assert!(cache.take_full_cert_budget_for_ip(ip, ttl).await);
}
assert!(cache.full_cert_sent_is_empty_for_tests().await);
}
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_sweeps_expired_entries_when_due() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
let ttl = Duration::from_secs(1);
let stale_seen_at = Instant::now()
.checked_sub(Duration::from_secs(10))
.unwrap_or_else(Instant::now);
cache
.insert_full_cert_sent_for_tests(stale_ip, stale_seen_at)
.await;
cache
.full_cert_sent_last_sweep_epoch_secs
.store(0, Ordering::Relaxed);
assert!(cache.take_full_cert_budget_for_ip(new_ip, ttl).await);
assert!(!cache.full_cert_sent_contains_for_tests(stale_ip).await);
assert!(cache.full_cert_sent_contains_for_tests(new_ip).await);
}
#[tokio::test]
async fn test_take_full_cert_budget_for_ip_does_not_sweep_every_call() {
let cache = TlsFrontCache::new(&["example.com".to_string()], 1024, "tlsfront-test-cache");
let stale_ip: IpAddr = "127.0.0.1".parse().expect("ip");
let new_ip: IpAddr = "127.0.0.2".parse().expect("ip");
let ttl = Duration::from_secs(1);
let stale_seen_at = Instant::now()
.checked_sub(Duration::from_secs(10))
.unwrap_or_else(Instant::now);
let now_epoch_secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
cache
.insert_full_cert_sent_for_tests(stale_ip, stale_seen_at)
.await;
cache
.full_cert_sent_last_sweep_epoch_secs
.store(now_epoch_secs, Ordering::Relaxed);
assert!(cache.take_full_cert_budget_for_ip(new_ip, ttl).await);
assert!(cache.full_cert_sent_contains_for_tests(stale_ip).await);
assert!(cache.full_cert_sent_contains_for_tests(new_ip).await);
}
}

View File

@@ -3,7 +3,9 @@
use dashmap::DashMap;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use anyhow::{Result, anyhow};
@@ -144,12 +146,37 @@ enum FetchErrorKind {
Other,
}
const PROFILE_CACHE_MAX_ENTRIES: usize = 4096;
static PROFILE_CACHE: OnceLock<DashMap<ProfileCacheKey, ProfileCacheValue>> = OnceLock::new();
static PROFILE_CACHE_INSERT_GUARD: OnceLock<Mutex<()>> = OnceLock::new();
static PROFILE_CACHE_CAP_DROPS: AtomicU64 = AtomicU64::new(0);
fn profile_cache() -> &'static DashMap<ProfileCacheKey, ProfileCacheValue> {
PROFILE_CACHE.get_or_init(DashMap::new)
}
fn profile_cache_insert_guard() -> &'static Mutex<()> {
PROFILE_CACHE_INSERT_GUARD.get_or_init(|| Mutex::new(()))
}
fn sweep_expired_profile_cache(ttl: Duration, now: Instant) {
if ttl.is_zero() {
return;
}
profile_cache().retain(|_, value| now.saturating_duration_since(value.updated_at) <= ttl);
}
/// Current number of adaptive TLS fetch profile-cache entries.
pub(crate) fn profile_cache_entries_for_metrics() -> usize {
profile_cache().len()
}
/// Number of fresh profile-cache winners skipped because the cache was full.
pub(crate) fn profile_cache_cap_drops_for_metrics() -> u64 {
PROFILE_CACHE_CAP_DROPS.load(Ordering::Relaxed)
}
fn route_hint(
upstream: Option<&std::sync::Arc<crate::transport::UpstreamManager>>,
unix_sock: Option<&str>,
@@ -267,6 +294,43 @@ fn remember_profile_success(
let Some(key) = cache_key else {
return;
};
remember_profile_success_with_cap(strategy, key, profile, now, PROFILE_CACHE_MAX_ENTRIES);
}
fn remember_profile_success_with_cap(
strategy: &TlsFetchStrategy,
key: ProfileCacheKey,
profile: TlsFetchProfile,
now: Instant,
max_entries: usize,
) {
let Ok(_guard) = profile_cache_insert_guard().lock() else {
PROFILE_CACHE_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return;
};
if max_entries == 0 {
PROFILE_CACHE_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return;
}
if profile_cache().contains_key(&key) {
profile_cache().insert(
key,
ProfileCacheValue {
profile,
updated_at: now,
},
);
return;
}
if profile_cache().len() >= max_entries {
// TLS fetch is control-plane work; sweeping under a tiny mutex keeps
// profile-cache cardinality hard-bounded without touching relay hot paths.
sweep_expired_profile_cache(strategy.profile_cache_ttl, now);
}
if profile_cache().len() >= max_entries {
PROFILE_CACHE_CAP_DROPS.fetch_add(1, Ordering::Relaxed);
return;
}
profile_cache().insert(
key,
ProfileCacheValue {

View File

@@ -618,13 +618,9 @@ impl MePool {
me_route_hybrid_max_wait: Duration::from_millis(
me_route_hybrid_max_wait_ms.max(50),
),
me_route_blocking_send_timeout: if me_route_blocking_send_timeout_ms == 0 {
None
} else {
Some(Duration::from_millis(
me_route_blocking_send_timeout_ms.min(5_000),
))
},
me_route_blocking_send_timeout: Some(Duration::from_millis(
me_route_blocking_send_timeout_ms.clamp(1, 5_000),
)),
me_route_last_success_epoch_ms: AtomicU64::new(0),
me_route_hybrid_timeout_warn_epoch_ms: AtomicU64::new(0),
me_async_recovery_last_trigger_epoch_ms: AtomicU64::new(0),

View File

@@ -365,7 +365,10 @@ impl MePool {
}
}
pub async fn zero_downtime_reinit_after_map_change(self: &Arc<Self>, rng: &SecureRandom) {
pub async fn zero_downtime_reinit_after_map_change(
self: &Arc<Self>,
rng: &SecureRandom,
) -> bool {
let desired_by_dc = self.desired_dc_endpoints().await;
let now_epoch_secs = Self::now_epoch_secs();
let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs);
@@ -380,7 +383,7 @@ impl MePool {
MeDrainGateReason::CoverageQuorum
};
self.set_last_drain_gate(false, false, reason, now_epoch_secs);
return;
return false;
}
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
@@ -490,7 +493,7 @@ impl MePool {
missing_dc = ?missing_dc,
"ME reinit coverage below threshold; keeping stale writers"
);
return;
return false;
}
if hardswap {
@@ -520,7 +523,7 @@ impl MePool {
missing_dc = ?fresh_missing_dc,
"ME hardswap pending: fresh generation DC coverage incomplete"
);
return;
return false;
}
}
@@ -567,7 +570,7 @@ impl MePool {
self.clear_pending_hardswap_state();
}
debug!("ME reinit cycle completed with no stale writers");
return;
return true;
}
let drain_timeout = self.force_close_timeout();
@@ -606,10 +609,11 @@ impl MePool {
if hardswap {
self.clear_pending_hardswap_state();
}
true
}
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) {
self.zero_downtime_reinit_after_map_change(rng).await;
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) -> bool {
self.zero_downtime_reinit_after_map_change(rng).await
}
}

View File

@@ -47,6 +47,7 @@ pub async fn me_reinit_scheduler(
rng: Arc<SecureRandom>,
config_rx: watch::Receiver<Arc<ProxyConfig>>,
mut trigger_rx: mpsc::Receiver<MeReinitTrigger>,
me_ready_tx: watch::Sender<u64>,
) {
info!("ME reinit scheduler started");
loop {
@@ -90,15 +91,25 @@ pub async fn me_reinit_scheduler(
if cfg.general.me_reinit_singleflight {
debug!(reason, "ME reinit scheduled (single-flight)");
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
if pool.zero_downtime_reinit_periodic(rng.as_ref()).await {
me_ready_tx.send_modify(|version| {
*version = version.saturating_add(1);
});
}
} else {
debug!(reason, "ME reinit scheduled (concurrent mode)");
let pool_clone = pool.clone();
let rng_clone = rng.clone();
let me_ready_tx_clone = me_ready_tx.clone();
tokio::spawn(async move {
pool_clone
if pool_clone
.zero_downtime_reinit_periodic(rng_clone.as_ref())
.await;
.await
{
me_ready_tx_clone.send_modify(|version| {
*version = version.saturating_add(1);
});
}
});
}
}

View File

@@ -18,6 +18,9 @@ const PROXY_V1_MIN_LEN: usize = 6;
/// Minimum length for v2 header
const PROXY_V2_MIN_LEN: usize = 16;
/// Maximum accepted PROXY v2 address and TLV payload.
const PROXY_V2_MAX_ADDR_LEN: usize = 216;
/// Address families for v2
mod address_family {
pub const UNSPEC: u8 = 0x0;
@@ -169,6 +172,9 @@ async fn parse_v2<R: AsyncRead + Unpin>(
let family_protocol = header[13];
let addr_len = u16::from_be_bytes([header[14], header[15]]) as usize;
if addr_len > PROXY_V2_MAX_ADDR_LEN {
return Err(ProxyError::InvalidProxyProtocol);
}
// Read address data
let mut addr_data = vec![0u8; addr_len];

File diff suppressed because it is too large Load Diff