Compare commits

...

5 Commits

Author SHA1 Message Date
dependabot[bot]
308fefc685 build(deps): bump github/codeql-action/init from 4.36.2 to 4.36.3
Bumps [github/codeql-action/init](https://github.com/github/codeql-action) from 4.36.2 to 4.36.3.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](8aad20d150...54f647b7e1)

---
updated-dependencies:
- dependency-name: github/codeql-action/init
  dependency-version: 4.36.3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-07-23 04:16:33 +00:00
Nikolay
344d3e1911 lib/httputil: add load-balancing http transport
This commit adds http client load-balancing with DNS discovery. For both A and SRV records.
It helps to route HTTP requests evenly for across discovered backends.

Discovered IP addresses are cached locally.

 The main benefit of this feature is to remove intermediate vmauth as a load-balancer between
vmagent (vmalert) and remote targets. Which simplifies components management and
reduces operational overhead.

 By default, it's disable and could be used with `dns+` or `srv+` hostname suffix at url.
For example, `-remoteWrite.url=http://dns+victoria-metrics:8428/api/v1/write`.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388
2026-07-22 18:34:23 +02:00
Andrei Baidarov
8b94ca0e30 lib/persistentqueue: expose fastQueue name in metrics
This commit adds `name` label to the `vm_persistentqueue_*` metrics. It provides more context for queues with human read-able format. Because `path` label contains only on-disk path, which is not really useful.

 For vmagent case it will contain remote.writeURL position at command-line args.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944
2026-07-22 18:28:58 +02:00
f41gh7
a9ae7e1da0 vendor: update golang.org/x/... packages 2026-07-22 18:00:29 +02:00
Max Kotliar
ba59d78f10 docs: update supported LTS versions 2026-07-22 12:56:17 +03:00
23 changed files with 720 additions and 68 deletions

View File

@@ -54,7 +54,7 @@ jobs:
restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
- name: Initialize CodeQL
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3
with:
languages: go

View File

@@ -151,17 +151,23 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
}
tr.Proxy = http.ProxyURL(pu)
}
hc := &http.Client{
Transport: authCfg.NewRoundTripper(tr),
Timeout: sendTimeout.GetOptionalArg(argIdx),
}
rwURL, err := url.Parse(remoteWriteURL)
if err != nil {
logger.Fatalf("BUG: cannot parse already parsed -remoteWrite.url=%q: %s", remoteWriteURL, err)
}
hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
retryMaxIntervalFlag := retryMaxTime
if retryMaxInterval.String() != "" {
retryMaxIntervalFlag = retryMaxInterval
}
c := &client{
sanitizedURL: sanitizedURL,
remoteWriteURL: remoteWriteURL,
remoteWriteURL: rwURL.String(),
authCfg: authCfg,
awsCfg: awsCfg,
fq: fq,

View File

@@ -11,6 +11,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)
@@ -94,6 +95,12 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
tr.MaxIdleConns = tr.MaxIdleConnsPerHost
}
tr.IdleConnTimeout = *idleConnectionTimeout
hc := &http.Client{Transport: tr}
datasourceURL, err := url.Parse(*addr)
if err != nil {
logger.Fatalf("BUG: cannot parse already parsed -datasource.url=%q: %s", *addr, err)
}
hc.Transport, datasourceURL = httputil.NewLoadBalancerTransport(tr, datasourceURL)
if extraParams == nil {
extraParams = url.Values{}
@@ -120,9 +127,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
}
return &Client{
c: &http.Client{Transport: tr},
c: hc,
authCfg: authCfg,
datasourceURL: strings.TrimSuffix(*addr, "/"),
datasourceURL: strings.TrimSuffix(datasourceURL.String(), "/"),
appendTypePrefix: *appendTypePrefix,
queryStep: *queryStep,
extraParams: extraParams,

View File

@@ -4,12 +4,14 @@ import (
"flag"
"fmt"
"net/http"
"net/url"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/vmalertutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)
@@ -76,7 +78,13 @@ func Init() (datasource.QuerierBuilder, error) {
return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
}
tr.IdleConnTimeout = *idleConnectionTimeout
c := &http.Client{Transport: tr}
rrURL, err := url.Parse(*addr)
if err != nil {
logger.Fatalf("BUG: cannot parse already parsed -remoteRead.url=%q: %s", *addr, err)
}
c.Transport, rrURL = httputil.NewLoadBalancerTransport(tr, rrURL)
endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
if err != nil {
return nil, fmt.Errorf("cannot parse JSON for -remoteRead.oauth2.endpointParams=%s: %w", *oauth2EndpointParams, err)
@@ -89,6 +97,5 @@ func Init() (datasource.QuerierBuilder, error) {
if err != nil {
return nil, fmt.Errorf("failed to configure auth: %w", err)
}
c := &http.Client{Transport: tr}
return datasource.NewPrometheusClient(*addr, authCfg, false, c), nil
return datasource.NewPrometheusClient(rrURL.String(), authCfg, false, c), nil
}

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"sync"
@@ -111,12 +112,18 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
if cfg.Concurrency > 0 {
cc = cfg.Concurrency
}
hc := &http.Client{
Timeout: *sendTimeout,
Transport: cfg.Transport,
}
rwURL, err := url.Parse(cfg.Addr)
if err != nil {
logger.Fatalf("cannot parse already parsed -remoteWrite.url=%q: %s", cfg.Addr, err)
}
hc.Transport, rwURL = httputil.NewLoadBalancerTransport(hc.Transport, rwURL)
c := &Client{
c: &http.Client{
Timeout: *sendTimeout,
Transport: cfg.Transport,
},
addr: strings.TrimSuffix(cfg.Addr, "/"),
c: hc,
addr: strings.TrimSuffix(rwURL.String(), "/"),
authCfg: cfg.AuthCfg,
flushInterval: cfg.FlushInterval,
maxBatchSize: cfg.MaxBatchSize,

View File

@@ -10,7 +10,7 @@ groups:
concurrency: 2
rules:
- alert: PersistentQueueIsDroppingData
expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path) > 0
expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) without (path,name) > 0
for: 10m
labels:
severity: critical
@@ -201,4 +201,4 @@ groups:
annotations:
summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing.
Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."

View File

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
## Currently supported LTS release lines
- v1.148.x - the latest one is [v1.148.0 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
- v1.136.x - the latest one is [v1.136.14 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.14)
- v1.122.x - the latest one is [v1.122.27 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.27)

View File

@@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
## [v1.148.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
Released at 2026-07-20
@@ -36,6 +38,7 @@ Released at 2026-07-20
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): support scraping metrics over Unix domain sockets. The socket path can be configured via the `__unix_socket__` target label. See [#11156](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11156). Thanks to @vinyas-bharadwaj for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Improve background discovery performance for [http_sd](https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs) discovery. See [#8838](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8838).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add client side least-loaded load-balancing with `DNS` discovery. See [#2388](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2388) and these [vmagent DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmagent/#dns-urls), [vmalert DNS URLs](https://docs.victoriametrics.com/victoriametrics/vmalert/#dns-urls).
* FEATURE: [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-maxBackfillAge` command-line flag for limiting ingestion of samples with historical timestamps, for example, when older data has been moved between storage tiers (nvme/hdd, hot/cold). See [#11199](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11199). Thanks to @AshwinRamaniPsg for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): automatically preload relabeling rules configured via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` in the [metrics relabel debug UI](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging). See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).

View File

@@ -466,6 +466,43 @@ and `-remoteWrite.streamAggr.config`:
There is also the `-promscrape.configCheckInterval` command-line flag, which can be used to automatically reload configs from the updated `-promscrape.config` file.
## DNS URLs
If `vmagent` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses
via [DNS A records](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1). The port from the original URL is appended to each discovered IP address.
Each discovered IP address is used for least-loaded balancing of write requests.
DNS URLs are supported in the following places:
* In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
`192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428/api/v1/write` is automatically resolved into
`-remoteWrite.url=http://192.168.1.15:8428/api/v1/write`.
DNS URLs are useful when client-side HTTP load balancing is needed. A good example
is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
which returns multiple IP addresses for a single hostname.
### DNS URLs and HTTPS
When a `dns+` URL uses the `https` scheme, `vmagent` connects to the discovered
IP addresses directly. This affects [TLS](https://en.wikipedia.org/wiki/Transport_Layer_Security)
in two ways:
* No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication) is sent in the TLS handshake,
since the connection target is an IP address rather than a hostname.
* The server certificate is verified against the IP address, so the verification fails
unless the certificate contains the corresponding
[IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
To use `dns+` URLs with HTTPS, pass the original hostname via the `-remoteWrite.tlsServerName`
command-line flag. It is used both as SNI and as the name the server certificate
is verified against:
```sh
-remoteWrite.url=https://dns+victoria-metrics:8428/api/v1/write
-remoteWrite.tlsServerName=victoria-metrics
```
## SRV URLs
If `vmagent` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
@@ -476,7 +513,7 @@ SRV URLs are supported in the following places:
* In `-remoteWrite.url` command-line flag. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
`victoria-metrics-host:8428` TCP address, then `-remoteWrite.url=http://srv+victoria-metrics/api/v1/write` is automatically resolved into
`-remoteWrite.url=http://victoria-metrics-host:8428/api/v1/write`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmagent`
uses a randomly chosen address for each connection it establishes to the remote storage.
performs per request least-loaded load-balancing.
* In scrape target addresses aka `__address__` label. See [these docs](https://docs.victoriametrics.com/victoriametrics/relabeling/#how-to-modify-scrape-urls-in-targets) for details.

View File

@@ -1512,6 +1512,59 @@ alert_relabel_configs:
The configuration file can be [hot-reloaded](#hot-config-reload).
## DNS URLs
If `vmalert` encounters URLs with the `dns+` prefix in the hostname (such as `http://dns+some-addr:8428/some/path`), it resolves `some-addr` into IP addresses via DNS A/AAAA records.
The port from the original URL is appended to each discovered IP address.
Each discovered IP address is used for least-loaded balancing of write requests.
DNS URLs are supported in the following places:
* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS A Record](https://datatracker.ietf.org/doc/html/rfc1035#section-3.4.1) record contains
`192.168.1.15` IP address, then `-remoteWrite.url=http://dns+victoria-metrics:8428` is automatically resolved into
`-remoteWrite.url=http://192.168.1.15:8428`.
DNS URLs are useful when client-side HTTP load balancing is needed. A good example
is a [Kubernetes headless Service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services),
which returns multiple IP addresses for a single hostname.
### DNS URLs and HTTPS
When a `dns+` URL uses the `https` scheme, `vmalert` connects to the discovered
IP addresses directly. No [SNI](https://en.wikipedia.org/wiki/Server_Name_Indication)
is sent in the TLS handshake, and the server certificate is verified against the IP address,
which fails unless the certificate contains the corresponding
[IP SAN](https://en.wikipedia.org/wiki/Subject_Alternative_Name) entries.
To use `dns+` URLs with HTTPS, pass the original hostname via the corresponding
`tlsServerName` command-line flag - `-datasource.tlsServerName`, `-remoteRead.tlsServerName`
or `-remoteWrite.tlsServerName`. It is used both as SNI and as the name the server
certificate is verified against:
```sh
-datasource.url=https://dns+victoria-metrics:8428
-datasource.tlsServerName=victoria-metrics
```
Alternatively, issue server certificates with IP SAN entries for every backend IP address.
Avoid `tlsInsecureSkipVerify` flags for working around this, since they disable
server certificate verification completely.
## SRV URLs
If `vmalert` encounters URLs with `srv+` prefix in hostname (such as `http://srv+some-addr/some/path`), then it resolves `some-addr` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record)
record into TCP address with hostname and TCP port, and then use the resulting URL when it needs to connect to it.
SRV URLs are supported in the following places:
* In `-remoteWrite.url`, `-remoteRead.url` and `-datasource.url` command-line flags. For example, if `victoria-metrics` [DNS SRV](https://en.wikipedia.org/wiki/SRV_record) record contains
`victoria-metrics-host:8085`, then `-remoteWrite.url=http://srv+victoria-metrics:8428` is automatically resolved into
`-remoteWrite.url=http://victoria-metrics-host:8085`. If the DNS SRV record is resolved into multiple TCP addresses, then `vmalert`
performs per request round-robin load-balancing.
SRV URLs are useful when HTTP services run on different TCP ports or when their TCP ports can change over time (for instance, after a restart).
## Contributing
`vmalert` is mostly designed and built by VictoriaMetrics community.

20
go.mod
View File

@@ -34,7 +34,7 @@ require (
github.com/valyala/gozstd v1.24.0
github.com/valyala/histogram v1.2.0
github.com/valyala/quicktemplate v1.8.0
golang.org/x/net v0.56.0
golang.org/x/net v0.57.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sys v0.47.0
google.golang.org/api v0.284.0
@@ -130,6 +130,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
github.com/yuin/goldmark v1.8.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/collector/component v1.60.0 // indirect
go.opentelemetry.io/collector/confmap v1.60.0 // indirect
@@ -155,12 +156,19 @@ require (
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.53.0 // indirect
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/term v0.44.0 // indirect
golang.org/x/text v0.38.0 // indirect
golang.org/x/crypto v0.54.0 // indirect
golang.org/x/exp v0.0.0-20260718201538-764159d718ef // indirect
golang.org/x/mod v0.38.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75 // indirect
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.48.0 // indirect
golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
golang.org/x/tools/godoc v0.1.0-deprecated // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20260615183401-62b3387ff324 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect

29
go.sum
View File

@@ -438,6 +438,8 @@ github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAz
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.8.4 h1:oat/nd3U6NeQqFEL3xpEJq7d7c86NI+DbSNGAs4xnjA=
github.com/yuin/goldmark v1.8.4/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/collector/component v1.60.0 h1:LpIjHMn7OOjUsFR84ROc2kqPbP1xnKyDCGi7ZVqEaKU=
@@ -521,18 +523,26 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M=
golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY=
golang.org/x/exp v0.0.0-20260718201538-764159d718ef h1:LkZ48HFgy/TvhTI0bcWkjgFkgLyKUwcTbDjS0DUjw+A=
golang.org/x/exp v0.0.0-20260718201538-764159d718ef/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -540,6 +550,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -548,12 +560,18 @@ golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75 h1:I9ygRooEYoVHV0SRNOSr/KVjTf5EeJ52BuNkVjsP2GU=
golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg=
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -562,10 +580,21 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.46.0 h1:7jTurBkPZu4moS/Uy4OQT1M+QBlsj3wejyZwsT8Z7rk=
golang.org/x/tools v0.46.0/go.mod h1:FrD85F8l+NWL+9XWBSyVSHO6Ne4jutsfIFba7AWQ5Ys=
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/api v0.284.0 h1:i+cKTgeQRcRySkP7QTl5PDO7/pAm8EcMFIUMlNbk4Vc=

View File

@@ -0,0 +1,334 @@
package httputil
import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"net/url"
"sort"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
)
const (
brokenBackendTimeout = 5 * time.Second
backendDiscoveryInterval = 10 * time.Second
backendDiscoveryTimeout = 10 * time.Second
)
// NewLoadBalancerTransport returns new RoundTripper that performs least-loaded HTTP requests loadbalancing
// based on discovered backends for the given url host
// and update url with load-balancing prefix
//
// It returns origin transport and url if load-balancing is not needed for given url
func NewLoadBalancerTransport(origin http.RoundTripper, originURL *url.URL) (http.RoundTripper, *url.URL) {
modifiedURL := *originURL
var discoverFunc func(context.Context, string, string) ([]*backend, error)
switch {
case strings.HasPrefix(originURL.Host, "dns+"):
modifiedURL.Host = modifiedURL.Host[4:]
discoverFunc = discoverDNSBackends
case strings.HasPrefix(originURL.Host, "srv+"):
modifiedURL.Host = modifiedURL.Host[4:]
discoverFunc = discoverSRVBackends
default:
return origin, originURL
}
host, port, err := net.SplitHostPort(modifiedURL.Host)
if err != nil {
host = modifiedURL.Host
port = "80"
if modifiedURL.Scheme == "https" {
port = "443"
}
}
t := &loadbalancerTransport{
tr: origin,
host: host,
port: port,
discoverFunc: discoverFunc,
}
t.discoverBackends()
return t, &modifiedURL
}
type loadbalancerTransport struct {
tr http.RoundTripper
host string
port string
discoverFunc func(context.Context, string, string) ([]*backend, error)
nextDiscoveryDeadline atomic.Uint64
discovering atomic.Bool
dbs atomic.Pointer[discoveredBackends]
}
type discoveredBackends struct {
backends []*backend
// n is an atomic counter, which is used for balancing load among available backends.
n atomic.Uint32
}
// getLeastLoadedBackend returns least loaded backend
// caller must release backend with backend.put() method
func (dbs *discoveredBackends) getLeastLoadedBackend() *backend {
firstB := dbs.backends[0]
if len(dbs.backends) == 1 {
firstB.get()
return firstB
}
// Slow path - select other backends.
n := dbs.n.Add(1) - 1
for i := range uint32(len(dbs.backends)) {
idx := (n + i) % uint32(len(dbs.backends))
bu := dbs.backends[idx]
if bu.isBroken() {
continue
}
// The Load() in front of CompareAndSwap() avoids CAS overhead for items with values bigger than 0.
if bu.concurrentRequests.Load() == 0 && bu.concurrentRequests.CompareAndSwap(0, 1) {
dbs.n.CompareAndSwap(n+1, idx+1)
// There is no need in the call b.get(), because we already incremented b.concurrentRequests above.
return bu
}
}
// Slow path - return the backend with the minimum number of concurrently executed requests.
bMinIdx := n % uint32(len(dbs.backends))
minRequests := dbs.backends[bMinIdx].concurrentRequests.Load()
for i := uint32(1); i < uint32(len(dbs.backends)); i++ {
idx := (n + i) % uint32(len(dbs.backends))
bu := dbs.backends[idx]
if bu.isBroken() {
continue
}
reqs := bu.concurrentRequests.Load()
if reqs < minRequests || dbs.backends[bMinIdx].isBroken() {
bMinIdx = idx
minRequests = reqs
}
}
bMin := dbs.backends[bMinIdx]
if bMin.isBroken() {
// If all backends are broken, then returns the first backend.
firstB.get()
return firstB
}
bMin.get()
dbs.n.CompareAndSwap(n+1, bMinIdx+1)
return bMin
}
type backend struct {
addr string
concurrentRequests atomic.Int32
brokenDeadline atomic.Uint64
}
func (b *backend) get() {
b.concurrentRequests.Add(1)
}
func (b *backend) put() {
b.concurrentRequests.Add(-1)
}
func (b *backend) isBroken() bool {
bd := b.brokenDeadline.Load()
if bd == 0 {
return false
}
ct := fasttime.UnixTimestamp()
return ct < bd
}
// RoundTrip implements http.RoundTripper interface
func (lb *loadbalancerTransport) RoundTrip(r *http.Request) (*http.Response, error) {
dbs := lb.getBackends()
if dbs == nil || len(dbs.backends) == 0 {
return nil, fmt.Errorf("no backends found for hostname=%q", lb.host)
}
maxRetries := len(dbs.backends)
var lastErr error
for range maxRetries {
b := dbs.getLeastLoadedBackend()
resp, err := lb.doRequest(r, b)
if err != nil {
ct := fasttime.UnixTimestamp()
brokenDeadline := ct + uint64(brokenBackendTimeout.Seconds())
b.brokenDeadline.Store(brokenDeadline)
if !netutil.IsTrivialNetworkError(err) {
return nil, err
}
// perform the same check for retry as http.Request.isReplayable does
canRetry := r.Body == nil || r.Body == http.NoBody || r.GetBody != nil
if !canRetry {
return nil, err
}
lastErr = err
continue
}
return resp, nil
}
return nil, fmt.Errorf("all backends are unavailable: %w", lastErr)
}
func (lb *loadbalancerTransport) doRequest(r *http.Request, b *backend) (*http.Response, error) {
r2 := r.Clone(r.Context())
if r.GetBody != nil {
body, err := r.GetBody()
if err != nil {
b.put()
return nil, err
}
r2.Body = body
}
r2.URL.Host = b.addr
if r2.Host == "" {
r2.Host = r.URL.Host
}
resp, err := lb.tr.RoundTrip(r2)
if err != nil {
b.put()
return nil, err
}
// wrap response body with readCloser that releases backend after Close call
// it's needed to properly account loaded backends at getLeastLoadedBackends
resp.Body = newReleaseReadCloser(resp.Body, b)
return resp, nil
}
func (lb *loadbalancerTransport) getBackends() *discoveredBackends {
ct := fasttime.UnixTimestamp()
deadline := lb.nextDiscoveryDeadline.Load()
if ct < deadline || !lb.discovering.CompareAndSwap(false, true) {
return lb.dbs.Load()
}
lb.discoverBackends()
return lb.dbs.Load()
}
func (lb *loadbalancerTransport) discoverBackends() {
ctx, cancel := context.WithTimeout(context.Background(), backendDiscoveryTimeout)
defer func() {
cancel()
ct := fasttime.UnixTimestamp()
nextDeadline := ct + uint64(backendDiscoveryInterval.Seconds())
lb.nextDiscoveryDeadline.Store(nextDeadline)
lb.discovering.Store(false)
}()
backends, err := lb.discoverFunc(ctx, lb.host, lb.port)
if err != nil {
logger.Errorf("cannot discover backends: %s, retry in %s", err, backendDiscoveryInterval)
return
}
dbs := &discoveredBackends{
backends: backends,
}
sort.Slice(dbs.backends, func(i, j int) bool {
return dbs.backends[i].addr < dbs.backends[j].addr
})
prevBackends := lb.dbs.Load()
if areBackendsEqual(prevBackends, dbs) {
return
}
lb.dbs.Store(dbs)
}
func discoverDNSBackends(ctx context.Context, host, port string) ([]*backend, error) {
addrs, err := netutil.Resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, fmt.Errorf("failed to lookupIPAddr for host: %q: %w", host, err)
}
backends := make([]*backend, 0, len(addrs))
for _, addr := range addrs {
if !netutil.TCP6Enabled() {
ip, ok := netip.AddrFromSlice(addr.IP)
if !ok {
logger.Panicf("BUG: cannot build netip Addr from slice addr: %q", addr.IP.String())
}
if !ip.Unmap().Is4() {
continue
}
}
ip := addr.IP.String()
if len(port) > 0 {
ip = net.JoinHostPort(ip, port)
}
backends = append(backends, &backend{addr: ip})
}
return backends, nil
}
func discoverSRVBackends(ctx context.Context, host, port string) ([]*backend, error) {
_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", host)
if err != nil {
return nil, fmt.Errorf("failed to LookupSRV records for host: %q: %w", host, err)
}
backends := make([]*backend, 0, len(addrs))
for _, addr := range addrs {
hostPort := port
if addr.Port > 0 {
hostPort = strconv.FormatUint(uint64(addr.Port), 10)
}
hostAddr := net.JoinHostPort(addr.Target, hostPort)
backends = append(backends, &backend{addr: hostAddr})
}
return backends, nil
}
func newReleaseReadCloser(responseBody io.ReadCloser, b *backend) io.ReadCloser {
return &releaseReadCloser{
ReadCloser: responseBody,
b: b,
}
}
type releaseReadCloser struct {
io.ReadCloser
b *backend
released atomic.Bool
}
func (rrc *releaseReadCloser) Close() error {
if rrc.released.CompareAndSwap(false, true) {
// Close method could be called multiple times
// and it must produce idempotent result
rrc.b.put()
}
return rrc.ReadCloser.Close()
}
func areBackendsEqual(left, right *discoveredBackends) bool {
if left == nil || right == nil {
return left == right
}
if len(left.backends) != len(right.backends) {
return false
}
for idx, leftB := range left.backends {
rightB := right.backends[idx]
if leftB.addr != rightB.addr {
return false
}
}
return true
}

View File

@@ -0,0 +1,127 @@
package httputil
import (
"context"
"fmt"
"net"
"net/http"
"net/netip"
"net/url"
"sync"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
)
type testRemoteServer struct {
mu sync.Mutex
requestsPerHost map[string]int
totalRequests int
firstError error
}
func (trs *testRemoteServer) RoundTrip(r *http.Request) (*http.Response, error) {
trs.mu.Lock()
if trs.firstError != nil && trs.totalRequests == 0 {
err := trs.firstError
trs.firstError = nil
trs.totalRequests++
trs.mu.Unlock()
return nil, err
}
trs.totalRequests++
if trs.requestsPerHost == nil {
trs.requestsPerHost = make(map[string]int)
}
trs.requestsPerHost[r.URL.Host]++
trs.mu.Unlock()
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
}
type testDNSResolver struct {
ips []net.IPAddr
}
func (tdr *testDNSResolver) LookupSRV(_ context.Context, _, _, name string) (cname string, addrs []*net.SRV, err error) {
return "", nil, fmt.Errorf("unexpected LookupMX call for name=%q", name)
}
func (tdr *testDNSResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) {
return tdr.ips, nil
}
func (tdr *testDNSResolver) LookupMX(_ context.Context, name string) ([]*net.MX, error) {
return nil, fmt.Errorf("unexpected LookupMX call for name=%q", name)
}
func TestLoadbalancerTransport(t *testing.T) {
f := func(discoveredIPs []string, trs *testRemoteServer) {
t.Helper()
parsedIPs := make([]net.IPAddr, 0, len(discoveredIPs))
for _, dIP := range discoveredIPs {
pIP, err := netip.ParseAddr(dIP)
if err != nil {
t.Fatalf("cannot parse IP=%q: %s", dIP, err)
}
parsedIPs = append(parsedIPs, net.IPAddr{IP: pIP.AsSlice()})
}
tdr := &testDNSResolver{ips: parsedIPs}
originResolver := netutil.Resolver
defer func() { netutil.Resolver = originResolver }()
netutil.Resolver = tdr
requestURL, err := url.Parse("http://dns+vmsingle.example.com:8429/api/v1/write")
if err != nil {
t.Fatalf("cannot parse url: %s", err)
}
lbt, requestURL := NewLoadBalancerTransport(trs, requestURL)
if len(discoveredIPs) == 0 {
r, err := http.NewRequest(http.MethodGet, requestURL.String(), nil)
if err != nil {
t.Fatalf("cannot create http request: %s", err)
}
_, err = lbt.RoundTrip(r)
if err == nil {
t.Fatalf("expected no backends found error")
}
return
}
expectedRequestsPerHost := 2
for range len(discoveredIPs) * expectedRequestsPerHost {
r, err := http.NewRequest(http.MethodGet, requestURL.String(), nil)
if err != nil {
t.Fatalf("cannot create http request: %s", err)
}
resp, err := lbt.RoundTrip(r)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
resp.Body.Close()
}
requestsPerHost := trs.requestsPerHost
for _, dIP := range discoveredIPs {
expectedHostPort := net.JoinHostPort(dIP, "8429")
gotRequestsPerHost, ok := requestsPerHost[expectedHostPort]
if !ok {
t.Fatalf("not found expected backend request for: %q", expectedHostPort)
}
if gotRequestsPerHost != expectedRequestsPerHost {
t.Fatalf("unexpected requests per host:%q %d:%d (-;+)", expectedHostPort, expectedRequestsPerHost, gotRequestsPerHost)
}
}
}
trs := testRemoteServer{}
f([]string{"1.1.1.1"}, &trs)
trs = testRemoteServer{}
f([]string{"1.1.1.1", "2.2.2.2", "5.5.5.5"}, &trs)
// empty backends, expecting error
trs = testRemoteServer{}
f([]string{}, &trs)
}

View File

@@ -95,14 +95,14 @@ func mustOpenFastQueue(path, name string, opts OpenFastQueueOpts) *FastQueue {
}
fq.cond.L = &fq.mu
fq.lastInmemoryBlockReadTime = fasttime.UnixTimestamp()
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_bytes_pending{path=%q}`, path), func() float64 {
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_bytes_pending{path=%q, name=%q}`, path, name), func() float64 {
fq.mu.Lock()
n := fq.pq.GetPendingBytes()
fq.mu.Unlock()
return float64(n)
})
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_free_disk_space_bytes{path=%q}`, path), func() float64 {
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vm_persistentqueue_free_disk_space_bytes{path=%q, name=%q}`, path, name), func() float64 {
freeSpaceBytes := fs.MustGetFreeSpace(path)
// Limited by disk space if remoteWrite.maxDiskUsagePerURL wasn't set
if maxPendingBytes == 0 {

View File

@@ -156,12 +156,12 @@ func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingB
q.dir = path
q.name = name
q.blocksDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_dropped_total{path=%q}`, path))
q.bytesDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_dropped_total{path=%q}`, path))
q.blocksWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_written_total{path=%q}`, path))
q.bytesWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_written_total{path=%q}`, path))
q.blocksRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_read_total{path=%q}`, path))
q.bytesRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_read_total{path=%q}`, path))
q.blocksDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_dropped_total{path=%q, name=%q}`, path, name))
q.bytesDropped = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_dropped_total{path=%q, name=%q}`, path, name))
q.blocksWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_written_total{path=%q, name=%q}`, path, name))
q.bytesWritten = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_written_total{path=%q, name=%q}`, path, name))
q.blocksRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_blocks_read_total{path=%q, name=%q}`, path, name))
q.bytesRead = metrics.GetOrCreateCounter(fmt.Sprintf(`vm_persistentqueue_bytes_read_total{path=%q, name=%q}`, path, name))
cleanOnError := func() {
if q.reader != nil {

View File

@@ -55,7 +55,7 @@ type transportConfig struct {
// Registered is called by net/http.Transport.RegisterProtocol,
// to let us know that it understands the registration mechanism we're using.
func (t transportConfig) Registered(t1 *http.Transport) {
t.t.t1 = t1
t.t.lazyt1 = t1
}
func (t transportConfig) DisableCompression() bool {
@@ -145,29 +145,30 @@ func (t transportConfig) DialFromContext(ctx context.Context, network, address s
type transportInternal struct {
initOnce sync.Once
t1 *http.Transport
lazyt1 *http.Transport
}
func (t *Transport) init() {
func (t *Transport) init() *http.Transport {
t.initOnce.Do(func() {
if t.t1 != nil {
if t.lazyt1 != nil {
return
}
t1 := &http.Transport{}
t.configure(t1)
})
return t.lazyt1
}
func (t *Transport) configure(t1 *http.Transport) {
t1.RegisterProtocol("http/2", transportConfig{t})
// tr2.t1 is set by transportConfig.Registered.
if t.t1 != t1 {
// tr2.lazyt1 is set by transportConfig.Registered.
if t.lazyt1 != t1 {
panic("http2: net/http does not support this version of x/net/http2")
}
}
func (t *Transport) roundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Response, error) {
t.init()
t1 := t.init()
if req.URL.Scheme == "http" && !t.AllowHTTP {
return nil, errors.New("http2: unencrypted HTTP/2 not enabled")
@@ -188,22 +189,23 @@ func (t *Transport) roundTripOpt(req *http.Request, opt RoundTripOpt) (*http.Res
ctx := context.WithValue(req.Context(), http2TransportContextKey{}, t)
req = req.WithContext(ctx)
return t.t1.RoundTrip(req)
return t1.RoundTrip(req)
}
func (t *Transport) closeIdleConnections() {
t.init()
t.t1.CloseIdleConnections()
t1 := t.init()
t1.CloseIdleConnections()
}
func (t *Transport) newUserClientConn(c net.Conn) (*ClientConn, error) {
t1 := t.init()
// http.Transport's NewClientConn doesn't provide a supported way to create
// a connection from a net.Conn. (This might be useful to add in the future?)
// We're going to craftily sneak one in via the context key, with the
// scheme of "http/2" telling NewClientConn to look for it.
ctx := context.WithValue(context.Background(), netConnContextKey{}, c)
nhcc, err := t.t1.NewClientConn(ctx, "http/2", "")
nhcc, err := t1.NewClientConn(ctx, "http/2", "")
if err != nil {
return nil, err
}

View File

@@ -400,7 +400,11 @@ func (p *Profile) process(s string, toASCII bool) (string, error) {
// Spec says keep the old label.
continue
}
if unicode16 && err == nil && len(u) > 0 && isASCII(u) {
if err == nil && len(u) > 0 && isASCII(u) {
// UTS 43 pre-revision 33 doesn't classify a xn-- label
// which contains only ASCII characters as an error,
// but that's a specification bug and a security issue.
// Always return an error in this case.
err = punyError(enc)
}
isBidi = isBidi || bidirule.DirectionString(u) != bidi.LeftToRight

View File

@@ -24,7 +24,7 @@ func NewWeighted(n int64) *Weighted {
}
// Weighted provides a way to bound concurrent access to a resource.
// The callers can request access with a given weight.
// The callers can request access with a given non-negative weight.
type Weighted struct {
size int64
cur int64
@@ -32,10 +32,13 @@ type Weighted struct {
waiters list.List
}
// Acquire acquires the semaphore with a weight of n, blocking until resources
// Acquire acquires the semaphore with a non-negative weight of n, blocking until resources
// are available or ctx is done. On success, returns nil. On failure, returns
// ctx.Err() and leaves the semaphore unchanged.
func (s *Weighted) Acquire(ctx context.Context, n int64) error {
if n < 0 {
panic("semaphore: n < 0")
}
done := ctx.Done()
s.mu.Lock()
@@ -106,9 +109,12 @@ func (s *Weighted) Acquire(ctx context.Context, n int64) error {
}
}
// TryAcquire acquires the semaphore with a weight of n without blocking.
// TryAcquire acquires the semaphore with a non-negative weight of n without blocking.
// On success, returns true. On failure, returns false and leaves the semaphore unchanged.
func (s *Weighted) TryAcquire(n int64) bool {
if n < 0 {
panic("semaphore: n < 0")
}
s.mu.Lock()
success := s.size-s.cur >= n && s.waiters.Len() == 0
if success {
@@ -118,8 +124,11 @@ func (s *Weighted) TryAcquire(n int64) bool {
return success
}
// Release releases the semaphore with a weight of n.
// Release releases the semaphore with a non-negative weight of n.
func (s *Weighted) Release(n int64) {
if n < 0 {
panic("semaphore: n < 0")
}
s.mu.Lock()
s.cur -= n
if s.cur < 0 {

View File

@@ -121,8 +121,12 @@ func (p Properties) BoundaryAfter() bool {
//
// When all 6 bits are zero, the character is inert, meaning it is never
// influenced by normalization.
//
// We set flags to 0x80 (high bit 7 unused in quick check data) to indicate an invalid rune.
type qcInfo uint8
func (p Properties) isInvalid() bool { return p.flags == 0x80 }
func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
@@ -247,6 +251,9 @@ func (f Form) PropertiesString(s string) Properties {
// to a Properties. See the comment at the top of the file
// for more information on the format.
func compInfo(v uint16, sz int) Properties {
if sz == 0 {
return Properties{flags: 0x80, size: 1}
}
if v == 0 {
return Properties{size: uint8(sz)}
} else if v >= 0x8000 {
@@ -254,7 +261,7 @@ func compInfo(v uint16, sz int) Properties {
size: uint8(sz),
ccc: uint8(v),
tccc: uint8(v),
flags: qcInfo(v >> 8),
flags: qcInfo(v>>8) & 0x3f,
}
if p.ccc > 0 || p.combinesBackward() {
p.nLead = uint8(p.flags & 0x3)

View File

@@ -376,16 +376,12 @@ func nextComposed(i *Iter) []byte {
goto doNorm
}
prevCC = i.info.tccc
sz := int(i.info.size)
if sz == 0 {
sz = 1 // illegal rune: copy byte-by-byte
}
p := outp + sz
p := outp + int(i.info.size)
if p > len(i.buf) {
break
}
outp = p
i.p += sz
i.p += int(i.info.size)
if i.p >= i.rb.nsrc {
i.setDone()
break

View File

@@ -148,7 +148,7 @@ func (f Form) IsNormalString(s string) bool {
// patched buffer and whether the decomposition is still in progress.
func patchTail(rb *reorderBuffer) bool {
info, p := lastRuneStart(&rb.f, rb.out)
if p == -1 || info.size == 0 {
if p == -1 || info.isInvalid() {
return true
}
end := p + int(info.size)
@@ -225,7 +225,7 @@ func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
}
fd := &rb.f
if doMerge {
var info Properties
info := Properties{flags: 0x80, size: 1} // invalid rune
if p < n {
info = fd.info(src, p)
if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
@@ -235,7 +235,7 @@ func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
p = decomposeSegment(rb, p, true)
}
}
if info.size == 0 {
if info.isInvalid() {
rb.doFlush()
// Append incomplete UTF-8 encoding.
return src.appendSlice(rb.out, p, n)
@@ -314,7 +314,7 @@ func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool)
continue
}
info := f.info(src, i)
if info.size == 0 {
if info.isInvalid() {
if atEOF {
// include incomplete runes
return n, true
@@ -379,7 +379,7 @@ func (f Form) firstBoundary(src input, nsrc int) int {
// CGJ insertion points correctly. Luckily it doesn't have to.
for {
info := fd.info(src, i)
if info.size == 0 {
if info.isInvalid() {
return -1
}
if s := ss.next(info); s != ssSuccess {
@@ -424,7 +424,7 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
}
fd := formTable[f]
info := fd.info(src, 0)
if info.size == 0 {
if info.isInvalid() {
if atEOF {
return 1
}
@@ -435,7 +435,7 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
for i := int(info.size); i < nsrc; i += int(info.size) {
info = fd.info(src, i)
if info.size == 0 {
if info.isInvalid() {
if atEOF {
return i
}
@@ -465,7 +465,7 @@ func lastBoundary(fd *formInfo, b []byte) int {
if p == -1 {
return -1
}
if info.size == 0 { // ends with incomplete rune
if info.isInvalid() { // ends with incomplete rune
if p == 0 { // starts with incomplete rune
return -1
}
@@ -504,7 +504,7 @@ func lastBoundary(fd *formInfo, b []byte) int {
func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
// Force one character to be consumed.
info := rb.f.info(rb.src, sp)
if info.size == 0 {
if info.isInvalid() {
return 0
}
if s := rb.ss.next(info); s == ssStarter {
@@ -528,7 +528,7 @@ func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
break
}
info = rb.f.info(rb.src, sp)
if info.size == 0 {
if info.isInvalid() {
if !atEOF {
return int(iShortSrc)
}

28
vendor/modules.txt vendored
View File

@@ -721,6 +721,8 @@ github.com/x448/float16
# github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
## explicit; go 1.15
github.com/xrash/smetrics
# github.com/yuin/goldmark v1.8.4
## explicit; go 1.22
# go.opentelemetry.io/auto/sdk v1.2.1
## explicit; go 1.24.0
go.opentelemetry.io/auto/sdk
@@ -858,7 +860,7 @@ go.yaml.in/yaml/v2
# go.yaml.in/yaml/v3 v3.0.4
## explicit; go 1.16
go.yaml.in/yaml/v3
# golang.org/x/crypto v0.53.0
# golang.org/x/crypto v0.54.0
## explicit; go 1.25.0
golang.org/x/crypto/chacha20
golang.org/x/crypto/chacha20poly1305
@@ -869,10 +871,12 @@ golang.org/x/crypto/internal/alias
golang.org/x/crypto/internal/poly1305
golang.org/x/crypto/pkcs12
golang.org/x/crypto/pkcs12/internal/rc2
# golang.org/x/exp v0.0.0-20260611194520-c48552f49976
# golang.org/x/exp v0.0.0-20260718201538-764159d718ef
## explicit; go 1.25.0
golang.org/x/exp/constraints
# golang.org/x/net v0.56.0
# golang.org/x/mod v0.38.0
## explicit; go 1.25.0
# golang.org/x/net v0.57.0
## explicit; go 1.25.0
golang.org/x/net/http/httpguts
golang.org/x/net/http/httpproxy
@@ -898,7 +902,7 @@ golang.org/x/oauth2/google/internal/stsexchange
golang.org/x/oauth2/internal
golang.org/x/oauth2/jws
golang.org/x/oauth2/jwt
# golang.org/x/sync v0.21.0
# golang.org/x/sync v0.22.0
## explicit; go 1.25.0
golang.org/x/sync/errgroup
golang.org/x/sync/semaphore
@@ -909,10 +913,12 @@ golang.org/x/sys/plan9
golang.org/x/sys/unix
golang.org/x/sys/windows
golang.org/x/sys/windows/registry
# golang.org/x/term v0.44.0
# golang.org/x/telemetry v0.0.0-20260717140457-bdb89881bb75
## explicit; go 1.25.0
# golang.org/x/term v0.45.0
## explicit; go 1.25.0
golang.org/x/term
# golang.org/x/text v0.38.0
# golang.org/x/text v0.40.0
## explicit; go 1.25.0
golang.org/x/text/secure/bidirule
golang.org/x/text/transform
@@ -921,6 +927,16 @@ golang.org/x/text/unicode/norm
# golang.org/x/time v0.15.0
## explicit; go 1.25.0
golang.org/x/time/rate
# golang.org/x/tools v0.48.0
## explicit; go 1.25.0
# golang.org/x/tools/go/expect v0.1.1-deprecated
## explicit; go 1.23.0
# golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated
## explicit; go 1.23.0
# golang.org/x/tools/godoc v0.1.0-deprecated
## explicit; go 1.24.0
# golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
## explicit; go 1.18
# google.golang.org/api v0.284.0
## explicit; go 1.25.8
google.golang.org/api/googleapi