Compare commits

..

2 Commits

Author SHA1 Message Date
Jayice
64d82113f0 ignore non-positive __max_scrape_size__ 2026-07-03 16:49:08 +08:00
Jayice
cf64fea376 support __max_scrape_size__ in target 2026-07-03 15:06:45 +08:00
13 changed files with 205 additions and 284 deletions

View File

@@ -118,10 +118,9 @@ type AccessLogFilters struct {
}
func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
if ui == nil || ui.AccessLog == nil {
if ui.AccessLog == nil {
return
}
filters := ui.AccessLog.Filters
if filters != nil && len(filters.SkipStatusCodes) > 0 {
if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -135,17 +134,6 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
}
// hasAnyURLs reports whether ui has at least one backend URL route configured.
// It is used only for unauthorized_user config section, since other users
// must always have either URLPrefix or URLMaps set.
func (ui *UserInfo) hasAnyURLs() bool {
if ui == nil {
return false
}
return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
}
// HeadersConf represents config for request and response headers.
type HeadersConf struct {
RequestHeaders []*Header `yaml:"headers,omitempty"`
@@ -995,11 +983,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
return nil, err
}
if ui.hasAnyURLs() {
if err := ui.initURLs(); err != nil {
return nil, err
}
if err := ui.initURLs(); err != nil {
return nil, err
}
metricLabels, err := ui.getMetricLabels()

View File

@@ -175,12 +175,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
if len(ats) == 0 {
// Process requests for unauthorized users
ui := authConfig.Load().UnauthorizedUser
if ui.hasAnyURLs() {
if ui != nil {
processUserRequest(w, r, ui, nil)
return true
}
ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
handleMissingAuthorizationError(w)
return true
}
@@ -193,24 +192,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
if tkn == nil {
logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
}
defer putToken(tkn)
// Call processUserRequest only if the token contains the vm_access claim
// or a default claim is configured; otherwise fall through to unauthorized_user.
if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
processUserRequest(w, r, ui, tkn)
if !tkn.HasVMAccessClaim() && ui.JWT.DefaultVMAccessClaim == nil {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return true
}
defer putToken(tkn)
processUserRequest(w, r, ui, tkn)
return true
}
uu := authConfig.Load().UnauthorizedUser
if uu.hasAnyURLs() {
if uu != nil {
processUserRequest(w, r, uu, nil)
return true
}
invalidAuthTokenRequests.Inc()
slowdownUnauthorizedResponse(r)
uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
if *logInvalidAuthTokens {
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
err = &httpserver.ErrorWithStatusCode{

View File

@@ -785,26 +785,7 @@ statusCode=401
Unauthorized`
f(simpleCfgStr, request, responseExpected)
// token without vm_access claim should fall through to unauthorized_user
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
responseExpected = `
statusCode=200
path: /bar/abc
query:
headers:`
f(fmt.Sprintf(`
unauthorized_user:
url_prefix: {BACKEND}/bar
users:
- jwt:
public_keys:
- %q
match_claims:
role: admin
url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
// token without vm_access claim is accepted when default_vm_access_claim configured
// token without vm_access claim is accepted when it
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
request.Header.Set(`Authorization`, `Bearer `+roleToken)
responseExpected = `

View File

@@ -471,54 +471,41 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
}
func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
if canPushdownCommonFilters(be) {
// Execute binary operation in the following way:
//
// 1) execute the exprFirst
// 2) get common label filters for series returned at step 1
// 3) push down the found common label filters to exprSecond. This filters out unneeded series
// during exprSecond execution instead of spending compute resources on extracting and processing these series
// before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
// 4) execute the exprSecond with possible additional filters found at step 3
//
// Typical use cases:
// - Kubernetes-related: show pod creation time with the node name:
//
// kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
//
// Without the optimization `kube_pod_info` would select and spend compute resources
// for more time series than needed. The selected time series would be dropped later
// when matching time series on the right and left sides of binary operand.
//
// - Generic alerting queries, which rely on `info` metrics.
// See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
//
// - Queries, which get additional labels from `info` metrics.
// See https://www.robustperception.io/exposing-the-software-version-to-prometheus
tssFirst, err := evalExpr(qt, ec, exprFirst)
if err != nil {
return nil, nil, err
canPushdown := canPushdownCommonFilters(be)
if !canPushdown && !shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
// from exprFirst to exprSecond.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
defer qt.Done()
var wg sync.WaitGroup
var tssFirst []*timeseries
var errFirst error
qtFirst := qt.NewChild("expr1")
wg.Go(func() {
tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
qtFirst.Done()
})
var tssSecond []*timeseries
var errSecond error
qtSecond := qt.NewChild("expr2")
wg.Go(func() {
tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
qtSecond.Done()
})
wg.Wait()
if errFirst != nil {
return nil, nil, errFirst
}
if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
// since the "exprFirst op exprSecond" would return an empty result in any case.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
return nil, nil, nil
}
lfs := getCommonLabelFilters(tssFirst)
lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
tssSecond, err := evalExpr(qt, ec, exprSecond)
if err != nil {
return nil, nil, err
if errSecond != nil {
return nil, nil, errSecond
}
return tssFirst, tssSecond, nil
}
// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
// in exprFirst and exprSecond.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
if !canPushdown {
qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
defer qt.Done()
@@ -538,35 +525,45 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
return tssFirst, tssSecond, nil
}
// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
// from exprFirst to exprSecond.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
defer qt.Done()
var wg sync.WaitGroup
var tssFirst []*timeseries
var errFirst error
qtFirst := qt.NewChild("expr1")
wg.Go(func() {
tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
qtFirst.Done()
})
var tssSecond []*timeseries
var errSecond error
qtSecond := qt.NewChild("expr2")
wg.Go(func() {
tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
qtSecond.Done()
})
wg.Wait()
if errFirst != nil {
return nil, nil, errFirst
// Execute binary operation in the following way:
//
// 1) execute the exprFirst
// 2) get common label filters for series returned at step 1
// 3) push down the found common label filters to exprSecond. This filters out unneeded series
// during exprSecond execution instead of spending compute resources on extracting and processing these series
// before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
// 4) execute the exprSecond with possible additional filters found at step 3
//
// Typical use cases:
// - Kubernetes-related: show pod creation time with the node name:
//
// kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
//
// Without the optimization `kube_pod_info` would select and spend compute resources
// for more time series than needed. The selected time series would be dropped later
// when matching time series on the right and left sides of binary operand.
//
// - Generic alerting queries, which rely on `info` metrics.
// See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
//
// - Queries, which get additional labels from `info` metrics.
// See https://www.robustperception.io/exposing-the-software-version-to-prometheus
tssFirst, err := evalExpr(qt, ec, exprFirst)
if err != nil {
return nil, nil, err
}
if errSecond != nil {
return nil, nil, errSecond
if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
// since the "exprFirst op exprSecond" would return an empty result in any case.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
return nil, nil, nil
}
lfs := getCommonLabelFilters(tssFirst)
lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
tssSecond, err := evalExpr(qt, ec, exprSecond)
if err != nil {
return nil, nil, err
}
return tssFirst, tssSecond, nil
}

View File

@@ -4833,137 +4833,13 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
// buckets that are consecutively empty at left and right ends will not be preserved.
t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
t.Run(`buckets_limit(zero)`, func(t *testing.T) {
t.Parallel()
q := `sort(buckets_limit(3, (
alias(label_set(36, "le", "+Inf"), "metric"),
alias(label_set(36, "le", "25"), "metric"),
alias(label_set(36, "le", "21"), "metric"),
alias(label_set(36, "le", "19"), "metric"),
alias(label_set(36, "le", "18"), "metric"),
alias(label_set(36, "le", "17"), "metric"),
alias(label_set(36, "le", "16"), "metric"),
alias(label_set(27, "le", "12"), "metric"),
alias(label_set(14, "le", "9"), "metric"),
alias(label_set(0, "le", "6"), "metric"),
alias(label_set(0, "le", "1"), "metric"),
)))`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{14, 14, 14, 14, 14, 14},
Timestamps: timestampsExpected,
}
r1.MetricName.MetricGroup = []byte("metric")
r1.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("9"),
},
}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{27, 27, 27, 27, 27, 27},
Timestamps: timestampsExpected,
}
r2.MetricName.MetricGroup = []byte("metric")
r2.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("12"),
},
}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{36, 36, 36, 36, 36, 36},
Timestamps: timestampsExpected,
}
r3.MetricName.MetricGroup = []byte("metric")
r3.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("16"),
},
}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
t.Parallel()
q := `sort(buckets_limit(5, (
alias(label_set(36, "le", "18"), "metric"),
alias(label_set(36, "le", "17"), "metric"),
alias(label_set(36, "le", "16"), "metric"),
alias(label_set(27, "le", "12"), "metric"),
alias(label_set(14, "le", "9"), "metric"),
alias(label_set(0, "le", "6"), "metric"),
alias(label_set(0, "le", "1"), "metric"),
)))`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0, 0, 0, 0, 0, 0},
Timestamps: timestampsExpected,
}
r1.MetricName.MetricGroup = []byte("metric")
r1.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("1"),
},
}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0, 0, 0, 0, 0, 0},
Timestamps: timestampsExpected,
}
r2.MetricName.MetricGroup = []byte("metric")
r2.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("6"),
},
}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{14, 14, 14, 14, 14, 14},
Timestamps: timestampsExpected,
}
r3.MetricName.MetricGroup = []byte("metric")
r3.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("9"),
},
}
r4 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{27, 27, 27, 27, 27, 27},
Timestamps: timestampsExpected,
}
r4.MetricName.MetricGroup = []byte("metric")
r4.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("12"),
},
}
r5 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{36, 36, 36, 36, 36, 36},
Timestamps: timestampsExpected,
}
r5.MetricName.MetricGroup = []byte("metric")
r5.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("16"),
},
}
resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
q := `buckets_limit(0, (
alias(label_set(100, "le", "inf", "x", "y"), "metric"),
alias(label_set(50, "le", "120", "x", "y"), "metric"),
))`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`buckets_limit(unused)`, func(t *testing.T) {
@@ -6352,6 +6228,50 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
f(q, resultExpected)
})
t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label(
buckets_limit(
3,
sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
), "le"
)`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{40, 40, 40, 40, 40, 40},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("+Inf"),
},
}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0, 0, 0, 0, 0, 0},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("1.000e+00"),
},
}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{40, 40, 40, 40, 40, 40},
Timestamps: timestampsExpected,
}
r3.MetricName.Tags = []storage.Tag{
{
Key: []byte("le"),
Value: []byte("2.448e+00"),
},
}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
t.Parallel()
q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`

View File

@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
return nil, err
}
if limit <= 0 {
return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
return nil, nil
}
if limit < 3 {
// Preserve the first and the last bucket for better accuracy for min and max values.
@@ -461,23 +461,6 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
prevValue = value
}
}
// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
epsilon := 1e-9
isEmptyBucket := func(hits float64) bool {
return !math.IsNaN(hits) && math.Abs(hits) < epsilon
}
l := 0
r := len(leGroup) - 1
for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
r--
}
for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
l++
}
leGroup = leGroup[l : r+1]
for len(leGroup) > limit {
// Preserve the first and the last bucket for better accuracy for min and max values
xxMinIdx := 1

View File

View File

@@ -1229,7 +1229,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
`buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`.
The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
The result will preserve the first and the last bucket to improve accuracy for min and max values.
So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).

View File

@@ -26,6 +26,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).
@@ -35,10 +36,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
@@ -46,7 +44,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly URL-encode `-vm-extra-label` values when building import requests, so special characters such as `&` don't get split into broken query parameters. See [#11144](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11144). Thanks to @immanuwell for contribution.
* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/): ignore `enable.auto.offset.store` option in `kafka.consumer.topic.options`, since `vmagent` manages offset storage internally. Previously, setting this option could cause `vmagent` to stop committing Kafka messages. See [#11208](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11208).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
## [v1.146.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.146.0)

View File

@@ -1923,6 +1923,9 @@ scrape_configs:
# Example values:
# - "10MiB" - 10 * 1024 * 1024 bytes
# - "100MB" - 100 * 1000 * 1000 bytes
# The max_scrape_size can be set on a per-target basis by specifying `__max_scrape_size__`
# label during target relabeling phase.
# See https://docs.victoriametrics.com/victoriametrics/relabeling/
#
# max_scrape_size: <size>

View File

@@ -1323,17 +1323,9 @@ unauthorized_user:
vmauth allows configuring access logs {{% available_from "v1.138.0" %}} printing per-user:
```yaml
users:
- username: foo
password: bar
url_prefix: 'http://localhost:8428/'
# Log all requests to this user
access_log: {}
```
If you want to log requests with missing or invalid auth tokens, use unauthorized_user without configuring any URL routes{{% available_from "#" %}}:
```yaml
unauthorized_user:
url_prefix: 'http://localhost:8428/'
# Log all requests to this user
access_log: {}
```

View File

@@ -1271,6 +1271,17 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
}
scrapeTimeout = d
}
// Read max_scrape_size option from __max_scrape_size__ label.
targetMaxScrapeSize := swc.maxScrapeSize
if s := labels.Get("__max_scrape_size__"); len(s) > 0 {
n, err := flagutil.ParseBytes(s)
if err != nil {
return nil, fmt.Errorf("cannot parse __max_scrape_size__=%q: %w", s, err)
}
if n > 0 {
targetMaxScrapeSize = n
}
}
// Read series_limit option from __series_limit__ label.
// See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
seriesLimit := swc.seriesLimit
@@ -1333,7 +1344,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
ScrapeURL: scrapeURL,
ScrapeInterval: scrapeInterval,
ScrapeTimeout: scrapeTimeout,
MaxScrapeSize: swc.maxScrapeSize,
MaxScrapeSize: targetMaxScrapeSize,
HonorLabels: swc.honorLabels,
HonorTimestamps: swc.honorTimestamps,
DenyRedirects: swc.denyRedirects,

View File

@@ -1153,6 +1153,57 @@ scrape_configs:
})
f(`
scrape_configs:
- job_name: foo
max_scrape_size: 8MiB
relabel_configs:
- source_labels: [__address__]
regex: foo1:.*
target_label: __max_scrape_size__
replacement: 2.5MiB
- source_labels: [__address__]
regex: foo2:.*
target_label: __max_scrape_size__
replacement: -1
static_configs:
- targets: ["foo1:1234", "foo2:1234", "foo3:1234"]
`, []*ScrapeWork{
{
ScrapeURL: "http://foo1:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
MaxScrapeSize: 2.5 * 1024 * 1024,
Labels: promutil.NewLabelsFromMap(map[string]string{
"instance": "foo1:1234",
"job": "foo",
}),
jobNameOriginal: "foo",
},
// invalid __max_scrape_size__ will be ignored
{
ScrapeURL: "http://foo2:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
MaxScrapeSize: 8 * 1024 * 1024,
Labels: promutil.NewLabelsFromMap(map[string]string{
"instance": "foo2:1234",
"job": "foo",
}),
jobNameOriginal: "foo",
},
{
ScrapeURL: "http://foo3:1234/metrics",
ScrapeInterval: defaultScrapeInterval,
ScrapeTimeout: defaultScrapeTimeout,
MaxScrapeSize: 8 * 1024 * 1024,
Labels: promutil.NewLabelsFromMap(map[string]string{
"instance": "foo3:1234",
"job": "foo",
}),
jobNameOriginal: "foo",
},
})
f(`
scrape_configs:
- job_name: foo
static_configs:
- targets: ["foo.bar:1234"]