mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-07-03 15:45:20 +03:00
Compare commits
2 Commits
master
...
issue-1118
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
64d82113f0 | ||
|
|
cf64fea376 |
@@ -118,10 +118,9 @@ type AccessLogFilters struct {
|
||||
}
|
||||
|
||||
func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
|
||||
if ui == nil || ui.AccessLog == nil {
|
||||
if ui.AccessLog == nil {
|
||||
return
|
||||
}
|
||||
|
||||
filters := ui.AccessLog.Filters
|
||||
if filters != nil && len(filters.SkipStatusCodes) > 0 {
|
||||
if slices.Contains(filters.SkipStatusCodes, statusCode) {
|
||||
@@ -135,17 +134,6 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
|
||||
r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
|
||||
}
|
||||
|
||||
// hasAnyURLs reports whether ui has at least one backend URL route configured.
|
||||
// It is used only for unauthorized_user config section, since other users
|
||||
// must always have either URLPrefix or URLMaps set.
|
||||
func (ui *UserInfo) hasAnyURLs() bool {
|
||||
if ui == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
|
||||
}
|
||||
|
||||
// HeadersConf represents config for request and response headers.
|
||||
type HeadersConf struct {
|
||||
RequestHeaders []*Header `yaml:"headers,omitempty"`
|
||||
@@ -995,11 +983,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
|
||||
if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ui.hasAnyURLs() {
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ui.initURLs(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metricLabels, err := ui.getMetricLabels()
|
||||
|
||||
@@ -175,12 +175,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
if len(ats) == 0 {
|
||||
// Process requests for unauthorized users
|
||||
ui := authConfig.Load().UnauthorizedUser
|
||||
if ui.hasAnyURLs() {
|
||||
if ui != nil {
|
||||
processUserRequest(w, r, ui, nil)
|
||||
return true
|
||||
}
|
||||
|
||||
ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
|
||||
handleMissingAuthorizationError(w)
|
||||
return true
|
||||
}
|
||||
@@ -193,24 +192,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
if tkn == nil {
|
||||
logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
|
||||
}
|
||||
defer putToken(tkn)
|
||||
// Call processUserRequest only if the token contains the vm_access claim
|
||||
// or a default claim is configured; otherwise fall through to unauthorized_user.
|
||||
if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
|
||||
processUserRequest(w, r, ui, tkn)
|
||||
if !tkn.HasVMAccessClaim() && ui.JWT.DefaultVMAccessClaim == nil {
|
||||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||||
return true
|
||||
}
|
||||
defer putToken(tkn)
|
||||
processUserRequest(w, r, ui, tkn)
|
||||
return true
|
||||
}
|
||||
|
||||
uu := authConfig.Load().UnauthorizedUser
|
||||
if uu.hasAnyURLs() {
|
||||
if uu != nil {
|
||||
processUserRequest(w, r, uu, nil)
|
||||
return true
|
||||
}
|
||||
|
||||
invalidAuthTokenRequests.Inc()
|
||||
slowdownUnauthorizedResponse(r)
|
||||
uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
|
||||
if *logInvalidAuthTokens {
|
||||
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
|
||||
@@ -785,26 +785,7 @@ statusCode=401
|
||||
Unauthorized`
|
||||
f(simpleCfgStr, request, responseExpected)
|
||||
|
||||
// token without vm_access claim should fall through to unauthorized_user
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
|
||||
responseExpected = `
|
||||
statusCode=200
|
||||
path: /bar/abc
|
||||
query:
|
||||
headers:`
|
||||
f(fmt.Sprintf(`
|
||||
unauthorized_user:
|
||||
url_prefix: {BACKEND}/bar
|
||||
users:
|
||||
- jwt:
|
||||
public_keys:
|
||||
- %q
|
||||
match_claims:
|
||||
role: admin
|
||||
url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
|
||||
|
||||
// token without vm_access claim is accepted when default_vm_access_claim configured
|
||||
// token without vm_access claim is accepted when it
|
||||
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
|
||||
request.Header.Set(`Authorization`, `Bearer `+roleToken)
|
||||
responseExpected = `
|
||||
|
||||
@@ -471,54 +471,41 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
|
||||
}
|
||||
|
||||
func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
|
||||
if canPushdownCommonFilters(be) {
|
||||
// Execute binary operation in the following way:
|
||||
//
|
||||
// 1) execute the exprFirst
|
||||
// 2) get common label filters for series returned at step 1
|
||||
// 3) push down the found common label filters to exprSecond. This filters out unneeded series
|
||||
// during exprSecond execution instead of spending compute resources on extracting and processing these series
|
||||
// before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
|
||||
// 4) execute the exprSecond with possible additional filters found at step 3
|
||||
//
|
||||
// Typical use cases:
|
||||
// - Kubernetes-related: show pod creation time with the node name:
|
||||
//
|
||||
// kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
|
||||
//
|
||||
// Without the optimization `kube_pod_info` would select and spend compute resources
|
||||
// for more time series than needed. The selected time series would be dropped later
|
||||
// when matching time series on the right and left sides of binary operand.
|
||||
//
|
||||
// - Generic alerting queries, which rely on `info` metrics.
|
||||
// See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
|
||||
//
|
||||
// - Queries, which get additional labels from `info` metrics.
|
||||
// See https://www.robustperception.io/exposing-the-software-version-to-prometheus
|
||||
tssFirst, err := evalExpr(qt, ec, exprFirst)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
canPushdown := canPushdownCommonFilters(be)
|
||||
if !canPushdown && !shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
|
||||
// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
|
||||
// from exprFirst to exprSecond.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
|
||||
qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
|
||||
defer qt.Done()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
var tssFirst []*timeseries
|
||||
var errFirst error
|
||||
qtFirst := qt.NewChild("expr1")
|
||||
wg.Go(func() {
|
||||
tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
|
||||
qtFirst.Done()
|
||||
})
|
||||
|
||||
var tssSecond []*timeseries
|
||||
var errSecond error
|
||||
qtSecond := qt.NewChild("expr2")
|
||||
wg.Go(func() {
|
||||
tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
|
||||
qtSecond.Done()
|
||||
})
|
||||
|
||||
wg.Wait()
|
||||
if errFirst != nil {
|
||||
return nil, nil, errFirst
|
||||
}
|
||||
if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
|
||||
// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
|
||||
// since the "exprFirst op exprSecond" would return an empty result in any case.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
|
||||
return nil, nil, nil
|
||||
}
|
||||
lfs := getCommonLabelFilters(tssFirst)
|
||||
lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
|
||||
exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
|
||||
tssSecond, err := evalExpr(qt, ec, exprSecond)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
if errSecond != nil {
|
||||
return nil, nil, errSecond
|
||||
}
|
||||
return tssFirst, tssSecond, nil
|
||||
}
|
||||
|
||||
// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
|
||||
// in exprFirst and exprSecond.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
|
||||
if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
|
||||
if !canPushdown {
|
||||
qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
|
||||
defer qt.Done()
|
||||
|
||||
@@ -538,35 +525,45 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
|
||||
return tssFirst, tssSecond, nil
|
||||
}
|
||||
|
||||
// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
|
||||
// from exprFirst to exprSecond.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
|
||||
qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
|
||||
defer qt.Done()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
var tssFirst []*timeseries
|
||||
var errFirst error
|
||||
qtFirst := qt.NewChild("expr1")
|
||||
wg.Go(func() {
|
||||
tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
|
||||
qtFirst.Done()
|
||||
})
|
||||
|
||||
var tssSecond []*timeseries
|
||||
var errSecond error
|
||||
qtSecond := qt.NewChild("expr2")
|
||||
wg.Go(func() {
|
||||
tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
|
||||
qtSecond.Done()
|
||||
})
|
||||
|
||||
wg.Wait()
|
||||
if errFirst != nil {
|
||||
return nil, nil, errFirst
|
||||
// Execute binary operation in the following way:
|
||||
//
|
||||
// 1) execute the exprFirst
|
||||
// 2) get common label filters for series returned at step 1
|
||||
// 3) push down the found common label filters to exprSecond. This filters out unneeded series
|
||||
// during exprSecond execution instead of spending compute resources on extracting and processing these series
|
||||
// before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
|
||||
// 4) execute the exprSecond with possible additional filters found at step 3
|
||||
//
|
||||
// Typical use cases:
|
||||
// - Kubernetes-related: show pod creation time with the node name:
|
||||
//
|
||||
// kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
|
||||
//
|
||||
// Without the optimization `kube_pod_info` would select and spend compute resources
|
||||
// for more time series than needed. The selected time series would be dropped later
|
||||
// when matching time series on the right and left sides of binary operand.
|
||||
//
|
||||
// - Generic alerting queries, which rely on `info` metrics.
|
||||
// See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
|
||||
//
|
||||
// - Queries, which get additional labels from `info` metrics.
|
||||
// See https://www.robustperception.io/exposing-the-software-version-to-prometheus
|
||||
tssFirst, err := evalExpr(qt, ec, exprFirst)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if errSecond != nil {
|
||||
return nil, nil, errSecond
|
||||
if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
|
||||
// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
|
||||
// since the "exprFirst op exprSecond" would return an empty result in any case.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
|
||||
return nil, nil, nil
|
||||
}
|
||||
lfs := getCommonLabelFilters(tssFirst)
|
||||
lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
|
||||
exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
|
||||
tssSecond, err := evalExpr(qt, ec, exprSecond)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return tssFirst, tssSecond, nil
|
||||
}
|
||||
|
||||
@@ -4833,137 +4833,13 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
// buckets that are consecutively empty at left and right ends will not be preserved.
|
||||
t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
|
||||
t.Run(`buckets_limit(zero)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(buckets_limit(3, (
|
||||
alias(label_set(36, "le", "+Inf"), "metric"),
|
||||
alias(label_set(36, "le", "25"), "metric"),
|
||||
alias(label_set(36, "le", "21"), "metric"),
|
||||
alias(label_set(36, "le", "19"), "metric"),
|
||||
alias(label_set(36, "le", "18"), "metric"),
|
||||
alias(label_set(36, "le", "17"), "metric"),
|
||||
alias(label_set(36, "le", "16"), "metric"),
|
||||
alias(label_set(27, "le", "12"), "metric"),
|
||||
alias(label_set(14, "le", "9"), "metric"),
|
||||
alias(label_set(0, "le", "6"), "metric"),
|
||||
alias(label_set(0, "le", "1"), "metric"),
|
||||
)))`
|
||||
r1 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{14, 14, 14, 14, 14, 14},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r1.MetricName.MetricGroup = []byte("metric")
|
||||
r1.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("9"),
|
||||
},
|
||||
}
|
||||
r2 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{27, 27, 27, 27, 27, 27},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r2.MetricName.MetricGroup = []byte("metric")
|
||||
r2.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("12"),
|
||||
},
|
||||
}
|
||||
r3 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{36, 36, 36, 36, 36, 36},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r3.MetricName.MetricGroup = []byte("metric")
|
||||
r3.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("16"),
|
||||
},
|
||||
}
|
||||
resultExpected := []netstorage.Result{r1, r2, r3}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
|
||||
// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
|
||||
t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort(buckets_limit(5, (
|
||||
alias(label_set(36, "le", "18"), "metric"),
|
||||
alias(label_set(36, "le", "17"), "metric"),
|
||||
alias(label_set(36, "le", "16"), "metric"),
|
||||
alias(label_set(27, "le", "12"), "metric"),
|
||||
alias(label_set(14, "le", "9"), "metric"),
|
||||
alias(label_set(0, "le", "6"), "metric"),
|
||||
alias(label_set(0, "le", "1"), "metric"),
|
||||
)))`
|
||||
|
||||
r1 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 0, 0, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r1.MetricName.MetricGroup = []byte("metric")
|
||||
r1.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("1"),
|
||||
},
|
||||
}
|
||||
r2 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 0, 0, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r2.MetricName.MetricGroup = []byte("metric")
|
||||
r2.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("6"),
|
||||
},
|
||||
}
|
||||
r3 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{14, 14, 14, 14, 14, 14},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r3.MetricName.MetricGroup = []byte("metric")
|
||||
r3.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("9"),
|
||||
},
|
||||
}
|
||||
r4 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{27, 27, 27, 27, 27, 27},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r4.MetricName.MetricGroup = []byte("metric")
|
||||
r4.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("12"),
|
||||
},
|
||||
}
|
||||
r5 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{36, 36, 36, 36, 36, 36},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r5.MetricName.MetricGroup = []byte("metric")
|
||||
r5.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("16"),
|
||||
},
|
||||
}
|
||||
|
||||
resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
|
||||
q := `buckets_limit(0, (
|
||||
alias(label_set(100, "le", "inf", "x", "y"), "metric"),
|
||||
alias(label_set(50, "le", "120", "x", "y"), "metric"),
|
||||
))`
|
||||
resultExpected := []netstorage.Result{}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`buckets_limit(unused)`, func(t *testing.T) {
|
||||
@@ -6352,6 +6228,50 @@ func TestExecSuccess(t *testing.T) {
|
||||
resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sort_by_label(
|
||||
buckets_limit(
|
||||
3,
|
||||
sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
|
||||
), "le"
|
||||
)`
|
||||
r1 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{40, 40, 40, 40, 40, 40},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r1.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("+Inf"),
|
||||
},
|
||||
}
|
||||
r2 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{0, 0, 0, 0, 0, 0},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r2.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("1.000e+00"),
|
||||
},
|
||||
}
|
||||
r3 := netstorage.Result{
|
||||
MetricName: metricNameExpected,
|
||||
Values: []float64{40, 40, 40, 40, 40, 40},
|
||||
Timestamps: timestampsExpected,
|
||||
}
|
||||
r3.MetricName.Tags = []storage.Tag{
|
||||
{
|
||||
Key: []byte("le"),
|
||||
Value: []byte("2.448e+00"),
|
||||
},
|
||||
}
|
||||
resultExpected := []netstorage.Result{r1, r2, r3}
|
||||
f(q, resultExpected)
|
||||
})
|
||||
t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`
|
||||
|
||||
@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 {
|
||||
return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
|
||||
return nil, nil
|
||||
}
|
||||
if limit < 3 {
|
||||
// Preserve the first and the last bucket for better accuracy for min and max values.
|
||||
@@ -461,23 +461,6 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
|
||||
prevValue = value
|
||||
}
|
||||
}
|
||||
|
||||
// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
|
||||
epsilon := 1e-9
|
||||
isEmptyBucket := func(hits float64) bool {
|
||||
return !math.IsNaN(hits) && math.Abs(hits) < epsilon
|
||||
}
|
||||
l := 0
|
||||
r := len(leGroup) - 1
|
||||
for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
|
||||
r--
|
||||
}
|
||||
for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
|
||||
l++
|
||||
}
|
||||
leGroup = leGroup[l : r+1]
|
||||
|
||||
for len(leGroup) > limit {
|
||||
// Preserve the first and the last bucket for better accuracy for min and max values
|
||||
xxMinIdx := 1
|
||||
|
||||
@@ -1229,7 +1229,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
|
||||
`buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
|
||||
of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`.
|
||||
|
||||
The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
|
||||
The result will preserve the first and the last bucket to improve accuracy for min and max values.
|
||||
So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
|
||||
|
||||
See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
|
||||
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).
|
||||
@@ -35,10 +36,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
|
||||
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
|
||||
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
|
||||
|
||||
* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
|
||||
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
|
||||
@@ -46,7 +44,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
|
||||
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
|
||||
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly URL-encode `-vm-extra-label` values when building import requests, so special characters such as `&` don't get split into broken query parameters. See [#11144](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11144). Thanks to @immanuwell for contribution.
|
||||
* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/): ignore `enable.auto.offset.store` option in `kafka.consumer.topic.options`, since `vmagent` manages offset storage internally. Previously, setting this option could cause `vmagent` to stop committing Kafka messages. See [#11208](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11208).
|
||||
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
|
||||
|
||||
## [v1.146.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.146.0)
|
||||
|
||||
|
||||
@@ -1923,6 +1923,9 @@ scrape_configs:
|
||||
# Example values:
|
||||
# - "10MiB" - 10 * 1024 * 1024 bytes
|
||||
# - "100MB" - 100 * 1000 * 1000 bytes
|
||||
# The max_scrape_size can be set on a per-target basis by specifying `__max_scrape_size__`
|
||||
# label during target relabeling phase.
|
||||
# See https://docs.victoriametrics.com/victoriametrics/relabeling/
|
||||
#
|
||||
# max_scrape_size: <size>
|
||||
|
||||
|
||||
@@ -1323,17 +1323,9 @@ unauthorized_user:
|
||||
|
||||
vmauth allows configuring access logs {{% available_from "v1.138.0" %}} printing per-user:
|
||||
```yaml
|
||||
users:
|
||||
- username: foo
|
||||
password: bar
|
||||
url_prefix: 'http://localhost:8428/'
|
||||
# Log all requests to this user
|
||||
access_log: {}
|
||||
```
|
||||
|
||||
If you want to log requests with missing or invalid auth tokens, use unauthorized_user without configuring any URL routes{{% available_from "#" %}}:
|
||||
```yaml
|
||||
unauthorized_user:
|
||||
url_prefix: 'http://localhost:8428/'
|
||||
# Log all requests to this user
|
||||
access_log: {}
|
||||
```
|
||||
|
||||
|
||||
@@ -1271,6 +1271,17 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
|
||||
}
|
||||
scrapeTimeout = d
|
||||
}
|
||||
// Read max_scrape_size option from __max_scrape_size__ label.
|
||||
targetMaxScrapeSize := swc.maxScrapeSize
|
||||
if s := labels.Get("__max_scrape_size__"); len(s) > 0 {
|
||||
n, err := flagutil.ParseBytes(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse __max_scrape_size__=%q: %w", s, err)
|
||||
}
|
||||
if n > 0 {
|
||||
targetMaxScrapeSize = n
|
||||
}
|
||||
}
|
||||
// Read series_limit option from __series_limit__ label.
|
||||
// See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter
|
||||
seriesLimit := swc.seriesLimit
|
||||
@@ -1333,7 +1344,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
|
||||
ScrapeURL: scrapeURL,
|
||||
ScrapeInterval: scrapeInterval,
|
||||
ScrapeTimeout: scrapeTimeout,
|
||||
MaxScrapeSize: swc.maxScrapeSize,
|
||||
MaxScrapeSize: targetMaxScrapeSize,
|
||||
HonorLabels: swc.honorLabels,
|
||||
HonorTimestamps: swc.honorTimestamps,
|
||||
DenyRedirects: swc.denyRedirects,
|
||||
|
||||
@@ -1153,6 +1153,57 @@ scrape_configs:
|
||||
})
|
||||
f(`
|
||||
scrape_configs:
|
||||
- job_name: foo
|
||||
max_scrape_size: 8MiB
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: foo1:.*
|
||||
target_label: __max_scrape_size__
|
||||
replacement: 2.5MiB
|
||||
- source_labels: [__address__]
|
||||
regex: foo2:.*
|
||||
target_label: __max_scrape_size__
|
||||
replacement: -1
|
||||
static_configs:
|
||||
- targets: ["foo1:1234", "foo2:1234", "foo3:1234"]
|
||||
`, []*ScrapeWork{
|
||||
{
|
||||
ScrapeURL: "http://foo1:1234/metrics",
|
||||
ScrapeInterval: defaultScrapeInterval,
|
||||
ScrapeTimeout: defaultScrapeTimeout,
|
||||
MaxScrapeSize: 2.5 * 1024 * 1024,
|
||||
Labels: promutil.NewLabelsFromMap(map[string]string{
|
||||
"instance": "foo1:1234",
|
||||
"job": "foo",
|
||||
}),
|
||||
jobNameOriginal: "foo",
|
||||
},
|
||||
// invalid __max_scrape_size__ will be ignored
|
||||
{
|
||||
ScrapeURL: "http://foo2:1234/metrics",
|
||||
ScrapeInterval: defaultScrapeInterval,
|
||||
ScrapeTimeout: defaultScrapeTimeout,
|
||||
MaxScrapeSize: 8 * 1024 * 1024,
|
||||
Labels: promutil.NewLabelsFromMap(map[string]string{
|
||||
"instance": "foo2:1234",
|
||||
"job": "foo",
|
||||
}),
|
||||
jobNameOriginal: "foo",
|
||||
},
|
||||
{
|
||||
ScrapeURL: "http://foo3:1234/metrics",
|
||||
ScrapeInterval: defaultScrapeInterval,
|
||||
ScrapeTimeout: defaultScrapeTimeout,
|
||||
MaxScrapeSize: 8 * 1024 * 1024,
|
||||
Labels: promutil.NewLabelsFromMap(map[string]string{
|
||||
"instance": "foo3:1234",
|
||||
"job": "foo",
|
||||
}),
|
||||
jobNameOriginal: "foo",
|
||||
},
|
||||
})
|
||||
f(`
|
||||
scrape_configs:
|
||||
- job_name: foo
|
||||
static_configs:
|
||||
- targets: ["foo.bar:1234"]
|
||||
|
||||
Reference in New Issue
Block a user