metricsql: add fill modifiers support

.golangci.yml

@@ -23,4 +23,3 @@ linters:
         text: 'SA(4003|1019|5011):'
     paths:
       - ^app/vmui/
-      - app/vmui/packages/vmui/node_modules/

app/vmalert/rule/group.go

@@ -97,7 +97,6 @@ type groupMetrics struct {
 	iterationMissed   *metrics.Counter
 	iterationReset    *metrics.Counter
 	iterationInterval *metrics.Gauge
-	iterationLimit    *metrics.Gauge
 }
 
 // merges group rule labels into result map
@@ -337,12 +336,6 @@ func (g *Group) Init() {
 		i := g.Interval.Seconds()
 		return i
 	})
-	g.metrics.iterationLimit = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_rule_group_results_limit{%s}`, labels), func() float64 {
-		g.mu.RLock()
-		limit := g.Limit
-		g.mu.RUnlock()
-		return float64(limit)
-	})
 	for i := range g.Rules {
 		g.Rules[i].registerMetrics(g.metrics.set)
 	}

app/vmauth/auth_config.go

@@ -118,10 +118,9 @@ type AccessLogFilters struct {
 }
 
 func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
-	if ui == nil || ui.AccessLog == nil {
+	if ui.AccessLog == nil {
 		return
 	}
-
 	filters := ui.AccessLog.Filters
 	if filters != nil && len(filters.SkipStatusCodes) > 0 {
 		if slices.Contains(filters.SkipStatusCodes, statusCode) {
@@ -135,17 +134,6 @@ func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int,
 		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
 }
 
-// hasAnyURLs reports whether ui has at least one backend URL route configured.
-// It is used only for unauthorized_user config section, since other users
-// must always have either URLPrefix or URLMaps set.
-func (ui *UserInfo) hasAnyURLs() bool {
-	if ui == nil {
-		return false
-	}
-
-	return ui.URLPrefix != nil || len(ui.URLMaps) > 0 || ui.DefaultURL != nil
-}
-
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
 	RequestHeaders     []*Header `yaml:"headers,omitempty"`
@@ -995,11 +983,8 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
 			return nil, err
 		}
-
-		if ui.hasAnyURLs() {
-			if err := ui.initURLs(); err != nil {
-				return nil, err
-			}
+		if err := ui.initURLs(); err != nil {
+			return nil, err
 		}
 
 		metricLabels, err := ui.getMetricLabels()

app/vmauth/main.go

@@ -175,12 +175,11 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if len(ats) == 0 {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
-		if ui.hasAnyURLs() {
+		if ui != nil {
 			processUserRequest(w, r, ui, nil)
 			return true
 		}
 
-		ui.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 		handleMissingAuthorizationError(w)
 		return true
 	}
@@ -193,24 +192,23 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		if tkn == nil {
 			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
-		defer putToken(tkn)
-		// Call processUserRequest only if the token contains the vm_access claim
-		// or a default claim is configured; otherwise fall through to unauthorized_user.
-		if tkn.HasVMAccessClaim() || ui.JWT.DefaultVMAccessClaim != nil {
-			processUserRequest(w, r, ui, tkn)
+		if !tkn.HasVMAccessClaim() && ui.JWT.DefaultVMAccessClaim == nil {
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
 			return true
 		}
+		defer putToken(tkn)
+		processUserRequest(w, r, ui, tkn)
+		return true
 	}
 
 	uu := authConfig.Load().UnauthorizedUser
-	if uu.hasAnyURLs() {
+	if uu != nil {
 		processUserRequest(w, r, uu, nil)
 		return true
 	}
 
 	invalidAuthTokenRequests.Inc()
 	slowdownUnauthorizedResponse(r)
-	uu.logRequest(r, `unauthorized`, http.StatusUnauthorized, 0)
 	if *logInvalidAuthTokens {
 		err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
 		err = &httpserver.ErrorWithStatusCode{

app/vmauth/main_test.go

@@ -785,26 +785,7 @@ statusCode=401
 Unauthorized`
 	f(simpleCfgStr, request, responseExpected)
 
-	// token without vm_access claim should fall through to unauthorized_user
-	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
-	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
-	responseExpected = `
-statusCode=200
-path: /bar/abc
-query:
-headers:`
-	f(fmt.Sprintf(`
-unauthorized_user:
-  url_prefix: {BACKEND}/bar
-users:
-- jwt:
-    public_keys:
-    - %q
-    match_claims:
-      role: admin
-  url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
-
-	// token without vm_access claim is accepted when default_vm_access_claim configured
+	// token without vm_access claim is accepted when it
 	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
 	request.Header.Set(`Authorization`, `Bearer `+roleToken)
 	responseExpected = `

app/vmctl/vm/vm.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -112,7 +111,7 @@ func AddExtraLabelsToImportPath(path string, extraLabels []string) (string, erro
 		if strings.Contains(dst, "?") {
 			separator = "&"
 		}
-		dst += fmt.Sprintf("%sextra_label=%s", separator, url.QueryEscape(extraLabel))
+		dst += fmt.Sprintf("%sextra_label=%s", separator, extraLabel)
 	}
 	return dst, nil
 }

app/vmctl/vm/vm_test.go

@@ -33,14 +33,11 @@ func TestAddExtraLabelsToImportPath_Success(t *testing.T) {
 	f("/api/v1/import", nil, "/api/v1/import")
 
 	// ok one extra label
-	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance%3Dhost-1")
+	f("/api/v1/import", []string{"instance=host-1"}, "/api/v1/import?extra_label=instance=host-1")
 
 	// ok two extra labels
-	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
+	f("/api/v1/import", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?extra_label=instance=host-2&extra_label=job=vmagent")
 
 	// ok two extra with exist param
-	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance%3Dhost-2&extra_label=job%3Dvmagent")
-
-	// ok special chars in label value
-	f("/api/v1/import", []string{"team=a&b"}, "/api/v1/import?extra_label=team%3Da%26b")
+	f("/api/v1/import?timeout=50", []string{"instance=host-2", "job=vmagent"}, "/api/v1/import?timeout=50&extra_label=instance=host-2&extra_label=job=vmagent")
 }

app/vminsert/main.go

@@ -68,11 +68,9 @@ var (
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey          = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	reloadAuthKey          = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented.")
-	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented. "+
-		"Value must be in range 1..65535.")
-	maxLabelValueLen = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented. "+
-		"Value must be in range 1..65535.")
+	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 40, "The maximum number of labels per time series to be accepted. Series with superfluous labels are ignored. In this case the vm_rows_ignored_total{reason=\"too_many_labels\"} metric at /metrics page is incremented")
+	maxLabelNameLen        = flag.Int("maxLabelNameLen", 256, "The maximum length of label name in the accepted time series. Series with longer label name are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_name\"} metric at /metrics page is incremented")
+	maxLabelValueLen       = flag.Int("maxLabelValueLen", 4*1024, "The maximum length of label values in the accepted time series. Series with longer label value are ignored. In this case the vm_rows_ignored_total{reason=\"too_long_label_value\"} metric at /metrics page is incremented")
 )
 
 var (
@@ -108,7 +106,7 @@ func Init() {
 	promscrape.Init(func(_ *auth.Token, wr *prompb.WriteRequest) {
 		prompush.Push(wr)
 	})
-	timeserieslimits.MustInit(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
+	timeserieslimits.Init(*maxLabelsPerTimeseries, *maxLabelNameLen, *maxLabelValueLen)
 }
 
 // Stop stops vminsert.

app/vmselect/promql/binary_op.go

@@ -172,7 +172,13 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
 			left = removeEmptySeries(left)
 			right = removeEmptySeries(right)
 		}
-		if len(left) == 0 || len(right) == 0 {
+		if len(left) == 0 && len(right) == 0 {
+			return nil, nil
+		}
+		if len(left) == 0 && bfa.be.FillLeft == nil {
+			return nil, nil
+		}
+		if len(right) == 0 && bfa.be.FillRight == nil {
 			return nil, nil
 		}
 		left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -226,7 +232,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		}
 	}
 
-	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
+	// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
 	var rvsLeft, rvsRight []*timeseries
 	mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
 	joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -239,10 +245,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 		// Add __name__ to groupTags if metric name must be preserved.
 		groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
 	}
+	// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
+	if be.FillLeft != nil {
+		for k := range mRight {
+			if _, ok := mLeft[k]; !ok {
+				mLeft[k] = nil
+			}
+		}
+	}
 	for k, tssLeft := range mLeft {
 		tssRight := mRight[k]
+		if len(tssLeft) == 0 {
+			if be.FillLeft == nil {
+				logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
+			}
+			tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0], be.FillLeft.N)}
+		}
 		if len(tssRight) == 0 {
-			continue
+			if be.FillRight == nil {
+				continue
+			}
+			tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0], be.FillRight.N)}
 		}
 		switch joinOp {
 		case "group_left":
@@ -287,6 +310,27 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
 	return rvsLeft, rvsRight, dst, nil
 }
 
+// newFillTimeseries returns a time series filled with fillValue for the fill_left()/fill_right()/fill() modifiers.
+func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries, fillValue float64) *timeseries {
+	var ts timeseries
+	ts.CopyFromShallowTimestamps(src)
+	if !be.KeepMetricNames {
+		ts.MetricName.ResetMetricGroup()
+	}
+	groupTags := be.GroupModifier.Args
+	switch strings.ToLower(be.GroupModifier.Op) {
+	case "on":
+		ts.MetricName.RemoveTagsOn(groupTags)
+	default:
+		ts.MetricName.RemoveTagsIgnoring(groupTags)
+	}
+	values := ts.Values
+	for i := range values {
+		values[i] = fillValue
+	}
+	return &ts
+}
+
 func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
 	if len(tss) == 0 {
 		logger.Panicf("BUG: tss must contain at least one value")

app/vmselect/promql/eval.go

@@ -424,18 +424,7 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	if bf == nil {
 		return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
 	}
-	var err error
-	var tssLeft, tssRight []*timeseries
-	switch strings.ToLower(be.Op) {
-	case "and", "if":
-		// Fetch right-side series at first, since it usually contains
-		// lower number of time series for `and` and `if` operator.
-		// This should produce more specific label filters for the left side of the query.
-		// This, in turn, should reduce the time to select series for the left side of the query.
-		tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
-	default:
-		tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
-	}
+	tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
 	if err != nil {
 		return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
 	}
@@ -451,6 +440,29 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
 	return rv, nil
 }
 
+// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
+// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
+func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
+	exprFirst, exprSecond = be.Left, be.Right
+	switch strings.ToLower(be.Op) {
+	case "and", "if":
+		// For `and` and `if`, fetch the right-side series first, since it usually contains
+		// fewer time series and yields more specific filters for the left side.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	if be.FillLeft != nil && be.FillRight == nil {
+		// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
+		exprFirst, exprSecond = be.Right, be.Left
+	}
+	return exprFirst, exprSecond
+}
+
+// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
+//
+// Common filters cannot be pushed down when:
+// - the operator is `or` or `default`;
+// - either side is an aggregation function without explicit grouping;
+// - fill(<value>) modifier is used.
 func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	switch strings.ToLower(be.Op) {
 	case "or", "default":
@@ -459,6 +471,10 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
 	if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
 		return false
 	}
+	// Filters cannot be propagated when fill(<value>) modifier is used.
+	if be.FillLeft != nil && be.FillRight != nil {
+		return false
+	}
 	return true
 }
 
@@ -470,55 +486,50 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
 	return len(afe.Modifier.Args) == 0
 }
 
-func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
-	if canPushdownCommonFilters(be) {
-		// Execute binary operation in the following way:
-		//
-		// 1) execute the exprFirst
-		// 2) get common label filters for series returned at step 1
-		// 3) push down the found common label filters to exprSecond. This filters out unneeded series
-		//    during exprSecond execution instead of spending compute resources on extracting and processing these series
-		//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
-		// 4) execute the exprSecond with possible additional filters found at step 3
-		//
-		// Typical use cases:
-		// - Kubernetes-related: show pod creation time with the node name:
-		//
-		//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
-		//
-		//   Without the optimization `kube_pod_info` would select and spend compute resources
-		//   for more time series than needed. The selected time series would be dropped later
-		//   when matching time series on the right and left sides of binary operand.
-		//
-		// - Generic alerting queries, which rely on `info` metrics.
-		//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
-		//
-		// - Queries, which get additional labels from `info` metrics.
-		//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
-		tssFirst, err := evalExpr(qt, ec, exprFirst)
-		if err != nil {
-			return nil, nil, err
+func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
+	exprFirst, exprSecond := binaryOpEvalOrder(be)
+	canPushdown := canPushdownCommonFilters(be)
+	firstIsLeft := exprFirst == be.Left
+	sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
+		if firstIsLeft {
+			return tssFirst, tssSecond, nil
 		}
-		if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
-			// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
-			// since the "exprFirst op exprSecond" would return an empty result in any case.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
-			return nil, nil, nil
-		}
-		lfs := getCommonLabelFilters(tssFirst)
-		lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
-		exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
-		tssSecond, err := evalExpr(qt, ec, exprSecond)
-		if err != nil {
-			return nil, nil, err
-		}
-		return tssFirst, tssSecond, nil
+		return tssSecond, tssFirst, nil
 	}
+	if !canPushdown && !shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
+		// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
+		// from exprFirst to exprSecond.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
+		qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
+		defer qt.Done()
+		var wg sync.WaitGroup
 
-	// Execute exprFirst and exprSecond sequentially if there are cacheable repeated subexpressions
-	// in exprFirst and exprSecond.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575
-	if shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
+		var tssFirst []*timeseries
+		var errFirst error
+		qtFirst := qt.NewChild("expr1")
+		wg.Go(func() {
+			tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
+			qtFirst.Done()
+		})
+
+		var tssSecond []*timeseries
+		var errSecond error
+		qtSecond := qt.NewChild("expr2")
+		wg.Go(func() {
+			tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
+			qtSecond.Done()
+		})
+
+		wg.Wait()
+		if errFirst != nil {
+			return nil, nil, errFirst
+		}
+		if errSecond != nil {
+			return nil, nil, errSecond
+		}
+		return sortResult(tssFirst, tssSecond)
+	}
+	if !canPushdown {
 		qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
 		defer qt.Done()
 
@@ -535,40 +546,50 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSec
 		if err != nil {
 			return nil, nil, err
 		}
-		return tssFirst, tssSecond, nil
+		return sortResult(tssFirst, tssSecond)
 	}
 
-	// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
-	// from exprFirst to exprSecond.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
-	qt = qt.NewChild("execute left and right sides of %q in parallel", be.Op)
-	defer qt.Done()
-	var wg sync.WaitGroup
-
-	var tssFirst []*timeseries
-	var errFirst error
-	qtFirst := qt.NewChild("expr1")
-	wg.Go(func() {
-		tssFirst, errFirst = evalExpr(qtFirst, ec, exprFirst)
-		qtFirst.Done()
-	})
-
-	var tssSecond []*timeseries
-	var errSecond error
-	qtSecond := qt.NewChild("expr2")
-	wg.Go(func() {
-		tssSecond, errSecond = evalExpr(qtSecond, ec, exprSecond)
-		qtSecond.Done()
-	})
-
-	wg.Wait()
-	if errFirst != nil {
-		return nil, nil, errFirst
+	// Execute binary operation in the following way:
+	//
+	// 1) execute the exprFirst
+	// 2) get common label filters for series returned at step 1
+	// 3) push down the found common label filters to exprSecond. This filters out unneeded series
+	//    during exprSecond execution instead of spending compute resources on extracting and processing these series
+	//    before they are dropped later when matching time series according to https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching
+	// 4) execute the exprSecond with possible additional filters found at step 3
+	//
+	// Typical use cases:
+	// - Kubernetes-related: show pod creation time with the node name:
+	//
+	//     kube_pod_created{namespace="prod"} * on (uid) group_left(node) kube_pod_info
+	//
+	//   Without the optimization `kube_pod_info` would select and spend compute resources
+	//   for more time series than needed. The selected time series would be dropped later
+	//   when matching time series on the right and left sides of binary operand.
+	//
+	// - Generic alerting queries, which rely on `info` metrics.
+	//   See https://grafana.com/blog/2021/08/04/how-to-use-promql-joins-for-more-effective-queries-of-prometheus-metrics-at-scale/
+	//
+	// - Queries, which get additional labels from `info` metrics.
+	//   See https://www.robustperception.io/exposing-the-software-version-to-prometheus
+	tssFirst, err := evalExpr(qt, ec, exprFirst)
+	if err != nil {
+		return nil, nil, err
 	}
-	if errSecond != nil {
-		return nil, nil, errSecond
+	if len(tssFirst) == 0 && !strings.EqualFold(be.Op, "or") {
+		// Fast path: there is no sense in executing the exprSecond when exprFirst returns an empty result,
+		// since the "exprFirst op exprSecond" would return an empty result in any case.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3349
+		return nil, nil, nil
 	}
-	return tssFirst, tssSecond, nil
+	lfs := getCommonLabelFilters(tssFirst)
+	lfs = metricsql.TrimFiltersByGroupModifier(lfs, be)
+	exprSecond = metricsql.PushdownBinaryOpFilters(exprSecond, lfs)
+	tssSecond, err := evalExpr(qt, ec, exprSecond)
+	if err != nil {
+		return nil, nil, err
+	}
+	return sortResult(tssFirst, tssSecond)
 }
 
 func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {

app/vmselect/promql/exec_test.go

@@ -4006,6 +4006,256 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`vector + vector fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill(0) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_left(10) fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{14, 14, 14, 14, 14, 14},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common")
+			or label_set(2, "foo", "left_only")
+		) + fill_right(20) (
+			label_set(3, "foo", "common")
+			or label_set(4, "foo", "right_only")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{22, 22, 22, 22, 22, 22},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "foo", "common", "extra", "l")
+			or label_set(2, "foo", "left_only", "extra", "l")
+		) + on(foo) fill(0) (
+			label_set(3, "foo", "common", "extra", "r")
+			or label_set(4, "foo", "right_only", "extra", "r")
+		), "foo")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("common"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("left_only"),
+		}}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{4, 4, 4, 4, 4, 4},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("right_only"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(1, "method", "get", "code", "500")
+			or label_set(2, "method", "get", "code", "404")
+			or label_set(3, "method", "put", "code", "501")
+		) + on(method) group_left() fill_right(0) (
+			label_set(10, "method", "get")
+		), "method", "code")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12, 12, 12, 12, 12, 12},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("404"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{11, 11, 11, 11, 11, 11},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("500"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{3, 3, 3, 3, 3, 3},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("code"),
+				Value: []byte("501"),
+			},
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label((
+			label_set(6, "method", "get", "code", "500")
+			or label_set(1, "method", "put", "code", "500")
+		) / ignoring(code) fill(0) (
+			label_set(12, "method", "get")
+			or label_set(5, "method", "post")
+			or label_set(10, "method", "put")
+		), "method")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("get"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("post"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("method"),
+				Value: []byte("put"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, time())`
@@ -4833,137 +5083,13 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
-	// buckets that are consecutively empty at left and right ends will not be preserved.
-	t.Run(`buckets_limit(trim_zero_preserve_empty_when_limit_not_reached)`, func(t *testing.T) {
+	t.Run(`buckets_limit(zero)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(buckets_limit(3, (
-			alias(label_set(36, "le", "+Inf"), "metric"),
-			alias(label_set(36, "le", "25"), "metric"),
-			alias(label_set(36, "le", "21"), "metric"),
-			alias(label_set(36, "le", "19"), "metric"),
-			alias(label_set(36, "le", "18"), "metric"),
-			alias(label_set(36, "le", "17"), "metric"),
-			alias(label_set(36, "le", "16"), "metric"),
-			alias(label_set(27, "le", "12"), "metric"),
-			alias(label_set(14, "le", "9"), "metric"),
-			alias(label_set(0, "le", "6"), "metric"),
-			alias(label_set(0, "le", "1"), "metric"),
-			)))`
-		r1 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{14, 14, 14, 14, 14, 14},
-			Timestamps: timestampsExpected,
-		}
-		r1.MetricName.MetricGroup = []byte("metric")
-		r1.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("9"),
-			},
-		}
-		r2 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{27, 27, 27, 27, 27, 27},
-			Timestamps: timestampsExpected,
-		}
-		r2.MetricName.MetricGroup = []byte("metric")
-		r2.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("12"),
-			},
-		}
-		r3 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{36, 36, 36, 36, 36, 36},
-			Timestamps: timestampsExpected,
-		}
-		r3.MetricName.MetricGroup = []byte("metric")
-		r3.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("16"),
-			},
-		}
-		resultExpected := []netstorage.Result{r1, r2, r3}
-		f(q, resultExpected)
-	})
-
-	// the number of non-empty bucket doesn't reach the given "limit", so some empty buckets will be preserved, and left buckets are preferred to be kept.
-	t.Run(`buckets_limit(trim_zero)`, func(t *testing.T) {
-		t.Parallel()
-		q := `sort(buckets_limit(5, (
-			alias(label_set(36, "le", "18"), "metric"),
-			alias(label_set(36, "le", "17"), "metric"),
-			alias(label_set(36, "le", "16"), "metric"),
-			alias(label_set(27, "le", "12"), "metric"),
-			alias(label_set(14, "le", "9"), "metric"),
-			alias(label_set(0, "le", "6"), "metric"),
-			alias(label_set(0, "le", "1"), "metric"),
-			)))`
-
-		r1 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		r1.MetricName.MetricGroup = []byte("metric")
-		r1.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("1"),
-			},
-		}
-		r2 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{0, 0, 0, 0, 0, 0},
-			Timestamps: timestampsExpected,
-		}
-		r2.MetricName.MetricGroup = []byte("metric")
-		r2.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("6"),
-			},
-		}
-		r3 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{14, 14, 14, 14, 14, 14},
-			Timestamps: timestampsExpected,
-		}
-		r3.MetricName.MetricGroup = []byte("metric")
-		r3.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("9"),
-			},
-		}
-		r4 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{27, 27, 27, 27, 27, 27},
-			Timestamps: timestampsExpected,
-		}
-		r4.MetricName.MetricGroup = []byte("metric")
-		r4.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("12"),
-			},
-		}
-		r5 := netstorage.Result{
-			MetricName: metricNameExpected,
-			Values:     []float64{36, 36, 36, 36, 36, 36},
-			Timestamps: timestampsExpected,
-		}
-		r5.MetricName.MetricGroup = []byte("metric")
-		r5.MetricName.Tags = []storage.Tag{
-			{
-				Key:   []byte("le"),
-				Value: []byte("16"),
-			},
-		}
-
-		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5}
+		q := `buckets_limit(0, (
+			alias(label_set(100, "le", "inf", "x", "y"), "metric"),
+			alias(label_set(50, "le", "120", "x", "y"), "metric"),
+		))`
+		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
 	t.Run(`buckets_limit(unused)`, func(t *testing.T) {
@@ -6352,6 +6478,50 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4, r5, r6, r7}
 		f(q, resultExpected)
 	})
+	t.Run(`sum(histogram_over_time) by (vmrange)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort_by_label(
+			buckets_limit(
+				3,
+				sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s])) by (vmrange)
+			), "le"
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{40, 40, 40, 40, 40, 40},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("+Inf"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("1.000e+00"),
+			},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{40, 40, 40, 40, 40, 40},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("le"),
+				Value: []byte("2.448e+00"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
 	t.Run(`sum(histogram_over_time)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sum(histogram_over_time(alias(label_set(rand(0)*1.3+1.1, "foo", "bar"), "xxx")[200s:5s]))`

app/vmselect/promql/transform.go

@@ -393,7 +393,7 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 		return nil, err
 	}
 	if limit <= 0 {
-		return nil, fmt.Errorf("limit must be greater than 0; got %d", limit)
+		return nil, nil
 	}
 	if limit < 3 {
 		// Preserve the first and the last bucket for better accuracy for min and max values.
@@ -461,23 +461,6 @@ func transformBucketsLimit(tfa *transformFuncArg) ([]*timeseries, error) {
 				prevValue = value
 			}
 		}
-
-		// Remove buckets that are consecutively empty at left and right ends to obtain more accurate max and min values.
-		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417.
-		epsilon := 1e-9
-		isEmptyBucket := func(hits float64) bool {
-			return !math.IsNaN(hits) && math.Abs(hits) < epsilon
-		}
-		l := 0
-		r := len(leGroup) - 1
-		for r-l+1 > limit && isEmptyBucket(leGroup[r].hits) {
-			r--
-		}
-		for r-l+1 > limit && isEmptyBucket(leGroup[l].hits) {
-			l++
-		}
-		leGroup = leGroup[l : r+1]
-
 		for len(leGroup) > limit {
 			// Preserve the first and the last bucket for better accuracy for min and max values
 			xxMinIdx := 1
@@ -1138,29 +1121,29 @@ func groupLeTimeseries(tss []*timeseries) map[string][]leTimeseries {
 
 func fixBrokenBuckets(i int, xss []leTimeseries) {
 	// Buckets are already sorted by le, so their values must be in ascending order,
-	// since the upper bucket includes all the lower buckets.
-	// If the upper bucket has lower value than the current bucket,
-	// then the upper bucket must be substituted with the current bucket value.
+	// since the next bucket includes all the previous buckets.
+	// If the next bucket has lower value than the current bucket,
+	// then the next bucket must be substituted with the current bucket value.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580#issuecomment-2186659102
 	if len(xss) < 2 {
 		return
 	}
 
-	vPrev := xss[0].ts.Values[i]
+	vNext := xss[0].ts.Values[i]
 	// Set the lowest bucket to 0 if its value is NaN, so it can be properly
 	// compared with upper buckets in the loop below.
-	if math.IsNaN(vPrev) {
-		vPrev = 0
-		xss[0].ts.Values[i] = vPrev
+	if math.IsNaN(vNext) {
+		vNext = 0
+		xss[0].ts.Values[i] = vNext
 	}
 	// Substitute upper bucket values with lower bucket values if the upper values are NaN
-	// or are smaller than the lower bucket values.
+	// or are bigger than the lower bucket values.
 	for j := 1; j < len(xss); j++ {
 		v := xss[j].ts.Values[i]
-		if math.IsNaN(v) || vPrev > v {
-			xss[j].ts.Values[i] = vPrev
+		if math.IsNaN(v) || vNext > v {
+			xss[j].ts.Values[i] = vNext
 		} else {
-			vPrev = v
+			vNext = v
 		}
 	}
 }

app/vmselect/vmui/assets/MetricsQL-DmQtQmkX.md

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
   Labels from the `on()` list aren't copied.
 * [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
   For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
-* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
+* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
   For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
-* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
+* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
   For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
 * [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
   and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -1229,7 +1229,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
+The result will preserve the first and the last bucket to improve accuracy for min and max values. 
+So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

app/vmselect/vmui/index.html

@@ -37,7 +37,7 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-xYKUiOTH.js"></script>
+  <script type="module" crossorigin src="./assets/index-CusQvJzs.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
   <link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">

app/vmui/packages/vmui/src/assets/MetricsQL.md

@@ -1229,7 +1229,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
+The result will preserve the first and the last bucket to improve accuracy for min and max values. 
+So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

dashboards/query-stats.json

@@ -1,59 +1,4 @@
 {
-  "__inputs": [
-    {
-      "name": "DS_VICTORIALOGS",
-      "label": "VictoriaLogs",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "victoriametrics-logs-datasource",
-      "pluginName": "VictoriaLogs"
-    }
-  ],
-  "__elements": {},
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "12.4.3"
-    },
-    {
-      "type": "panel",
-      "id": "logs",
-      "name": "Logs",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "stat",
-      "name": "Stat",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "table",
-      "name": "Table",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "text",
-      "name": "Text",
-      "version": ""
-    },
-    {
-      "type": "panel",
-      "id": "timeseries",
-      "name": "Time series",
-      "version": ""
-    },
-    {
-      "type": "datasource",
-      "id": "victoriametrics-logs-datasource",
-      "name": "VictoriaLogs",
-      "version": "0.29.0"
-    }
-  ],
   "annotations": {
     "list": [
       {
@@ -73,6 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
+  "id": 861,
   "links": [
     {
       "icon": "doc",
@@ -132,8 +78,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               }
             ]
           }
@@ -164,7 +109,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -172,7 +117,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| type:=\"instant\"\n| count()",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| type:=\"instant\"\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -195,8 +140,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               }
             ]
           }
@@ -227,7 +171,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -235,7 +179,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| type:=\"range\"\n| count()",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| type:=\"range\"\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -258,8 +202,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               }
             ]
           }
@@ -290,7 +233,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -298,7 +241,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| series_fetched:=0\n| count()",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| series_fetched:=0\n| count()",
           "queryType": "stats",
           "refId": "A"
         }
@@ -322,8 +265,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               }
             ]
           },
@@ -355,7 +297,7 @@
         "textMode": "auto",
         "wideLayout": true
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -363,7 +305,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash \n| stats min(start_ms)\n",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash) \n| stats min(start_ms)\n",
           "queryType": "stats",
           "refId": "A"
         }
@@ -385,6 +327,10 @@
       "type": "row"
     },
     {
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
       "gridPos": {
         "h": 2,
         "w": 24,
@@ -401,7 +347,8 @@
         "content": "To filter by specific query copy its hash from the table and put it into `query_hash` filter on the top. To disable filtering enter `*`.",
         "mode": "markdown"
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
+      "title": "",
       "transparent": true,
       "type": "text"
     },
@@ -420,9 +367,6 @@
             "cellOptions": {
               "type": "auto"
             },
-            "footer": {
-              "reducers": []
-            },
             "inspect": false
           },
           "mappings": [],
@@ -430,8 +374,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -480,6 +423,18 @@
                 "value": 204
               }
             ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "duration_max"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 122
+              }
+            ]
           }
         ]
       },
@@ -492,10 +447,18 @@
       "id": 4,
       "options": {
         "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -503,7 +466,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(execution_duration_ms) duration_max \n| sort by(duration_max) desc | limit $top",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(execution_duration_ms) duration_max \n| sort by(duration_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -515,7 +478,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "labels"
+            "source": "Line"
           }
         },
         {
@@ -570,7 +533,6 @@
               "type": "linear"
             },
             "showPoints": "auto",
-            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -585,8 +547,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -619,7 +580,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -627,7 +588,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(execution_duration_ms) execution_duration_max",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(execution_duration_ms) execution_duration_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -650,9 +611,6 @@
             "cellOptions": {
               "type": "auto"
             },
-            "footer": {
-              "reducers": []
-            },
             "inspect": false
           },
           "mappings": [],
@@ -660,8 +618,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -723,10 +680,18 @@
       "interval": "1m",
       "options": {
         "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -734,7 +699,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(series_fetched) series_fetched_max\n| sort by(series_fetched_max) desc | limit $top",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(series_fetched) series_fetched_max\n| sort by(series_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -746,7 +711,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "labels"
+            "source": "Line"
           }
         },
         {
@@ -801,7 +766,6 @@
               "type": "linear"
             },
             "showPoints": "auto",
-            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -816,8 +780,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -850,7 +813,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -858,7 +821,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(series_fetched) series_fetched_max",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(series_fetched) series_fetched_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -881,9 +844,6 @@
             "cellOptions": {
               "type": "auto"
             },
-            "footer": {
-              "reducers": []
-            },
             "inspect": false
           },
           "mappings": [],
@@ -891,8 +851,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -953,10 +912,18 @@
       "id": 5,
       "options": {
         "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -964,7 +931,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(samples_fetched) samples_fetched_max\n| sort by(samples_fetched_max) desc | limit $top",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(samples_fetched) samples_fetched_max\n| sort by(samples_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -975,10 +942,8 @@
           "id": "extractFields",
           "options": {
             "delimiter": ",",
-            "format": "json",
-            "keepTime": false,
             "replace": true,
-            "source": "labels"
+            "source": "Line"
           }
         },
         {
@@ -1033,7 +998,6 @@
               "type": "linear"
             },
             "showPoints": "auto",
-            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1048,8 +1012,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1082,7 +1045,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1090,7 +1053,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(samples_fetched) samples_fetched_max",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(samples_fetched) samples_fetched_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1113,9 +1076,6 @@
             "cellOptions": {
               "type": "auto"
             },
-            "footer": {
-              "reducers": []
-            },
             "inspect": false
           },
           "mappings": [],
@@ -1123,8 +1083,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1185,10 +1144,18 @@
       "id": 11,
       "options": {
         "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1196,7 +1163,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(bytes) bytes_fetched_max \n| sort by(bytes_fetched_max) desc | limit $top",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(bytes) bytes_fetched_max \n| sort by(bytes_fetched_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1208,7 +1175,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "labels"
+            "source": "Line"
           }
         },
         {
@@ -1263,7 +1230,6 @@
               "type": "linear"
             },
             "showPoints": "auto",
-            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1278,8 +1244,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1312,7 +1277,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1320,7 +1285,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(bytes) bytes_fetched",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(bytes) bytes_fetched",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1343,9 +1308,6 @@
             "cellOptions": {
               "type": "auto"
             },
-            "footer": {
-              "reducers": []
-            },
             "inspect": false
           },
           "mappings": [],
@@ -1353,8 +1315,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1427,10 +1388,18 @@
       "id": 13,
       "options": {
         "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1438,7 +1407,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(memory_estimated_bytes) memory_estimated_max\n| sort by(memory_estimated_max) desc | limit $top",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(memory_estimated_bytes) memory_estimated_max\n| sort by(memory_estimated_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1450,7 +1419,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "labels"
+            "source": "Line"
           }
         },
         {
@@ -1505,7 +1474,6 @@
               "type": "linear"
             },
             "showPoints": "auto",
-            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1520,8 +1488,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1554,7 +1521,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1562,7 +1529,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats max(memory_estimated_bytes) memory_estimated_bytes",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats max(memory_estimated_bytes) memory_estimated_bytes",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1585,9 +1552,6 @@
             "cellOptions": {
               "type": "auto"
             },
-            "footer": {
-              "reducers": []
-            },
             "inspect": false
           },
           "mappings": [],
@@ -1595,8 +1559,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1657,10 +1620,18 @@
       "id": 18,
       "options": {
         "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1668,7 +1639,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:$query_hash\n| stats by(tenant,query,query_hash) max(range_ms) range_max \n| sort by(range_max) desc | limit $top",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| tenant:in($tenant)\n| query_hash:in($query_hash)\n| stats by(tenant,query,query_hash) max(range_ms) range_max \n| sort by(range_max) desc | limit $top",
           "queryType": "instant",
           "refId": "A"
         }
@@ -1680,7 +1651,7 @@
           "options": {
             "delimiter": ",",
             "replace": true,
-            "source": "labels"
+            "source": "Line"
           }
         },
         {
@@ -1735,7 +1706,6 @@
               "type": "linear"
             },
             "showPoints": "auto",
-            "showValues": false,
             "spanNulls": false,
             "stacking": {
               "group": "A",
@@ -1750,8 +1720,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": 0
+                "color": "green"
               },
               {
                 "color": "red",
@@ -1784,7 +1753,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "12.4.3",
+      "pluginVersion": "11.6.0",
       "targets": [
         {
           "datasource": {
@@ -1792,7 +1761,7 @@
             "uid": "${ds}"
           },
           "editorMode": "code",
-          "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:$query_hash\n| stats max(range_ms) range_max",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:in($query_hash)\n| stats max(range_ms) range_max",
           "queryType": "statsRange",
           "refId": "A"
         }
@@ -1801,7 +1770,7 @@
       "type": "timeseries"
     },
     {
-      "collapsed": true,
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -1809,59 +1778,52 @@
         "y": 56
       },
       "id": 12,
-      "panels": [
+      "panels": [],
+      "title": "Query log",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "victoriametrics-logs-datasource",
+        "uid": "${ds}"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 14,
+        "w": 24,
+        "x": 0,
+        "y": 57
+      },
+      "id": 6,
+      "options": {
+        "dedupStrategy": "none",
+        "enableInfiniteScrolling": false,
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.6.0",
+      "targets": [
         {
           "datasource": {
             "type": "victoriametrics-logs-datasource",
             "uid": "${ds}"
           },
-          "fieldConfig": {
-            "defaults": {},
-            "overrides": []
-          },
-          "gridPos": {
-            "h": 14,
-            "w": 24,
-            "x": 0,
-            "y": 57
-          },
-          "id": 6,
-          "options": {
-            "dedupStrategy": "none",
-            "detailsMode": "sidebar",
-            "enableInfiniteScrolling": false,
-            "enableLogDetails": true,
-            "fontSize": "small",
-            "prettifyLogMessage": false,
-            "showCommonLabels": false,
-            "showControls": true,
-            "showLabels": false,
-            "showTime": false,
-            "sortOrder": "Descending",
-            "syntaxHighlighting": false,
-            "unwrappedColumns": false,
-            "wrapLogMessage": true
-          },
-          "pluginVersion": "12.4.3",
-          "targets": [
-            {
-              "datasource": {
-                "type": "victoriametrics-logs-datasource",
-                "uid": "${ds}"
-              },
-              "direction": "desc",
-              "editorMode": "code",
-              "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:$query_hash\n| limit 200",
-              "queryType": "instant",
-              "refId": "A"
-            }
-          ],
-          "title": "Raw logs",
-          "type": "logs"
+          "editorMode": "code",
+          "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| query_hash:in($query_hash)\n| limit 200",
+          "queryType": "instant",
+          "refId": "A"
         }
       ],
-      "title": "Query log",
-      "type": "row"
+      "title": "Raw logs",
+      "type": "logs"
     },
     {
       "collapsed": true,
@@ -1869,7 +1831,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 57
+        "y": 71
       },
       "id": 17,
       "panels": [
@@ -1886,7 +1848,7 @@
             "h": 14,
             "w": 24,
             "x": 0,
-            "y": 58
+            "y": 70
           },
           "id": 15,
           "options": {
@@ -1895,14 +1857,12 @@
             "enableLogDetails": true,
             "prettifyLogMessage": false,
             "showCommonLabels": false,
-            "showControls": false,
             "showLabels": false,
             "showTime": false,
             "sortOrder": "Descending",
-            "unwrappedColumns": false,
             "wrapLogMessage": false
           },
-          "pluginVersion": "12.4.3",
+          "pluginVersion": "11.6.0",
           "targets": [
             {
               "datasource": {
@@ -1910,7 +1870,7 @@
                 "uid": "${ds}"
               },
               "editorMode": "code",
-              "expr": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| series_fetched:=0\n| query_hash:$query_hash",
+              "expr": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats \n| series_fetched:=0\n| query_hash:in($query_hash)",
               "queryType": "instant",
               "refId": "A"
             }
@@ -1925,7 +1885,7 @@
   ],
   "preload": false,
   "refresh": "",
-  "schemaVersion": 42,
+  "schemaVersion": 41,
   "tags": [
     "victoriametrics",
     "victorialogs"
@@ -1934,9 +1894,8 @@
     "list": [
       {
         "current": {
-          "text": "",
-          "value": "${ds}",
-          "selected": true
+          "text": "VictoriaLogs",
+          "value": "PD775F2863313E6C7"
         },
         "name": "ds",
         "options": [],
@@ -1974,17 +1933,21 @@
           }
         ],
         "query": "5,10,15,20",
-        "type": "custom",
-        "valuesFormat": "csv"
+        "type": "custom"
       },
       {
         "allValue": "*",
-        "current": {},
+        "current": {
+          "text": "All",
+          "value": [
+            "$__all"
+          ]
+        },
         "datasource": {
           "type": "victoriametrics-logs-datasource",
           "uid": "${ds}"
         },
-        "definition": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
+        "definition": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
         "includeAll": true,
         "multi": true,
         "name": "tenant",
@@ -1992,13 +1955,12 @@
         "query": {
           "field": "tenant",
           "limit": 25,
-          "query": "\"vm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
+          "query": "\"\\tvm_slow_query_stats\" | extract 'vm_slow_query_stats <vm_slow_query_stats>' | unpack_logfmt from vm_slow_query_stats  | fields tenant",
           "refId": "VictoriaLogsVariableQueryEditor-VariableQuery",
           "type": "fieldValue"
         },
-        "refresh": 2,
+        "refresh": 1,
         "regex": "",
-        "regexApplyTo": "value",
         "type": "query"
       },
       {
@@ -2038,6 +2000,5 @@
   "timezone": "browser",
   "title": "Query Stats (cluster)",
   "uid": "feg3od1zt1fy8e",
-  "version": 1,
-  "weekStart": ""
-}
+  "version": 1
+}

dashboards/victoriametrics.json

@@ -896,7 +896,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
+          "expr": "sum(min_over_time(vm_app_version{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
           "format": "time_series",
           "instant": false,
           "legendFormat": "{{job}}",

dashboards/vm/victoriametrics.json

@@ -897,7 +897,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
+          "expr": "sum(min_over_time(vm_app_version{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
           "format": "time_series",
           "instant": false,
           "legendFormat": "{{job}}",

dashboards/vm/vmagent.json

@@ -892,7 +892,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
+          "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)",
           "format": "time_series",
           "instant": false,
           "interval": "",

dashboards/vmagent.json

@@ -891,7 +891,7 @@
             "uid": "$ds"
           },
           "editorMode": "code",
-          "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
+          "expr": "sum(up{job=~\"$job\", instance=~\"$instance\"}) by (job)",
           "format": "time_series",
           "instant": false,
           "interval": "",

deployment/docker/rules/alerts-vmalert.yml

@@ -120,39 +120,3 @@ groups:
           summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
           description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
             Check vmalert's logs for detailed error message."
-
-      - alert: AlertingRuleResultsApproachingLimit
-        expr: |
-          (
-            vmalert_alerting_rules_last_evaluation_samples
-              > on(group,file) group_left()
-            (vmalert_group_rule_results_limit * 0.9)
-          )
-            and on(group,file)
-          (vmalert_group_rule_results_limit > 0)
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Alerting rule {{ $labels.alertname }} in group {{ $labels.group }} is approaching the configured results limit"
-          description: "Alerting rule \"{{ $labels.alertname }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" produced {{ $value }} samples in last evaluation, which approaches the configured results limit.
-            If the produced results exceed the limit, the rule will be marked with an error and all its results will be discarded. 
-            Try increasing the results limit for the group or reducing the number of series produced by the rule. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups."
-
-      - alert: RecordingRuleResultsApproachingLimit
-        expr: |
-          (
-            vmalert_recording_rules_last_evaluation_samples
-              > on(group,file) group_left()
-            (vmalert_group_rule_results_limit * 0.9)
-          )
-            and on(group,file)
-          (vmalert_group_rule_results_limit > 0)
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "Recording rule {{ $labels.recording }} in group {{ $labels.group }} is approaching the configured results limit"
-          description: "Recording rule \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" produced {{ $value }} samples in last evaluation, which approaches the configured results limit.
-            If the produced results exceed the limit, the rule will be marked with an error and all its results will be discarded. 
-            Try increasing the results limit for the group or reducing the number of series produced by the rule. See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups."

deployment/docker/rules/alerts-vmauth.yml

@@ -56,20 +56,3 @@ groups:
           summary: "Too many errors served for user {{ $labels.username }} (instance {{ $labels.instance }})"
           description: "Requests from user {{ $labels.username }} are receiving errors.
             Please check the vmauth logs to verify that the configuration is correct and clients are sending valid requests."
-      - alert: InvalidAuthTokenRequestErrors
-        expr: sum(increase(vmauth_http_request_errors_total{reason="invalid_auth_token"}[5m])) without (instance, reason) > 0
-        for: 15m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "{{ $externalURL }}/d/nbuo5Mr4k?viewPanel=16&var-job={{ $labels.job }}"
-          summary: "vmauth {{ $labels.job }} is receiving many requests with invalid auth tokens"
-          description: |
-            vmauth {{ $labels.job }} received {{ $value }} requests with invalid auth tokens in the last 5 minutes.
-            This may indicate:
-              - credentials have been updated on vmauth but not on clients
-              - client misconfiguration or use of an expired token
-              - a brute-force attack.
-
-            Check vmauth metrics for longevity and scale of the issue. 
-            Check access log for detailed information: https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log

docs/victoriametrics/MetricsQL.md

@@ -1229,7 +1229,8 @@ Metric names are stripped from the resulting series. Add [keep_metric_names](#ke
 `buckets_limit(limit, buckets)` is a [transform function](#transform-functions), which limits the number
 of [histogram buckets](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) to the given `limit`. 
 
-The given `limit` should be greater than `0`. If it is less than `3`, it will be automatically raised to `3` to preserve the first and last buckets for better accuracy of min and max values.
+The result will preserve the first and the last bucket to improve accuracy for min and max values. 
+So, if the `limit` is greater than 0 and less than 3, the function will still return 3 buckets: the first bucket, the last bucket, and a selected bucket.
 
 See also [prometheus_buckets](#prometheus_buckets) and [histogram_quantile](#histogram_quantile).
 

docs/victoriametrics/README.md

@@ -401,7 +401,6 @@ Resources:
 
 * [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
 * [Cardinality explorer blog post](https://victoriametrics.com/blog/cardinality-explorer/).
-* [skills/victoriametrics-cardinality-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-cardinality-analysis/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
 
 ### Cardinality explorer statistic inaccuracy
 
@@ -1981,9 +1980,6 @@ in [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victori
 via [cache removal](https://docs.victoriametrics.com/victoriametrics/#cache-removal) procedure. This reset state endpoint can be protected via `-metricNamesStatsResetAuthKey`
 cmd-line flag. See [Security](https://docs.victoriametrics.com/victoriametrics/#security) for details.
 
-See [skills/victoriametrics-unused-metrics-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-unused-metrics-analysis/SKILL.md)
-for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis of unused metrics.
-
 ## Query tracing
 
 VictoriaMetrics supports query tracing, which can be used for determining bottlenecks during query processing.
@@ -2052,9 +2048,6 @@ Query tracing is allowed by default. It can be denied by passing `-denyQueryTrac
 * for query tracing - just click `Trace query` checkbox and re-run the query in order to investigate its' trace.
 * for exploring custom trace - go to the tab `Trace analyzer` and upload or paste JSON with trace information.
 
-See also [skills/vm-trace-analyzer](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/vm-trace-analyzer/SKILL.md)
-for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
-
 ## Cardinality limiter
 
 By default, VictoriaMetrics doesn't limit the number of stored time series. The limit can be enforced by setting the following command-line flags:

docs/victoriametrics/Troubleshooting.md

@@ -212,7 +212,6 @@ These are the most common reasons for slow data ingestion in VictoriaMetrics:
    - Reduce the number of active time series. The [official Grafana dashboards for VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#monitoring)
      contain a graph showing the number of active time series. Use the [cardinality explorer](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-explorer)
      to determine and fix the source of [high cardinality](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-cardinality).
-     See also [skills/victoriametrics-cardinality-analysis](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/victoriametrics-cardinality-analysis/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
 
    - Insert performance can degrade when the same time series arrives with labels in a different order.
      Ensure your ingestion client always sends labels in a consistent order for each series.
@@ -305,8 +304,7 @@ to logs.
 These are the solutions that exist for improving the performance of slow queries:
 
 - Investigating the bottleneck in query execution using [query tracing](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing).
-  It will show the percentage of time spent on each execution step and help understand the volume of processed data. See also [skills/vm-trace-analyzer](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/vm-trace-analyzer/SKILL.md)
-  for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) analysis.
+  It will show the percentage of time spent on each execution step and help understand the volume of processed data.
 
 - Adding more CPU and memory to VictoriaMetrics, so it may perform the slow query faster.
   If you use the cluster version of VictoriaMetrics, then migrating `vmselect` nodes to machines

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,33 +26,22 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
-## [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0)
-
-Release candidate
 
 * SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
 
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduces CPU usage by 10% at [sharding among remote storages](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages). See [#11113](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11113). Thanks to @bennf for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `64KiB` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
-* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
-* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
-* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
-* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
-* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_group_rule_results_limit` metric to indicate the number of alerts or recording results that a single rule within the group can produce. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
+* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
 
 * BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
-* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fixes unexpected rare rerouting. See [#11162](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11162).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly URL-encode `-vm-extra-label` values when building import requests, so special characters such as `&` don't get split into broken query parameters. See [#11144](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11144). Thanks to @immanuwell for contribution.
-* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/): ignore `enable.auto.offset.store` option in `kafka.consumer.topic.options`, since `vmagent` manages offset storage internally. Previously, setting this option could cause `vmagent` to stop committing Kafka messages. See [#11208](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11208).
+
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
 
 ## [v1.146.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.146.0)
 
@@ -113,6 +102,7 @@ Released at 2026-05-22
 
 * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
+* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `clusternative.tls` `vminsert` configuration flags for [multi-level cluster setups](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multi-level-cluster-setup). See [#10958](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10958).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)

docs/victoriametrics/stream-aggregation/README.md

@@ -42,8 +42,6 @@ Stream aggregation can be used in the following cases:
 * [Reducing the number of stored samples](#reducing-the-number-of-stored-samples)
 * [Reducing the number of stored series](#reducing-the-number-of-stored-series)
 
-See [skills/stream-aggregation-helper](https://github.com/VictoriaMetrics/skills/blob/main/plugins/diagnostics/skills/stream-aggregation-helper/SKILL.md) for [agent-assisted](https://docs.victoriametrics.com/ai-tools/#agent-skills) configuration.
-
 ## Statsd alternative
 
 Stream aggregation can be used as [statsd](https://github.com/statsd/statsd) alternative in the following cases:

docs/victoriametrics/vmagent.md

@@ -271,7 +271,7 @@ for the collected samples. Examples:
   
 ### Monitoring Data eXchange
 
-The Monitoring Data eXchange (MDX){{% available_from "v1.147.0" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
+The Monitoring Data eXchange (MDX){{% available_from "#" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
 
 To enable MDX, set `-remoteWrite.mdx.enable=true` for the target URL and `-remoteWrite.mdx.enable=false` for other URLs:
 

docs/victoriametrics/vmauth.md

@@ -270,7 +270,7 @@ users:
   url_prefix: "http://victoria-metrics:8428/"
 ```
 
-The `vm_access` claim is optional starting from {{% available_from "v1.147.0" %}}: when present it is used for [request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), and when absent the default tenant `0:0` is assumed for any `vm_access`-based placeholders. Routing can rely solely on other token claims via [JWT claim matching](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-matching).
+The `vm_access` claim is optional starting from {{% available_from "#" %}}: when present it is used for [request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), and when absent the default tenant `0:0` is assumed for any `vm_access`-based placeholders. Routing can rely solely on other token claims via [JWT claim matching](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-matching).
 
 For testing, skip signature verification with `skip_verify: true` (not recommended for production).
 
@@ -520,7 +520,7 @@ for dynamic URL rewriting based on `vm_access` claim fields.
 
 `vmauth` can dynamically rewrite{{% available_from "v1.137.0" %}} upstream URLs and request headers using values from the JWT `vm_access` claim. 
 This enables routing different users to different backends or tenants based solely on the JWT token, 
-without maintaining separate user configs per tenant. In addition `vm_access` claim could be defined at `jwt` section with `default_vm_access_claim` {{% available_from "v1.147.0" %}}.
+without maintaining separate user configs per tenant. In addition `vm_access` claim could be defined at `jwt` section with `default_vm_access_claim` {{% available_from "#" %}}.
 In this case, if JWT token doesn't have `vm_access` claim defined, value from `default_vm_access_claim` will be used for templaing.
 
 Example: minimal valid JWT. If vm_access is empty, tenant `0:0` is assumed and no additional filters are applied.
@@ -1323,17 +1323,9 @@ unauthorized_user:
 
 vmauth allows configuring access logs {{% available_from "v1.138.0" %}} printing per-user:
 ```yaml
-users:
-  - username: foo
-    password: bar
-    url_prefix: 'http://localhost:8428/'
-    # Log all requests to this user
-    access_log: {}
-```
-
-If you want to log requests with missing or invalid auth tokens, use unauthorized_user without configuring any URL routes{{% available_from "v1.147.0" %}}:
-```yaml
 unauthorized_user:
+  url_prefix: 'http://localhost:8428/'
+  # Log all requests to this user
   access_log: {}
 ```
 

lib/promscrape/targetstatus.go

@@ -752,67 +752,32 @@ func newCompressedLabels(src *promutil.Labels) *compressedLabels {
 	bb := compressedLabelsBufPool.Get()
 	bb.Grow(sizeNeeded)
 	// manually craft json in order to reduce memory allocations
-	bb.B = append(bb.B, '{')
-
-	escapeBB := compressedLabelsEscapePool.Get()
-	escapeBuf := escapeBB.B
-
+	fmt.Fprintf(bb, `{`) //nolint:errcheck
+	var tmpBuf []byte
 	for i, label := range srcLabels {
-		escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Name, true)
-		bb.Write(escapeBuf)   //nolint:errcheck
+		tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Name, true)
+		bb.Write(tmpBuf)      //nolint:errcheck
 		bb.Write([]byte(`:`)) //nolint:errcheck
-		escapeBuf = quicktemplate.AppendJSONString(escapeBuf[:0], label.Value, true)
-		bb.Write(escapeBuf) //nolint:errcheck
+		tmpBuf = quicktemplate.AppendJSONString(tmpBuf[:0], label.Value, true)
+		bb.Write(tmpBuf) //nolint:errcheck
 		if i+1 < len(srcLabels) {
 			bb.Write([]byte(`,`)) //nolint:errcheck
 		}
 	}
-	escapeBB.B = escapeBuf
-	compressedLabelsEscapePool.Put(escapeBB)
-
 	fmt.Fprint(bb, `}`) //nolint:errcheck
-
 	dst := zstd.CompressLevel(nil, bb.B, 1)
 
 	compressedLabelsBufPool.Put(bb)
 	cls := &compressedLabels{
-		hashKey: h,
-		data:    dst,
+		hashKey:      h,
+		addressLabel: strings.Clone(src.Get("__address__")),
+		jobLabel:     strings.Clone(src.Get("job")),
+		data:         dst,
 	}
-	addressLabelValue := src.Get("__address__")
-	jobLabelValue := src.Get("job")
-	addressLen := len(addressLabelValue)
-	jobLen := len(jobLabelValue)
-	// pre-allocate buffer to recuce GC pressure for tracking individual strings
-	packedBuf := make([]byte, 0, jobLen+addressLen+16)
-
-	packedBuf = append(packedBuf, addressLabelValue...)
-	cls.addressLabel = bytesutil.ToUnsafeString(packedBuf[:addressLen])
-
-	packedBuf = append(packedBuf, jobLabelValue...)
-	cls.jobLabel = bytesutil.ToUnsafeString(packedBuf[addressLen:])
-
-	packedBuf = appendHex16(packedBuf, uint64(uintptr(unsafe.Pointer(cls))))
-	cls.targetID = bytesutil.ToUnsafeString(packedBuf[addressLen+jobLen:])
-
+	cls.targetID = fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls)))
 	return cls
 }
 
-// appendHex16 is an equvialent for fmt.Sprintf("%016x", uintptr(unsafe.Pointer(cls)))
-// but with zero allocations
-func appendHex16(dst []byte, v uint64) []byte {
-	const hexChars = "0123456789abcdef"
-	var buf [16]byte
-	for i := 15; i >= 0; i-- {
-		buf[i] = hexChars[v&0xf]
-		v >>= 4
-	}
-	dst = append(dst, buf[:]...)
-	return dst
-}
-
-var compressedLabelsEscapePool = &bytesutil.ByteBufferPool{}
-
 func (cls *compressedLabels) getTargetID() string {
 	if cls == nil {
 		return ""

lib/promscrape/targetstatus_timing_test.go

@@ -1,42 +0,0 @@
-package promscrape
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
-)
-
-func BenchmarkNewCompressedLabels(b *testing.B) {
-	const numTargets = 1000
-	labelSet := func(idx int) *promutil.Labels {
-		return promutil.NewLabelsFromMap(map[string]string{
-			"__address__":                                           fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff),
-			"__meta_kubernetes_namespace":                           "default",
-			"__meta_kubernetes_pod_name":                            fmt.Sprintf("test-%d", idx),
-			"__meta_kubernetes_pod_uid":                             fmt.Sprintf("00000000-0000-0000-0000-%012d", idx),
-			"__meta_kubernetes_pod_ip":                              fmt.Sprintf("10.0.%d.%d", idx>>8, idx&0xff),
-			"__meta_kubernetes_pod_node_name":                       fmt.Sprintf("node-%d", idx%50),
-			"__meta_kubernetes_pod_label_app":                       "monitoring",
-			"__meta_kubernetes_pod_label_release":                   "prod",
-			"__meta_kubernetes_pod_annotation_prometheus_io_scrape": "true",
-			"__meta_kubernetes_pod_annotation_prometheus_io_port":   "9100",
-			"job":      "k8spod",
-			"instance": fmt.Sprintf("10.0.%d.%d:9100", idx>>8, idx&0xff),
-		})
-	}
-
-	labelss := make([]*promutil.Labels, numTargets)
-	for i := range labelss {
-		labelss[i] = labelSet(i)
-	}
-	b.ResetTimer()
-	b.ReportAllocs()
-	b.RunParallel(func(pb *testing.PB) {
-		var i int
-		for pb.Next() {
-			_ = newCompressedLabels(labelss[i%numTargets])
-			i++
-		}
-	})
-}

lib/timeserieslimits/timeseries_limits.go

@@ -30,18 +30,6 @@ var (
 	maxLabelsPerTimeseries = 40
 )
 
-// MustInit checks if limits are with-in supported range and prepares package for usage
-func MustInit(inputMaxLabelsPerTimeseries, inputMaxLabelNameLen, inputMaxLabelValueLen int) {
-	mustBeInRange := func(name string, limit int) {
-		if limit <= 0 || limit > math.MaxUint16 {
-			logger.Fatalf("incorrect limit: %q value: %d, must be in range 1..%d", name, limit, math.MaxUint16)
-		}
-	}
-	mustBeInRange("maxLabelNameLen", inputMaxLabelNameLen)
-	mustBeInRange("maxLabelValueLen", inputMaxLabelValueLen)
-	Init(inputMaxLabelsPerTimeseries, inputMaxLabelNameLen, inputMaxLabelValueLen)
-}
-
 // Init prepares package for usage
 func Init(inputMaxLabelsPerTimeseries, inputMaxLabelNameLen, inputMaxLabelValueLen int) {
 	maxLabelsPerTimeseries = inputMaxLabelsPerTimeseries