Makefile: fix synctest tests

The `synctest` experiment has been removed in go 1.26. This causes the test
build targets to fail. To fix that, this commit removes the
`GOEXPERIMENT=synctest` env var from the build rule recipes.

This forces the go tests to expect different synctest method names and function
signatures. Thus, the synctest tests also have been fixed.

VictoriaMetrics code uses `lib/fasttime` to get the current time. This does not
work with synctest tests since they require `time.Now()` to properly override
the current time. On the other hand, the rest of the tests (and especially
benchmarks) want to continue using `lib/fasttime`. Therefore the `synctest`
build label is used to link modified version of `lib/fasttime` when the synctest
tests need to be run.

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/build.yml

@@ -71,7 +71,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Build victoria-metrics for ${{ matrix.os }}-${{ matrix.arch }}
         run: make victoria-metrics-${{ matrix.os }}-${{ matrix.arch }}

.github/workflows/check-licenses.yml

@@ -21,9 +21,11 @@ jobs:
         id: go
         uses: actions/setup-go@v6
         with:
-          go-version: stable
+          go-version-file: 'go.mod'
           cache: false
 
+      - run: go version
+
       - name: Cache Go artifacts
         uses: actions/cache@v4
         with:

.github/workflows/codeql-analysis-go.yml

@@ -36,7 +36,8 @@ jobs:
         uses: actions/setup-go@v6
         with:
           cache: false
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Cache Go artifacts
         uses: actions/cache@v4

.github/workflows/test.yml

@@ -42,8 +42,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
-
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Cache golangci-lint
         uses: actions/cache@v4
@@ -81,7 +81,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Run tests
         run: GOGC=10 make ${{ matrix.scenario}}
@@ -107,7 +108,8 @@ jobs:
             go.sum
             Makefile
             app/**/Makefile
-          go-version: stable
+          go-version-file: 'go.mod'
+      - run: go version
 
       - name: Run integration tests
         run: make integration-test

Makefile

@@ -443,7 +443,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet -tags=synctest ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -452,19 +452,19 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	go test -tags=synctest ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	go test -tags=synctest -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test -tags=synctest ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -tags=synctest -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -tags=synctest -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test:
 	$(MAKE) apptest
@@ -490,11 +490,11 @@ integration-test-legacy: victoria-metrics vmbackup vmrestore
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
+	go test -bench=. ./lib/...
 	go test -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
+	CGO_ENABLED=0 go test -bench=. ./lib/...
 	CGO_ENABLED=0 go test -bench=. ./app/...
 
 vendor-update:
@@ -524,7 +524,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run --build-tags=synctest
 
 install-golangci-lint:
 	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)

app/vmagent/remotewrite/relabel.go

@@ -20,8 +20,7 @@ import (
 )
 
 var (
-	unparsedLabelsGlobal = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. "+
-		"Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage")
+	unparsedLabelsGlobal    = flagutil.NewArrayString("remoteWrite.label", "Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.")
 	relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabeling configs, which are applied "+
 		"to all the metrics before sending them to -remoteWrite.url. See also -remoteWrite.urlRelabelConfig. "+
 		"The path can point either to local file or to http url. "+

app/vmagent/remotewrite/remotewrite.go

@@ -1080,7 +1080,7 @@ func (rwctx *remoteWriteCtx) tryPushTimeSeriesInternal(tss []prompb.TimeSeries)
 	}()
 
 	if len(labelsGlobal) > 0 {
-		// Make a copy of tss before adding extra labels in order to prevent
+		// Make a copy of tss before adding extra labels to prevent
 		// from affecting time series for other remoteWrite.url configs.
 		rctx = getRelabelCtx()
 		v = tssPool.Get().(*[]prompb.TimeSeries)

app/vminsert/common/insert_ctx.go

@@ -182,6 +182,7 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
 		mm.Type = mmpb.Type
 		mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
 	}
+	ctx.mms = mms
 
 	err := vmstorage.AddMetadataRows(mms)
 	if err != nil {
@@ -206,6 +207,7 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
 		mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
 		mm.Type = mmpb.Type
 	}
+	ctx.mms = mms
 
 	err := vmstorage.AddMetadataRows(mms)
 	if err != nil {

app/vmselect/graphite/eval.go

@@ -9,6 +9,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -196,12 +197,17 @@ func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr g
 				pathExpression: safePathExpression(expr),
 			}
 			s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
-			t := timerpool.Get(30 * time.Second)
+
+			// A negative or zero duration will cause timer.C to return immediately
+			remainingTimeout := ec.deadline.Deadline() - fasttime.UnixTimestamp()
+			t := timerpool.Get(time.Duration(remainingTimeout) * time.Second)
 			defer timerpool.Put(t)
+
 			select {
 			case seriesCh <- s:
 			case <-t.C:
-				logger.Errorf("resource leak when processing the %s (full query: %s); please report this error to VictoriaMetrics developers",
+				logger.Errorf("reached timeout when processing the %s (full query: %s), it can be due to the amount of storageNodes configured in vmselect is more than vmselect’s available CPU count "+
+					"or vmselect is heavy loaded. Consider adding resources or increasing `-search.maxQueryDuration` or `timeout` parameter in the query.",
 					expr.AppendString(nil), ec.originalQuery)
 			}
 			return nil

app/vmselect/netstorage/netstorage.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"math"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -578,6 +579,7 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 		return
 	}
 	heap.Init(sbh)
+	var dedupSamples int
 	for {
 		sbs := sbh.sbs
 		top := sbs[0]
@@ -593,6 +595,7 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 		if n := equalSamplesPrefix(top, sbNext); n > 0 && dedupInterval > 0 {
 			// Skip n replicated samples at top if deduplication is enabled.
 			top.NextIdx = topNextIdx + n
+			dedupSamples += n
 		} else {
 			// Copy samples from top to dst with timestamps not exceeding tsNext.
 			top.NextIdx = topNextIdx + binarySearchTimestamps(top.Timestamps[topNextIdx:], tsNext)
@@ -607,8 +610,8 @@ func mergeSortBlocks(dst *Result, sbh *sortBlocksHeap, dedupInterval int64) {
 		}
 	}
 	timestamps, values := storage.DeduplicateSamples(dst.Timestamps, dst.Values, dedupInterval)
-	dedups := len(dst.Timestamps) - len(timestamps)
-	dedupsDuringSelect.Add(dedups)
+	dedupSamples += len(dst.Timestamps) - len(timestamps)
+	dedupsDuringSelect.Add(dedupSamples)
 	dst.Timestamps = timestamps
 	dst.Values = values
 }
@@ -634,7 +637,7 @@ func equalTimestampsPrefix(a, b []int64) int {
 
 func equalValuesPrefix(a, b []float64) int {
 	for i, v := range a {
-		if i >= len(b) || v != b[i] {
+		if i >= len(b) || math.Float64bits(v) != math.Float64bits(b[i]) {
 			return i
 		}
 	}

app/vmselect/netstorage/netstorage_test.go

@@ -1,8 +1,11 @@
 package netstorage
 
 import (
+	"math"
 	"reflect"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
 func TestMergeSortBlocks(t *testing.T) {
@@ -194,3 +197,111 @@ func TestMergeSortBlocks(t *testing.T) {
 		Values:     []float64{7, 24, 26},
 	})
 }
+
+func TestEqualSamplesPrefix(t *testing.T) {
+	f := func(a, b *sortBlock, expected int) {
+		t.Helper()
+
+		actual := equalSamplesPrefix(a, b)
+		if actual != expected {
+			t.Fatalf("unexpected result: got %d, want %d", actual, expected)
+		}
+	}
+
+	// Empty blocks
+	f(&sortBlock{}, &sortBlock{}, 0)
+
+	// Identical blocks
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, 4)
+
+	// Non-zero NextIdx
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+		NextIdx:    2,
+	}, &sortBlock{
+		Timestamps: []int64{10, 20, 3, 4},
+		Values:     []float64{50, 60, 7, 8},
+		NextIdx:    2,
+	}, 2)
+
+	// Non-zero NextIdx with mismatch
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+		NextIdx:    1,
+	}, &sortBlock{
+		Timestamps: []int64{10, 2, 3, 4},
+		Values:     []float64{50, 6, 7, 80},
+		NextIdx:    1,
+	}, 2)
+
+	// Different lengths
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3},
+		Values:     []float64{5, 6, 7},
+	}, 3)
+
+	// Timestamps diverge
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 30, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, 2)
+
+	// Values diverge
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 60, 7, 8},
+	}, 1)
+
+	// Zero matches
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, 6, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{5, 6, 7, 8},
+		Values:     []float64{1, 2, 3, 4},
+	}, 0)
+
+	// Compare staleness markers, matching
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, decimal.StaleNaN, 7, 8},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{5, decimal.StaleNaN, 7, 8},
+	}, 4)
+
+	// Special float values: +Inf, -Inf, 0, -0
+	f(&sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{math.Inf(1), math.Inf(-1), math.Copysign(0, +1), math.Copysign(0, -1)},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2, 3, 4},
+		Values:     []float64{math.Inf(1), math.Inf(-1), math.Copysign(0, +1), math.Copysign(0, -1)},
+	}, 4)
+
+	// Positive zero vs negative zero (bitwise different)
+	f(&sortBlock{
+		Timestamps: []int64{1, 2},
+		Values:     []float64{5, math.Copysign(0, +1)},
+	}, &sortBlock{
+		Timestamps: []int64{1, 2},
+		Values:     []float64{5, math.Copysign(0, -1)},
+	}, 1)
+}

app/vmselect/vmui/index.html

@@ -37,7 +37,7 @@
   <meta property="og:title" content="UI for VictoriaMetrics">
   <meta property="og:url" content="https://victoriametrics.com/">
   <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-C4RD5Sxk.js"></script>
+  <script type="module" crossorigin src="./assets/index-BTL1Td9z.js"></script>
   <link rel="modulepreload" crossorigin href="./assets/vendor-EZef-S_8.js">
   <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
   <link rel="stylesheet" crossorigin href="./assets/index-D7CzMv1O.css">

app/vmstorage/main.go

@@ -655,6 +655,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/metricID"}`, idbm.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSize)
 
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/tsid"}`, m.TSIDCacheSizeBytes)
@@ -670,6 +671,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSizeBytes)
 
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/tsid"}`, m.TSIDCacheSizeMaxBytes)
@@ -681,6 +683,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheSizeMaxBytes)
 
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="storage/tsid"}`, m.TSIDCacheRequests)
@@ -692,6 +695,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheRequests)
 	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheRequests)
+	metrics.WriteCounterUint64(w, `vm_cache_requests_total{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheRequests)
 
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/indexBlocks"}`, tm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="storage/tsid"}`, m.TSIDCacheMisses)
@@ -703,6 +707,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheMisses)
 	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheMisses)
+	metrics.WriteCounterUint64(w, `vm_cache_misses_total{type="indexdb/tagFiltersLoops"}`, idbm.LoopsPerDateTagFilterCacheMisses)
 
 	metrics.WriteCounterUint64(w, `vm_cache_resets_total{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheResets)
 

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.25.6 AS build-web-stage
+FROM golang:1.25.7 AS build-web-stage
 COPY build /build
 
 WORKDIR /build

app/vmui/packages/vmui/package.json

@@ -24,10 +24,10 @@
     "dayjs": "^1.11.19",
     "lodash.debounce": "^4.0.8",
     "marked": "^17.0.1",
-    "preact": "^10.28.2",
+    "preact": "^10.28.3",
     "qs": "^6.14.1",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^7.12.0",
+    "react-router-dom": "^7.13.0",
     "uplot": "^1.6.32",
     "vite": "^7.3.1",
     "web-vitals": "^5.1.0"
@@ -35,29 +35,29 @@
   "devDependencies": {
     "@eslint/eslintrc": "^3.3.3",
     "@eslint/js": "^9.39.2",
-    "@preact/preset-vite": "^2.10.2",
+    "@preact/preset-vite": "^2.10.3",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
     "@types/lodash.debounce": "^4.0.9",
-    "@types/node": "^25.0.8",
+    "@types/node": "^25.2.0",
     "@types/qs": "^6.14.0",
-    "@types/react": "^19.2.8",
+    "@types/react": "^19.2.10",
     "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@typescript-eslint/eslint-plugin": "^8.53.0",
-    "@typescript-eslint/parser": "^8.53.0",
+    "@typescript-eslint/eslint-plugin": "^8.54.0",
+    "@typescript-eslint/parser": "^8.54.0",
     "cross-env": "^10.1.0",
     "eslint": "^9.39.2",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-unused-imports": "^4.3.0",
-    "globals": "^17.0.0",
+    "globals": "^17.3.0",
     "http-proxy-middleware": "^3.0.5",
-    "jsdom": "^27.4.0",
+    "jsdom": "^28.0.0",
     "postcss": "^8.5.6",
     "rollup-plugin-visualizer": "^6.0.5",
-    "sass-embedded": "^1.97.2",
+    "sass-embedded": "^1.97.3",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.17"
+    "vitest": "^4.0.18"
   },
   "browserslist": {
     "production": [

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/QueryEditorAutocomplete.tsx

@@ -6,6 +6,7 @@ import { QueryContextType } from "../../../types";
 import { AUTOCOMPLETE_LIMITS } from "../../../constants/queryAutocomplete";
 import { QueryEditorAutocompleteProps } from "./QueryEditor";
 import { getExprLastPart, getValueByContext, getContext } from "./autocompleteUtils";
+import { extractCurrentLabel, extractLabelMatchers, extractMetric, splitByCursor } from "./utils/parser";
 
 const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   value,
@@ -20,45 +21,39 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
   const metricsqlFunctions = useGetMetricsQL(includeFunctions);
 
   const values = useMemo(() => {
-    if (caretPosition[0] !== caretPosition[1]) return { beforeCursor: value, afterCursor: "" };
-    const beforeCursor = value.substring(0, caretPosition[0]);
-    const afterCursor = value.substring(caretPosition[1]);
-    return { beforeCursor, afterCursor };
+    return splitByCursor(value, caretPosition);
   }, [value, caretPosition]);
 
-  const exprLastPart = useMemo(() => getExprLastPart(values.beforeCursor), [values]);
+  const exprLastPart = useMemo(() => {
+    return getExprLastPart(values.beforeCursor);
+  }, [values.beforeCursor]);
 
   const metric = useMemo(() => {
-    const regex1 = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
-    const matchAlt = [...exprLastPart.matchAll(regex1)];
-    if (matchAlt.length > 0 && matchAlt[0].groups && matchAlt[0].groups.metricName) {
-      return matchAlt[0].groups.metricName;
-    }
-
-    const regex2 = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
-    const match = [...exprLastPart.matchAll(regex2)];
-    if (match.length > 0 && match[0].groups && match[0].groups.metricName) {
-      return match[0].groups.metricName;
-    }
-
-    return "";
+    return extractMetric(exprLastPart);
   }, [exprLastPart]);
 
   const label = useMemo(() => {
-    const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
-    const match = exprLastPart.match(regexp);
-    return match ? match[match.length - 1] : "";
+    return extractCurrentLabel(exprLastPart);
   }, [exprLastPart]);
 
-  const context = useMemo(() => getContext(values.beforeCursor, metric, label), [values, metric, label]);
+  const context = useMemo(() => {
+    return getContext(values.beforeCursor, metric, label);
+  }, [values.beforeCursor, metric, label]);
 
-  const valueByContext = useMemo(() => getValueByContext(values.beforeCursor), [values.beforeCursor]);
+  const valueByContext = useMemo(() => {
+    return getValueByContext(values.beforeCursor);
+  }, [values.beforeCursor]);
+
+  const labelMatchers = useMemo(() => {
+    return extractLabelMatchers(values.beforeCursor, label);
+  }, [values.beforeCursor, label]);
 
   const { metrics, labels, labelValues, loading } = useFetchQueryOptions({
     valueByContext,
     metric,
     label,
     context,
+    labelMatchers,
   });
 
   const options = useMemo(() => {
@@ -72,18 +67,18 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
       default:
         return [];
     }
-  }, [context, metrics, labels, labelValues]);
+  }, [context, metrics, labels, labelValues, metricsqlFunctions]);
 
   const handleSelect = useCallback((insert: string) => {
     // Find the start and end of valueByContext in the query string
-    const value = values.beforeCursor;
+    const beforeCursor = values.beforeCursor;
     let valueAfterCursor = values.afterCursor;
-    const startIndexOfValueByContext = value.lastIndexOf(valueByContext, caretPosition[0]);
+    const startIndexOfValueByContext = beforeCursor.lastIndexOf(valueByContext, caretPosition[0]);
     const endIndexOfValueByContext = startIndexOfValueByContext + valueByContext.length;
 
     // Split the original string into parts: before, during, and after valueByContext
-    const beforeValueByContext = value.substring(0, startIndexOfValueByContext);
-    const afterValueByContext = value.substring(endIndexOfValueByContext);
+    const beforeValueByContext = beforeCursor.substring(0, startIndexOfValueByContext);
+    const afterValueByContext = beforeCursor.substring(endIndexOfValueByContext);
 
     // Add quotes around the value if the context is labelValue
     if (context === QueryContextType.labelValue) {
@@ -104,7 +99,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
     // Assemble the new value with the inserted text
     const newVal = `${beforeValueByContext}${insert}${afterValueByContext}${valueAfterCursor}`;
     onSelect(newVal, beforeValueByContext.length + insert.length);
-  }, [values]);
+  }, [values.beforeCursor, values.afterCursor, valueByContext, caretPosition, context, onSelect]);
 
   useEffect(() => {
     if (!anchorEl.current) {
@@ -142,7 +137,7 @@ const QueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
 
     span.remove();
     marker.remove();
-  }, [anchorEl, caretPosition, hasHelperText]);
+  }, [anchorEl, caretPosition, hasHelperText, values.beforeCursor, values.afterCursor]);
 
   return (
     <>

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/utils/parser.test.ts

@@ -0,0 +1,139 @@
+import { describe, it, expect } from "vitest";
+import {
+  splitByCursor,
+  extractMetric,
+  extractCurrentLabel,
+  extractLabelMatchers,
+} from "./parser";
+
+describe("splitByCursor", () => {
+  it("splits by caret when selection is collapsed", () => {
+    const res = splitByCursor("abcdef", [2, 2]);
+    expect(res).toEqual({ beforeCursor: "ab", afterCursor: "cdef" });
+  });
+
+  it("returns whole value as beforeCursor when selection is not collapsed", () => {
+    const res = splitByCursor("abcdef", [1, 3]);
+    expect(res).toEqual({ beforeCursor: "abcdef", afterCursor: "" });
+  });
+
+  it("handles caret at 0", () => {
+    const res = splitByCursor("abc", [0, 0]);
+    expect(res).toEqual({ beforeCursor: "", afterCursor: "abc" });
+  });
+
+  it("handles caret at end", () => {
+    const res = splitByCursor("abc", [3, 3]);
+    expect(res).toEqual({ beforeCursor: "abc", afterCursor: "" });
+  });
+
+  it("treats reversed selection as non-collapsed (browser may return [end,start])", () => {
+    const res = splitByCursor("abcdef", [4, 2]);
+    expect(res).toEqual({ beforeCursor: "abcdef", afterCursor: "" });
+  });
+});
+
+describe("extractMetric", () => {
+  it("extracts metric from plain selector", () => {
+    expect(extractMetric("kube_pod_info{job=\"x\"}")).toBe("kube_pod_info");
+  });
+
+  it("extracts metric from plain expr with leading spaces", () => {
+    expect(extractMetric("   http_requests_total")).toBe("http_requests_total");
+  });
+
+  it("extracts metric from expr with braces right after metric", () => {
+    expect(extractMetric("foo_bar{a=\"b\"}")).toBe("foo_bar");
+  });
+
+  it("extracts metric before grouping modifiers (by/without/on/ignoring)", () => {
+    expect(extractMetric("sum(kube_pod_info) by (pod)")).toBe("kube_pod_info");
+    expect(extractMetric("sum(kube_pod_info) without (pod)")).toBe("kube_pod_info");
+    expect(extractMetric("sum(kube_pod_info) on (pod)")).toBe("kube_pod_info");
+    expect(extractMetric("sum(kube_pod_info) ignoring (pod)")).toBe("kube_pod_info");
+  });
+
+  it("returns empty string when no metric found", () => {
+    expect(extractMetric("{job=\"x\"}")).toBe("");
+    expect(extractMetric("")).toBe("");
+    expect(extractMetric("()")).toBe("");
+  });
+});
+
+describe("extractCurrentLabel", () => {
+  it("returns last label before operator", () => {
+    expect(extractCurrentLabel("metric{job=\"foo\", instance=\"bar\"}")).toBe(
+      "instance"
+    );
+  });
+
+  it("supports spaces around operator", () => {
+    expect(extractCurrentLabel("metric{job=\"foo\", instance = \"bar\"}")).toBe(
+      "instance"
+    );
+  });
+
+  it("supports regexp operators", () => {
+    expect(extractCurrentLabel("metric{pod=~\"api-.*\",namespace=\"dev\"}")).toBe(
+      "namespace"
+    );
+  });
+
+  it("supports label chars : - . /", () => {
+    expect(extractCurrentLabel("m{foo-bar.baz/qux=\"1\"}")).toBe("foo-bar.baz/qux");
+  });
+
+  it("returns empty string when no label pattern", () => {
+    expect(extractCurrentLabel("metric{}").trim()).toBe("");
+    expect(extractCurrentLabel("metric")).toBe("");
+  });
+});
+
+describe("extractLabelMatchers", () => {
+  it("returns all matchers (quoted only)", () => {
+    const expr = "metric{job=\"foo\", instance=\"bar\"}";
+    expect(extractLabelMatchers(expr)).toEqual(["job=\"foo\"", "instance=\"bar\""]);
+  });
+
+  it("keeps original spacing", () => {
+    const expr = "metric{ job = \"foo\" , instance = \"bar\" }";
+    expect(extractLabelMatchers(expr)).toEqual(["job = \"foo\"", "instance = \"bar\""]);
+  });
+
+  it("supports !=, =~, !~", () => {
+    const expr = "m{env!=\"prod\",pod=~\"api-.*\",zone!~\"eu-.*\"}";
+    expect(extractLabelMatchers(expr)).toEqual([
+      "env!=\"prod\"",
+      "pod=~\"api-.*\"",
+      "zone!~\"eu-.*\"",
+    ]);
+  });
+
+  it("excludes only the specified currentLabel matcher (exact label, not prefix)", () => {
+    const expr = "m{job=\"foo\", instance=\"bar\", pod=~\"api-.*\"}";
+    expect(extractLabelMatchers(expr, "instance")).toEqual([
+      "job=\"foo\"",
+      "pod=~\"api-.*\"",
+    ]);
+  });
+
+  it("does not exclude other labels that share a prefix with currentLabel", () => {
+    const expr = "m{instance=\"bar\", insight=\"x\"}";
+    expect(extractLabelMatchers(expr, "insight")).toEqual(["instance=\"bar\""]);
+  });
+
+  it("excludes currentLabel matcher even with spaces around operator", () => {
+    const expr = "m{job=\"foo\", instance = \"bar\"}";
+    expect(extractLabelMatchers(expr, "instance")).toEqual(["job=\"foo\""]);
+  });
+
+  it("returns [] when no matchers", () => {
+    expect(extractLabelMatchers("m{}")).toEqual([]);
+    expect(extractLabelMatchers("m")).toEqual([]);
+  });
+
+  it("does not include unclosed quotes", () => {
+    const expr = "m{job=\"foo\", instance=\"ba";
+    expect(extractLabelMatchers(expr)).toEqual(["job=\"foo\""]);
+  });
+});

app/vmui/packages/vmui/src/components/Configurators/QueryEditor/utils/parser.ts

@@ -0,0 +1,52 @@
+
+export const splitByCursor = (
+  value: string,
+  caret: [number, number]
+) => {
+  if (caret[0] !== caret[1]) {
+    return { beforeCursor: value, afterCursor: "" };
+  }
+
+  return {
+    beforeCursor: value.substring(0, caret[0]),
+    afterCursor: value.substring(caret[1]),
+  };
+};
+
+
+export const extractMetric = (expr: string): string => {
+  const fnRegex = /\w+\((?<metricName>[^)]+)\)\s+(by|without|on|ignoring)\s*\(\w*/gi;
+  const fnMatch = [...expr.matchAll(fnRegex)];
+
+  if (fnMatch[0]?.groups?.metricName) {
+    return fnMatch[0].groups.metricName;
+  }
+
+  const plainRegex = /^\s*\b(?<metricName>[^{}(),\s]+)(?={|$)/g;
+  const match = [...expr.matchAll(plainRegex)];
+  return match[0]?.groups?.metricName || "";
+};
+
+export const extractCurrentLabel = (expr: string): string => {
+  const regexp = /[a-z_:-][\w\-.:/]*\b(?=\s*(=|!=|=~|!~))/g;
+  const match = expr.match(regexp);
+  return match ? match[match.length - 1] : "";
+};
+
+
+export const extractLabelMatchers = (
+  expr: string,
+  currentLabel?: string
+): string[] => {
+  const regexp = /([a-z_:-][\w\-.:/]*)\s*(?:=|!=|=~|!~)\s*"[^"]*"/g;
+
+  const matches = [...expr.matchAll(regexp)];
+  // m[1] = label name
+  // m[0] = full matcher string
+
+  if (!currentLabel) return matches.map(m => m[0]);
+
+  return matches
+    .filter(m => m[1] !== currentLabel)
+    .map(m => m[0]);
+};

app/vmui/packages/vmui/src/hooks/useFetchQueryOptions.tsx

@@ -30,6 +30,7 @@ type FetchQueryArguments = {
   metric: string;
   label: string;
   context: QueryContextType
+  labelMatchers: string[];
 }
 
 const icons = {
@@ -38,7 +39,7 @@ const icons = {
   [TypeData.labelValue]: <ValueIcon/>,
 };
 
-export const useFetchQueryOptions = ({ valueByContext, metric, label, context }: FetchQueryArguments) => {
+export const useFetchQueryOptions = ({ valueByContext, metric, label, context, labelMatchers }: FetchQueryArguments) => {
   const { serverUrl } = useAppState();
   const { period: { start, end } } = useTimeState();
   const { autocompleteCache } = useQueryState();
@@ -143,17 +144,19 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
     setLabels([]);
 
     const metricEscaped = escapeDoubleQuotes(metric);
+    const matchMetric = metric ? `__name__="${metricEscaped}"` : "";
+    const matchValue = [matchMetric, ...labelMatchers].filter(Boolean).join(",");
 
     fetchData({
       value,
       urlSuffix: "labels",
       setter: setLabels,
       type: TypeData.label,
-      params: getQueryParams(metric ? { "match[]": `{__name__="${metricEscaped}"}` } : undefined)
+      params: getQueryParams({ "match[]": `{${matchValue}}` })
     });
 
     return () => abortControllerRef.current?.abort();
-  }, [serverUrl, value, context, metric]);
+  }, [serverUrl, value, context, metric, labelMatchers]);
 
   // fetch labelValues
   useEffect(() => {
@@ -166,7 +169,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
     const valueReEscaped = escapeDoubleQuotes(escapeRegexp(value));
     const matchMetric = metric ? `__name__="${metricEscaped}"` : "";
     const matchLabel = `${label}=~".*${valueReEscaped}.*"`;
-    const matchValue = [matchMetric, matchLabel].filter(Boolean).join(",");
+    const matchValue = [matchMetric, ...labelMatchers, matchLabel].filter(Boolean).join(",");
 
     fetchData({
       value,
@@ -177,7 +180,7 @@ export const useFetchQueryOptions = ({ valueByContext, metric, label, context }:
     });
 
     return () => abortControllerRef.current?.abort();
-  }, [serverUrl, value, context, metric, label]);
+  }, [serverUrl, value, context, metric, label, labelMatchers]);
 
   return {
     metrics,

app/vmui/packages/vmui/src/pages/TopQueries/hooks/useFetchTopQueries.ts

@@ -26,8 +26,7 @@ const getQueryUrl = (row: TopQuery, timeRange: string) => {
 };
 
 const processResponse = (data: TopQueriesData) => {
-  const list = ["topByAvgDuration", "topByCount", "topBySumDuration"] as (keyof TopQueriesData)[];
-
+  const list = ["topByAvgDuration", "topByCount", "topBySumDuration", "topByAvgMemoryUsage"] as (keyof TopQueriesData)[];
   list.forEach(key => {
     const target = data[key] as TopQuery[];
     if (!Array.isArray(target)) return;

app/vmui/packages/vmui/src/pages/TopQueries/index.tsx

@@ -178,6 +178,17 @@ const TopQueries: FC = () => {
               { key: "count" }
             ]}
           />
+          <TopQueryPanel
+            rows={data.topByAvgMemoryUsage}
+            title={"Queries with most memory to execute"}
+            columns={[
+              { key: "query" },
+              { key: "avgMemoryBytes", title: "avg memory usage, bytes" },
+              { key: "timeRange", sortBy: "timeRangeSeconds", title: "query time interval" },
+              { key: "count" }
+            ]}
+            defaultOrderBy={"avgMemoryBytes"}
+          />
         </div>
       </>)}
     </div>

app/vmui/packages/vmui/src/types/index.ts

@@ -92,6 +92,7 @@ export interface TopQuery {
   query: string;
   timeRangeSeconds: number;
   sumDurationSeconds: number;
+  avgMemoryBytes: number;
   timeRange: string;
   url?: string;
 }
@@ -107,6 +108,7 @@ export interface TopQueriesData extends TopQueryStats {
   topByAvgDuration: TopQuery[];
   topByCount: TopQuery[];
   topBySumDuration: TopQuery[];
+  topByAvgMemoryUsage: TopQuery[];
   error?: string;
 }
 

app/vmui/packages/vmui/vite.config.ts

@@ -10,25 +10,19 @@ const getProxy = (): Record<string, ProxyOptions> | undefined => {
     return undefined;
   }
 
+  const commonProxy: ProxyOptions = {
+    target: "https://play.victoriametrics.com/select/0",
+    changeOrigin: true,
+    configure: (proxy) => {
+      proxy.on("error", (err) => {
+        console.error("[proxy error]", err.message);
+      });
+    },
+  };
+
   return {
-    "^/(api|vmalert)/.*": {
-      target: "https://play.victoriametrics.com/select/0/prometheus",
-      changeOrigin: true,
-      configure: (proxy) => {
-        proxy.on("error", (err) => {
-          console.error("[proxy error]", err.message);
-        });
-      },
-    },
-    "/prometheus/vmui/config.json": {
-      target: "https://play.victoriametrics.com/select/0",
-      changeOrigin: true,
-      configure: (proxy) => {
-        proxy.on("error", (err) => {
-          console.error("[proxy error]", err.message);
-        });
-      },
-    },
+    "^/prometheus/(api|vmalert)/.*": { ...commonProxy },
+    "/prometheus/vmui/config.json": { ...commonProxy },
   };
 };
 

dashboards/victoriametrics-cluster.json

@@ -6363,7 +6363,7 @@
             "type": "prometheus",
             "uid": "$ds"
           },
-          "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).",
+          "description": "Shows the number of ongoing Cluster Native insert API operations sent to `vmstorage`, where:\n* `max` - the maximum number of calls that can be processed in parallel;\n* `current` - Shows the number of ongoing insert operations being processed by the storage.\n\nEach successful API call results in a flush to disk. The `max` value is controlled by the `-maxConcurrentInserts` flag set on the storage (2 * cgroup.AvailableCPUs() by default). \n\nIf `current` consistently reaches `max`, it indicates that the storage is overloaded and likely requires more CPU (see CPU usage) or disks with higher IOPS (see disk read/write panels in the Resource Usage row)",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -6500,7 +6500,7 @@
               "refId": "B"
             }
           ],
-          "title": "Concurrent flushes on disk ($instance)",
+          "title": "Concurrent inserts ($instance)",
           "type": "timeseries"
         },
         {
@@ -11207,7 +11207,13 @@
                   "mode": "off"
                 }
               },
-              "links": [],
+              "links": [
+                {
+                  "targetBlank": true,
+                  "title": "Source Code",
+                  "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"
+                }
+              ],
               "mappings": [],
               "min": 0,
               "thresholds": {
@@ -11262,12 +11268,12 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version,location) > 0",
               "format": "time_series",
               "hide": false,
               "interval": "5m",
               "intervalFactor": 1,
-              "legendFormat": "__auto",
+              "legendFormat": "{{instance}} ({{job}}) - {{level}}: {{location}}",
               "range": true,
               "refId": "A"
             }

dashboards/victoriametrics.json

@@ -1508,8 +1508,8 @@
           "links": [
             {
               "targetBlank": true,
-              "title": "asd",
-              "url": "asd"
+              "title": "Drilldown",
+              "url": "/d/wNf0q_kZk?viewPanel=154&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
             }
           ],
           "mappings": [],
@@ -8531,8 +8531,8 @@
               "links": [
                 {
                   "targetBlank": true,
-                  "title": "Drilldown",
-                  "url": "/d/wNf0q_kZk?viewPanel=154&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
+                  "title": "Source Code",
+                  "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"
                 }
               ],
               "mappings": [],
@@ -8589,11 +8589,11 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version, location) > 0",
               "format": "time_series",
               "interval": "5m",
               "intervalFactor": 1,
-              "legendFormat": "__auto",
+              "legendFormat": "{{instance}} ({{job}}) - {{level}}: {{location}}",
               "range": true,
               "refId": "A"
             }

dashboards/vm/victoriametrics-cluster.json

@@ -6364,7 +6364,7 @@
             "type": "victoriametrics-metrics-datasource",
             "uid": "$ds"
           },
-          "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU (see CPU usage) or disks with more IOPS (see disk writes and reads panels in Resource Usage row).",
+          "description": "Shows the number of ongoing Cluster Native insert API operations sent to `vmstorage`, where:\n* `max` - the maximum number of calls that can be processed in parallel;\n* `current` - Shows the number of ongoing insert operations being processed by the storage.\n\nEach successful API call results in a flush to disk. The `max` value is controlled by the `-maxConcurrentInserts` flag set on the storage (2 * cgroup.AvailableCPUs() by default). \n\nIf `current` consistently reaches `max`, it indicates that the storage is overloaded and likely requires more CPU (see CPU usage) or disks with higher IOPS (see disk read/write panels in the Resource Usage row)",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -6501,7 +6501,7 @@
               "refId": "B"
             }
           ],
-          "title": "Concurrent flushes on disk ($instance)",
+          "title": "Concurrent inserts ($instance)",
           "type": "timeseries"
         },
         {
@@ -11208,7 +11208,13 @@
                   "mode": "off"
                 }
               },
-              "links": [],
+              "links": [
+                {
+                  "targetBlank": true,
+                  "title": "Source Code",
+                  "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"
+                }
+              ],
               "mappings": [],
               "min": 0,
               "thresholds": {
@@ -11263,12 +11269,12 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version,location) > 0",
               "format": "time_series",
               "hide": false,
               "interval": "5m",
               "intervalFactor": 1,
-              "legendFormat": "__auto",
+              "legendFormat": "{{instance}} ({{job}}) - {{level}}: {{location}}",
               "range": true,
               "refId": "A"
             }

dashboards/vm/victoriametrics.json

@@ -1509,8 +1509,8 @@
           "links": [
             {
               "targetBlank": true,
-              "title": "asd",
-              "url": "asd"
+              "title": "Drilldown",
+              "url": "/d/wNf0q_kZk_vm?viewPanel=154&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
             }
           ],
           "mappings": [],
@@ -8532,8 +8532,8 @@
               "links": [
                 {
                   "targetBlank": true,
-                  "title": "Drilldown",
-                  "url": "/d/wNf0q_kZk_vm?viewPanel=154&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
+                  "title": "Source Code",
+                  "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"
                 }
               ],
               "mappings": [],
@@ -8590,11 +8590,11 @@
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, location) > 0",
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version, location) > 0",
               "format": "time_series",
               "interval": "5m",
               "intervalFactor": 1,
-              "legendFormat": "__auto",
+              "legendFormat": "{{instance}} ({{job}}) - {{level}}: {{location}}",
               "range": true,
               "refId": "A"
             }

dashboards/vm/vmagent.json

@@ -1227,7 +1227,13 @@
               "mode": "off"
             }
           },
-          "links": [],
+          "links": [
+            {
+              "targetBlank": true,
+              "title": "Drilldown",
+              "url": "/d/G7Z9GzMGz_vm?viewPanel=162&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
+            }
+          ],
           "mappings": [],
           "min": 0,
           "thresholds": {
@@ -8296,6 +8302,124 @@
           ],
           "title": "Persistent queue Full ETA ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "victoriametrics-metrics-datasource",
+            "uid": "$ds"
+          },
+          "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "bars",
+                "fillOpacity": 100,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [
+                {
+                  "targetBlank": true,
+                  "title": "Source Code",
+                  "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"
+                }
+              ],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 461
+          },
+          "id": 154,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "victoriametrics-metrics-datasource",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": true,
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version, location) > 0",
+              "format": "time_series",
+              "interval": "5m",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}} - {{level}}: {{location}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Logging rate",
+          "type": "timeseries"
         }
       ],
       "title": "Drilldown",

dashboards/vmagent.json

@@ -1226,7 +1226,13 @@
               "mode": "off"
             }
           },
-          "links": [],
+          "links": [
+            {
+              "targetBlank": true,
+              "title": "Drilldown",
+              "url": "/d/G7Z9GzMGz?viewPanel=162&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}"
+            }
+          ],
           "mappings": [],
           "min": 0,
           "thresholds": {
@@ -8295,6 +8301,124 @@
           ],
           "title": "Persistent queue Full ETA ($instance)",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$ds"
+          },
+          "description": "Shows the rate of logging the messages by their level. Unexpected spike in rate is a good reason to check logs.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "barWidthFactor": 0.6,
+                "drawStyle": "bars",
+                "fillOpacity": 100,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "never",
+                "showValues": false,
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [
+                {
+                  "targetBlank": true,
+                  "title": "Source Code",
+                  "url": "https://l2s.victoriametrics.com/?app_version=${__field.labels.app_version}&location=${__field.labels.location}"
+                }
+              ],
+              "mappings": [],
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 461
+          },
+          "id": 154,
+          "options": {
+            "legend": {
+              "calcs": [
+                "mean",
+                "lastNotNull",
+                "max"
+              ],
+              "displayMode": "table",
+              "placement": "bottom",
+              "showLegend": true,
+              "sortBy": "Last *",
+              "sortDesc": true
+            },
+            "tooltip": {
+              "hideZeros": false,
+              "mode": "multi",
+              "sort": "desc"
+            }
+          },
+          "pluginVersion": "12.2.0",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$ds"
+              },
+              "editorMode": "code",
+              "exemplar": true,
+              "expr": "sum(rate(vm_log_messages_total{job=~\"$job\",instance=~\"$instance\", level!=\"info\"}[$__rate_interval])) by (job, instance, level, app_version, location) > 0",
+              "format": "time_series",
+              "interval": "5m",
+              "intervalFactor": 1,
+              "legendFormat": "{{instance}} - {{level}}: {{location}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Logging rate",
+          "type": "timeseries"
         }
       ],
       "title": "Drilldown",

deployment/docker/Makefile

@@ -3,11 +3,11 @@
 DOCKER_REGISTRIES ?= docker.io quay.io
 DOCKER_NAMESPACE ?= victoriametrics
 
-ROOT_IMAGE ?= alpine:3.23.2
+ROOT_IMAGE ?= alpine:3.23.3
 ROOT_IMAGE_SCRATCH ?= scratch
-CERTS_IMAGE := alpine:3.23.2
+CERTS_IMAGE := alpine:3.23.3
 
-GO_BUILDER_IMAGE := golang:1.25.6
+GO_BUILDER_IMAGE := golang:1.25.7
 
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.134.0
+    image: victoriametrics/vmagent:v1.135.0
     depends_on:
       - "vmauth"
     ports:
@@ -37,14 +37,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.134.0-cluster
+    image: victoriametrics/vmstorage:v1.135.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.134.0-cluster
+    image: victoriametrics/vmstorage:v1.135.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -54,7 +54,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.134.0-cluster
+    image: victoriametrics/vminsert:v1.135.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -63,7 +63,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.134.0-cluster
+    image: victoriametrics/vminsert:v1.135.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -75,7 +75,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.134.0-cluster
+    image: victoriametrics/vmselect:v1.135.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -85,7 +85,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.134.0-cluster
+    image: victoriametrics/vmselect:v1.135.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -100,7 +100,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.134.0
+    image: victoriametrics/vmauth:v1.135.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -114,7 +114,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.134.0
+    image: victoriametrics/vmalert:v1.135.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.134.0
+    image: victoriametrics/vmagent:v1.135.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.134.0
+    image: victoriametrics/victoria-metrics:v1.135.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -54,7 +54,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.134.0
+    image: victoriametrics/vmalert:v1.135.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/rules/alerts.yml

@@ -136,3 +136,16 @@ groups:
           description: "High rate of slow inserts on \"{{ $labels.instance }}\" may be a sign of resource exhaustion
             for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series.
             See also https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3976#issuecomment-1476883183"
+
+      - alert: MetadataCacheUtilizationIsTooHigh
+        expr: |
+          vm_metrics_metadata_storage_size_bytes / vm_metrics_metadata_storage_max_size_bytes > 0.95
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Metadata cache capacity on {{ $labels.instance }} (job={{ $labels.job }}) is utilized for more than 95% for the last 15min"
+          description: "Metadata cache stores meta information about ingested time series - see https://docs.victoriametrics.com/victoriametrics/#metrics-metadata.
+           When cache is overutilized, the oldest entries will be dropped out automatically. It may result into incomplete
+           response for /api/v1/metadata API calls. It doesn't impact regular queries or alerts. Cache size is controlled
+           via -storage.maxMetadataStorageSize cmd-line flag."

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.134.0
+    image: victoriametrics/vmagent:v1.135.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.134.0
+    image: victoriametrics/victoria-metrics:v1.135.0
     ports:
       - 8428:8428
     volumes:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.134.0
+    image: victoriametrics/vmalert:v1.135.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.28.5
+    image: victoriametrics/vmanomaly:v1.28.7
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,18 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.28.7
+Released: 2026-02-09
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.4.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v142) to [v1.4.3](https://docs.victoriametrics.com/anomaly-detection/ui/#v143), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v143) for details.
+
+- BUGFIX: Resolved an issue with `Logs/Traces` datasource type in the vmanomaly UI where the `step` parameter from UI state wasn't properly passed to the backend, sometimes resulting in 422 errors when the backend expected a string (e.g. "10s") but received a number (e.g. 10) instead.
+
+## v1.28.6
+Released: 2026-01-27
+
+- IMPROVEMENT: Support additional backward-compatible [CLI arguments](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments) formats, including key-value pairs (`key=value`, `-key=value`, `--key=value`), value shortcuts for boolean flag values (e.g. `dryRun=1`, `-dryRun true`, `--dryRun false`, `--dryRun 0`) and all respective combinations.
+
 ## v1.28.5
 Released: 2026-01-17
 

docs/anomaly-detection/FAQ.md

@@ -419,7 +419,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.28.5
+    image: victoriametrics/vmanomaly:v1.28.7
     # ...
     restart: always
     volumes:
@@ -637,7 +637,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.28.5 && docker image tag victoriametrics/vmanomaly:v1.28.5 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.28.6 && docker image tag victoriametrics/vmanomaly:v1.28.7 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/Migration.md

@@ -45,8 +45,8 @@ There are 2 types of compatibilitity to consider when migrating in stateful mode
 
 | Group start | Group end | Compatibility | Notes |
 |---------|--------- |------------|-------|
-| [v1.28.5](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1285) | Latest* | Fully Compatible | Just a placeholder for new releases |
-| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.5](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1285) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
+| [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Latest* | Fully Compatible | Just a placeholder for new releases |
+| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
 | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |
 | [v1.25.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1251) | [v1.25.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1252) | Fully Compatible | In [v1.25.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1251) there was a change to `vmanomaly.db` metadata database format, so migrating from v1.24.0-v1.25.0 requires deletion of a state, see note above the table |
 | [v1.24.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1241) | [v1.25.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1250) | Partially Compatible* | In [v1.25.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1250) there were changes to **data dump layout** and to `online_quantile` and `isolation_forest_multivariate` [model](https://docs.victoriametrics.com/anomaly-detection/components/models/) states, so to migrate from v1.24.0-v1.24.1 it is recommended to drop the state |

docs/anomaly-detection/QuickStart.md

@@ -24,6 +24,7 @@ The following options are available:
 
 - [To run Docker image](#docker)
 - [To run in Kubernetes with Helm charts](#kubernetes-with-helm-charts)
+- [To run with VM Operator](#vm-operator)
 
 > Anomaly detection models can be kept {{% available_from "v1.13.0" anomaly %}} **on host filesystem after `fit` stage** (instead of default in-memory option); This will drastically reduce RAM for larger configurations. Similar optimization {{% available_from "v1.16.0" anomaly %}} can be applied to data read from VictoriaMetrics TSDB. See instructions of how to enable it [here](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
 
@@ -121,7 +122,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.28.5
+docker pull victoriametrics/vmanomaly:v1.28.7
 ```
 
 2. Create the license file with your license key.
@@ -141,7 +142,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.28.5 \
+    victoriametrics/vmanomaly:v1.28.7 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -158,7 +159,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.28.5 \
+    victoriametrics/vmanomaly:v1.28.7 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -171,7 +172,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.28.4
+    image: victoriametrics/vmanomaly:v1.28.7
     # ...
     restart: always
     volumes:
@@ -307,6 +308,84 @@ writer:
 {{% available_from "v1.26.0" anomaly %}} `vmanomaly`'s built-in web UI can be used for prototyping and interactive experimenting to produce vmanomaly's and vmalert's configuration files. Please refer to the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/) for detailed instructions and examples.
 
 ![vmanomaly-ui-overview](vmanomaly-ui-overview.webp)
+> [!TIP]
+Public playgrounds with pre-configured `vmanomaly` instances and VictoriaMetrics/VictoriaLogs/VictoriaTraces datasources are available for interactive experimenting without the need to set up your own instance or getting an enterprise license. You can find them in the [UI documentation](https://docs.victoriametrics.com/anomaly-detection/ui/#playgrounds) or access them directly via the links - [metrics](https://play-vmanomaly.victoriametrics.com/metrics/), [logs](https://play-vmanomaly.victoriametrics.com/logs/), [traces](https://play-vmanomaly.victoriametrics.com/traces/) - or embedded versions in the collapsible blocks.
+
+{{% collapse name="Playground on VictoriaMetrics Datasource" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-vmanomaly-metrics')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-vmanomaly-metrics"
+    title="VictoriaMetrics Anomaly Detection Playground (Metrics)"
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmanomaly.victoriametrics.com/metrics/"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
+{{% collapse name="Playground on VictoriaLogs Datasource" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-vmanomaly-logs')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-vmanomaly-logs"
+    title="VictoriaLogs Anomaly Detection Playground (Logs)"
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmanomaly.victoriametrics.com/logs/"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
+{{% collapse name="Playground on VictoriaTraces Datasource" %}} 
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-vmanomaly-traces')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-vmanomaly-traces"
+    title="VictoriaTraces Anomaly Detection Playground (Traces)"
+    allowfullscreen
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmanomaly.victoriametrics.com/traces/"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
 
 ### Recommended steps
 

docs/anomaly-detection/UI.md

@@ -37,6 +37,86 @@ server:
 
 For impactful parameters please refer to [optimize resource usage](#optimize-resource-usage) section of this page.
 
+## Playgrounds
+
+To start exploring the UI, you can use embedded demo with preconfigured queries and models down below on public playgrounds (VictoriaMetrics, VictoriaLogs and VictoriaTraces):
+
+{{% collapse name="Playground on VictoriaMetrics Datasource" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-vmanomaly-metrics')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-vmanomaly-metrics"
+    title="VictoriaMetrics Anomaly Detection Playground (Metrics)"
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmanomaly.victoriametrics.com/metrics/"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
+{{% collapse name="Playground on VictoriaLogs Datasource" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-vmanomaly-logs')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-vmanomaly-logs"
+    title="VictoriaMetrics Anomaly Detection Playground (Logs)"
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmanomaly.victoriametrics.com/logs/"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
+{{% collapse name="Playground on VictoriaTraces Datasource" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-vmanomaly-traces')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-vmanomaly-traces"
+    title="VictoriaMetrics Anomaly Detection Playground (Traces)"
+    allowfullscreen
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmanomaly.victoriametrics.com/traces/"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
 ## Authentication
 
 {{% available_from "v1.27.0" anomaly %}} The vmanomaly UI supports proxying authentication headers from [v1.1.0](#v110) and onwards.
@@ -421,6 +501,15 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+### v1.4.3
+Released: 2026-02-09
+
+vmanomaly version: [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287)
+
+- Update color palette in dark theme for more contrast and better visibility of anomalies and confidence intervals in the Visualization Panel.
+
+- Align Model Panel content to improve layout for smaller screens and embedded documentation.
+
 ### v1.4.2
 Released: 2026-01-17
 
@@ -484,10 +573,9 @@ vmanomaly version: [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/
 
 - IMPROVEMENT: datasource value is initialized from the server reader config (on the first UI initialization) if [mixed mode is used](#mixed-usage). Can be reset to the default value anytime by hitting the "Reset to Default" button next to the datasource field in the [Settings Panel](#settings-panel).
 
-
 ### v1.0.0
 Released: 2025-10-02
 
 vmanomaly version: [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1260)
 
-Initial public release of the vmanomaly UI.
+Initial public release of the vmanomaly UI.

docs/anomaly-detection/components/README.md

@@ -46,37 +46,38 @@ settings:
 # how and when to run the models is defined by schedulers
 # https://docs.victoriametrics.com/anomaly-detection/components/scheduler/
 schedulers:
-  periodic_1d:  # alias
+  periodic_online:  # alias
     class: 'periodic' # scheduler class
-    infer_every: "30s"
-    fit_every: "1d"
-    fit_window: "24h"
+    infer_every: "30s"  # how often to produce anomaly scores for new data
+    fit_every: "365d"  # how often to re-fit the models, for online models used effectively once, then they are updated with new data and won't require re-fit
+    fit_window: "3d"  # how much historical data to use for fit stage
     start_from: "00:00"  # start from specified time, i.e. 00:00 given timezone and do daily fits as `fit_every` is 1 day
     tz: "Europe/Kyiv"  # timezone to use for start_from
-  periodic_1w:
+  periodic_offline_1w:
     class: 'periodic'
     infer_every: "15m"
-    fit_every: "1h"
-    fit_window: "7d"
+    fit_every: "24h"
+    fit_window: "14d"
     # if no start_from is specified, jobs will start immediately after service starts
 
 # what model types and with what hyperparams to run on your data
 # https://docs.victoriametrics.com/anomaly-detection/components/models/
 models:
   zscore:  # we can set up alias for model
-    class: 'zscore'  # model class
+    class: 'zscore_online'  # model class
     z_threshold: 3.5
-    provide_series: ['anomaly_score']  # what series to produce
+    decay: 0.99  # weight for data points value should be in (0, 1], 1 means to give equal weight to all data
+    provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_upper']  # what series to produce as output of the model
     queries: ['host_network_receive_errors']  # what queries to run particular model on
-    schedulers: ['periodic_1d']  # will be attached to 1-day schedule, fit every 10m and infer every 30s
+    schedulers: ['periodic_online']  # will be fit once, used for infer every 30s
     min_dev_from_expected: 0.0  # turned off. if |y - yhat| < min_dev_from_expected, anomaly score will be 0
     detection_direction: 'above_expected' # detect anomalies only when y > yhat, "peaks"
     clip_predictions: True  # clip predictions to expected data range, i.e. [0, inf] for this query `host_network_receive_errors
-  prophet: # we can set up alias for model
+  prophet_weekly: # we can set up alias for model
     class: 'prophet'
-    provide_series: ['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper']
+    provide_series: ['anomaly_score', 'y', 'yhat', 'yhat_lower', 'yhat_upper']
     queries: ['cpu_seconds_total']
-    schedulers: ['periodic_1w']  # will be attached to 1-week schedule, fit every 1h and infer every 15m
+    schedulers: ['periodic_offline_1w']  # will be attached to 1-week scheduler, re-fit every 24h and infer every 15m
     min_dev_from_expected: [0.01, 0.01]  # minimum deviation from expected value to be even considered as anomaly
     anomaly_score_outside_data_range: 1.5  # override default anomaly score outside expected data range
     detection_direction: 'above_expected'
@@ -111,13 +112,18 @@ reader:
 writer:
   datasource_url: "http://victoriametrics:8428/"
   # tenant_id: "0:0"  # for VictoriaMetrics cluster, can support "multitenant"
+  # https://docs.victoriametrics.com/anomaly-detection/components/writer/#metrics-formatting
+  metric_format:
+    __name__: $VAR
+    for: $QUERY_KEY
+    
 
 # enable self-monitoring in pull and/or push mode
 # https://docs.victoriametrics.com/anomaly-detection/components/monitoring/
 monitoring:
-  pull: # Enable /metrics endpoint.
-    addr: "0.0.0.0"
-    port: 8490
+  # pull: # Enable /metrics endpoint.
+  #   addr: "0.0.0.0"
+  #   port: 8490
 
   push: # Enable pushing self-monitoring metrics
     url: "http://victoriametrics:8428"

docs/anomaly-detection/components/models.md

@@ -1219,7 +1219,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.28.5
+docker pull victoriametrics/vmanomaly:v1.28.7
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1233,7 +1233,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.28.5 /config.yaml \
+victoriametrics/vmanomaly:v1.28.7 /config.yaml \
 --licenseFile=/license
 --watch
 ```

docs/anomaly-detection/components/reader.md

@@ -12,13 +12,22 @@ aliases:
   - /anomaly-detection/components/reader.html
 ---
 
-VictoriaMetrics Anomaly Detection (`vmanomaly`) primarily uses [VmReader](#vm-reader) to ingest data. This reader focuses on fetching time-series data directly from VictoriaMetrics with the help of powerful [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) expressions for aggregating, filtering and grouping your data, ensuring seamless integration and efficient data handling.
+VictoriaMetrics Anomaly Detection (`vmanomaly`) has an input of Prometheus-compatible metrics from either [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/) accessed with [VmReader](#vm-reader) with [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) queries or from [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) / [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) accessed with [VLogsReader](#victorialogs-reader) with [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries.
 
 Future updates will introduce additional readers, expanding the range of data sources `vmanomaly` can work with.
 
+## Playgrounds
+
+To ease the development and testing of queries for `vmanomaly`'s input data, following playgrounds can be used for experimenting with MetricsQL and LogsQL queries:
+
+Please see respective sections below for specific reader:
+- [MetricsQL playground](#metricsql-playground) for `VmReader`
+- [LogsQL playground](#logsql-playground) for `VLogsReader`
 
 ## VM reader
 
+{{% collapse name="Queries format migration (to v1.13.0+)" %}}
+
 > There is backward-compatible change{{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg of [VmReader](#vm-reader). New format allows to specify per-query parameters, like `step` to reduce amount of data read from VictoriaMetrics TSDB and to allow config flexibility. Please see [per-query parameters](#per-query-parameters) section for the details.
 
 Old format like
@@ -51,10 +60,11 @@ reader:
       tz: 'UTC'  # by default, tz-free data is used throughout the model lifecycle
       # new query-level arguments will be added in backward-compatible way in future releases
 ```
+{{% /collapse %}}
 
 ### Per-query parameters
 
-There is change{{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg format. Now each query alias supports the next (sub)fields, which *override reader-level parameters*, if set:
+There is change {{% available_from "v1.13.0" anomaly %}} of [`queries`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) arg format. Now each query alias supports the next (sub)fields, which *override reader-level parameters*, if set:
 
 - `expr` (string): MetricsQL/PromQL expression that defines an input for VmReader. As accepted by `/query_range?query=%s`. i.e. `avg(vm_blocks)`
 
@@ -418,7 +428,8 @@ Optional argument{{% available_from "v1.25.3" anomaly %}} allows specifying a ti
 </table>
 
 <br>
-Config section example:
+
+**Config section example**:
 
 ```yaml
 reader:
@@ -441,6 +452,35 @@ reader:
   latency_offset: '1ms'
 ```
 
+### MetricsQL Playground
+
+To experiment with MetricsQL queries for `VmReader`, you can use the [VictoriaMetrics MetricsQL Playground](https://play.victoriametrics.com/), which provides an interactive environment to test and visualize your queries against sample data. You can also access embedded version of the playground below:
+
+{{% collapse name="VictoriaMetrics Playground" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-metricsql')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-metricsql"
+    title="VictoriaMetrics MetricsQL Playground"
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play.victoriametrics.com/select/0/vmui/?#/?g0.range_input=24h&g0.end_input=2026-02-09T09%3A57%3A57&g0.relative_time=last_24_hours&g0.tab=0&g0.step_input=30m&g0.expr=sum%28%28rate%28node_cpu_seconds_total%7Bmode%21%3D%22idle%22%7D%5B5m%5D%29%29%29+by+%28service%29&legend_view=table&legend_hide_duplicates=true"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
 ### mTLS protection
 
 `vmanomaly` supports [mutual TLS (mTLS)](https://en.wikipedia.org/wiki/Mutual_authentication){{% available_from "v1.16.3" anomaly %}} for secure communication across its components, including [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader), [VmWriter](https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer), and [Monitoring/Push](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#push-config-parameters). This allows for mutual authentication between the client and server when querying or writing data to [VictoriaMetrics Enterprise, configured for mTLS](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#mtls-protection).
@@ -474,7 +514,6 @@ reader:
 # other config sections, like models, schedulers, writer, ...
 ```
 
-
 ### Healthcheck metrics
 
 `VmReader` exposes [several healthchecks metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics).
@@ -482,11 +521,11 @@ reader:
 
 ## VictoriaLogs reader
 
-{{% available_from "v1.26.0" anomaly %}} `vmanomaly` adds support for reading data from [VictoriaLogs stats queries](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats) endpoint with `VLogsReader`. This reader allows quering and analyzing log data stored in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/), enabling anomaly detection on metrics generated from logs. **Querying [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) is supported with the same reader, as the endpoints for both are equivalent.**
+{{% available_from "v1.26.0" anomaly %}} `vmanomaly` can read data from [VictoriaLogs stats queries](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-range-stats) endpoint with `VLogsReader`. This reader allows quering and analyzing log data stored in [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/), enabling anomaly detection on metrics generated from logs. **Querying [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) is supported with the same reader, as the endpoints for both are equivalent.**
 
-Its queries should be expressed in a subset of [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/), which is similar to MetricsQL/PromQL but adapted for log data.
+Its queries should be expressed in [LogsQL*](https://docs.victoriametrics.com/victorialogs/logsql/) language that both VictoriaLogs and VictoriaTraces support, with the focus on using [stats pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) functions to calculate metrics from logs.
 
-> Please be aware that `VLogsReader` is designed to work with a `/select/stats_query_range` endpoint of [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/), so the `<query>` expressions must contain `stats` [pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) (see [query-examples](#query-examples) section below). The calculated stats is converted into metrics with labels from `by(...)` clause of the `| stats by(...)` pipe, where `stats_func*` is any of the supported [stats function subset](#valid-stats-functions) of [available stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions), while the `result_name*` is the name of the log field to store the result of the corresponding stats function. The `as` keyword is optional.
+> Please be aware that `VLogsReader` is designed to work with a `/select/stats_query_range` endpoint of [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/), so the `<query>` expressions must ends with `stats` [pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) (see [query-examples](#query-examples) section below). The calculated stats is converted into metrics with labels from `by(...)` clause of the `| stats by(...)` pipe, where `stats_func*` is any of the supported [stats function subset](#valid-stats-functions) of [available stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions), while the `result_name*` is the name of the log field to store the result of the corresponding stats function. The `as` keyword is optional.
 
 ### Valid stats functions
 `VLogsReader` relies on [stats pipe functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions) that return **numeric values**, which can be used for anomaly detection on timeseries (metrics). The future addition of similar stats functions in VictoriaLogs will be supported automatically, as long as they return **numeric values**.
@@ -508,22 +547,110 @@ The supported stats functions currently include:
 
 ### Query Examples
 
-> You can test your LogsQL queries with stats pipe functions using our [VictoriaLogs playground](https://play-vmlogs.victoriametrics.com/) or [VictoriaTraces playground](https://play-vtraces.victoriametrics.com/). Use either UI to access graphical results or the `/select/logsql/stats_query_range` endpoint to run your queries and see the raw results, e.g. as this [sample query](https://play-vmlogs.victoriametrics.com/select/logsql/stats_query_range?query=_time%3A5m%20%7C%20stats%20by%20%28_stream%29%20count%28%29%20as%20sample_row&step=1m).
+#### VictoriaLogs
 
-Here are examples of simple valid LogsQL queries with stats pipe functions that can be used with `VLogsReader`.
-
-The following query returns the average value for the duration field over logs matching the [filter](https://docs.victoriametrics.com/victorialogs/logsql/#filters) for `error` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word):
+Here are examples of simple [valid LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) queries with stats pipe functions that can be used with `VLogsReader`.
 
+1. Ingestion volume - good baseline time series, for detecting dropouts/spikes without depending on any schema.
+```shellhelp
+* | stats count() as logs
 ```
-error | stats avg(duration) as avg_error_duration
-``` 
 
-It is possible to calculate the average over fields with common prefix via `avg(prefix*)` syntax. For example, the following query calculates the number of logs with `foo` prefix having `error` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word):
+2. Ingestion rate (normalized) - good for detecting dropouts/spikes without depending on any schema, and also for detecting changes in log volume trends.
+```shellhelp
+* | stats rate() as logs_per_sec
+```
 
+3. Per-stream rate - good for detecting dropouts/spikes on individual streams, and also for detecting changes in log volume trends on stream level.
+```shellhelp
+* | stats by (_stream) rate() as logs_per_sec
 ```
-error | stats count(foo*) as foo_error_count
+
+4. Active stream churn - good for detecting changes in the number of active streams. Catches "new sources exploded"/"sources disappeared" patterns.
+```shellhelp
+* | stats count_uniq(_stream) as active_streams
 ```
 
+5. Avg logs per stream - good for detecting changes in log volume trends on stream level, without depending on the number of streams (sources). Catches "new sources exploded"/"sources disappeared" patterns, as well as changes in log volume on stream level.
+```shellhelp
+* | stats count() as logs, count_uniq(_stream_id) as streams | math (logs / max(streams, 1)) as logs_per_stream
+```
+
+6. Max message size - good for detecting changes in log message size patterns, which can be an indicator of changes in log structure or content.
+```shellhelp
+* | len(_msg) as msg_len | stats max(msg_len) as max_msg_len
+```
+
+7. P90 word per message - good for detecting changes in the distribution of words per message, which can indicate changes in log content or structure.
+```shellhelp
+* | unpack_words as words drop_duplicates | json_array_len(words) as words_count | stats quantile(0.9, words_count) as p90_words_per_msg
+```
+
+#### VictoriaTraces
+
+> [!TIP]
+Almost identical to VictoriaLogs, but in VictoriaTraces you'll need to exclude index entries. Basically replace `*` with a filter, e.g. `{"resource_attr:service.name"!=""}` to select only spans with non-empty `service.name` resource attribute.
+
+1. Ingestion volume - good baseline time series, for detecting dropouts/spikes without depending on any schema.
+```
+{"resource_attr:service.name"!=""} | stats count() as spans
+```
+or for the rate:
+```shellhelp
+{"resource_attr:service.name"!=""} | stats rate() as spans_per_sec
+```
+
+2. Error volume - good for detecting dropouts/spikes in error spans, without depending on any schema. In VictoriaLogs you can use similar query with `status_code` field, if it exists in your logs.
+```
+# spans with `status_code=2`, see: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto#L323
+{"resource_attr:service.name"!=""} AND status_code := "2" | stats count() as error_spans
+```
+or for the rate:
+```shellhelp
+{"resource_attr:service.name"!=""} AND status_code := "2" | stats rate() as error_spans_per_sec
+```
+
+> [!TIP]
+[The stream](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) in VictoriaTraces means "service_name & span_name" combination.
+
+```shellhelp
+{"resource_attr:service.name"!=""} | stats count_uniq(_stream) as active_streams
+```
+
+### LogsQL playground
+
+To experiment with LogsQL queries for `VLogsReader`, you can use the [VictoriaLogs LogsQL Playground](https://play-vmlogs.victoriametrics.com/), which provides an interactive environment to test and visualize your queries against sample log data.
+
+Similarly, [VictoriaTraces LogsQL Playground](https://play-vtraces.victoriametrics.com/) can be used for testing LogsQL queries against sample trace data.
+
+You can also access **embedded version of the playground below** (VictoriaLogs datasource):
+
+{{% collapse name="VictoriaLogs LogsQL Playground" %}}
+
+<div class="position-relative mb-3">
+  <button
+    type="button"
+    class="btn btn-primary btn-sm position-absolute top-0 end-0 m-2"
+    style="z-index: 2;"
+    onclick="document.getElementById('vmui-playground-logsql')?.requestFullscreen?.()"
+  >
+    Fullscreen
+  </button>
+
+  <iframe
+    id="vmui-playground-logsql"
+    title="VictoriaLogs LogsQL Playground"
+    allow="fullscreen"
+    loading="lazy"
+    class="w-100 border rounded"
+    style="height: 400px; background: white;"
+    src="https://play-vmlogs.victoriametrics.com/select/vmui/?#/?query=*+%7C+stats+rate%28%29+as+logs_per_sec&g0.range_input=30m&g0.end_input=2026-02-09T10%3A01%3A26&g0.relative_time=last_30_minutes&graph_mode=stats&limit=100&bars_count=48"
+  ></iframe>
+</div>
+
+{{% /collapse %}}
+
+
 ### Config parameters
 
 <table class="params">

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -10,9 +10,9 @@ sitemap:
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.134.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.134.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.134.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.135.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.135.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.135.0)
 - [Grafana](https://grafana.com/) (v.10.2.1)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.134.0
+    image: victoriametrics/vmagent:v1.135.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -340,7 +340,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.134.0
+    image: victoriametrics/victoria-metrics:v1.135.0
     ports:
       - 8428:8428
     volumes:
@@ -373,7 +373,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.134.0
+    image: victoriametrics/vmalert:v1.135.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmgateway-openid-configuration/README.md

@@ -249,27 +249,27 @@ services:
       - grafana_data:/var/lib/grafana/
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.134.0
+    image: victoriametrics/victoria-metrics:v1.135.0
     command:
       - -httpListenAddr=0.0.0.0:8429
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.134.0-cluster
+    image: victoriametrics/vmstorage:v1.135.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.134.0-cluster
+    image: victoriametrics/vminsert:v1.135.0-cluster
     command:
       - -storageNode=vmstorage:8400
       - -httpListenAddr=0.0.0.0:8480
 
   vmselect:
-    image: victoriametrics/vmselect:v1.134.0-cluster
+    image: victoriametrics/vmselect:v1.135.0-cluster
     command:
       - -storageNode=vmstorage:8401
       - -httpListenAddr=0.0.0.0:8481
 
   vmagent:
-    image: victoriametrics/vmagent:v1.134.0
+    image: victoriametrics/vmagent:v1.135.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -278,7 +278,7 @@ services:
       - -remoteWrite.url=http://vmsingle:8429/api/v1/write
 
   vmgateway-cluster:
-    image: victoriametrics/vmgateway:v1.134.0-enterprise
+    image: victoriametrics/vmgateway:v1.135.0-enterprise
     ports:
       - 8431:8431
     volumes:
@@ -294,7 +294,7 @@ services:
       - -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
 
   vmgateway-single:
-    image: victoriametrics/vmgateway:v1.134.0-enterprise
+    image: victoriametrics/vmgateway:v1.135.0-enterprise
     ports:
       - 8432:8431
     volumes:
@@ -405,7 +405,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
 
 ```yaml
   vmagent:
-    image: victoriametrics/vmagent:v1.134.0
+    image: victoriametrics/vmagent:v1.135.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
       - ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret

docs/guides/k8s-monitoring-via-vm-single/README.md

@@ -6,163 +6,108 @@ build:
 sitemap:
   disable: true
 ---
-**This guide covers:**
 
-* The setup of a [VictoriaMetrics Single](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) in [Kubernetes](https://kubernetes.io/) via Helm charts
-* How to scrape metrics from k8s components using service discovery 
-* How to visualize stored data 
-* How to store metrics in [VictoriaMetrics](https://victoriametrics.com) tsdb
+This guide walks you through deploying a [single-node version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) on Kubernetes using Helm.
+
+At the end of this guide, you will know:
+
+- How to install VictoriaMetrics single node in Kubernetes.
+- How to scrape metrics from Kubernetes components using service discovery.
+- How to store metrics in [VictoriaMetrics](https://victoriametrics.com) time series database.
+- How to visualize stored data with Grafana.
 
 **Precondition**
 
 We will use:
-* [Kubernetes cluster 1.31.1-gke.1678000](https://cloud.google.com/kubernetes-engine)
-> We use GKE cluster from [GCP](https://cloud.google.com/) but this guide is also applied on any Kubernetes cluster. For example [Amazon EKS](https://aws.amazon.com/ru/eks/).
-* [Helm 3.14+](https://helm.sh/docs/intro/install)
-* [kubectl 1.31](https://kubernetes.io/docs/tasks/tools/install-kubectl)
+
+- [Kubernetes cluster 1.34](https://cloud.google.com/kubernetes-engine)
+- [Helm 4.1.0+](https://helm.sh/docs/intro/install)
+- [kubectl 1.34.3](https://kubernetes.io/docs/tasks/tools/install-kubectl)
+
+  > We use a GKE cluster from [GCP](https://cloud.google.com/), but this guide also applies to any Kubernetes cluster. For example, [Amazon EKS](https://aws.amazon.com/ru/eks/) or an on-premises cluster.
 
 ![VictoriaMetrics Single on Kubernetes cluster](k8s-scheme.webp)
 
 ## 1. VictoriaMetrics Helm repository
 
-You need to add the VictoriaMetrics Helm repository to install VictoriaMetrics components. We’re going to use [VictoriaMetrics Single](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/). You can do this by running the following command:
-
+Run the following command to add the VictoriaMetrics Helm repository:
 
 ```shell
 helm repo add vm https://victoriametrics.github.io/helm-charts/
-```
-
-
-Update Helm repositories:
-
-```shell
 helm repo update
 ```
 
-To verify that everything is set up correctly you may run this command:
+To verify that everything is set up correctly, you may run this command:
 
 ```shell
 helm search repo vm/
 ```
 
-The expected output is:
+You should get a list of charts similar to this:
 
 ```text
-NAME                           	CHART VERSION	APP VERSION	DESCRIPTION                                       
-vm/victoria-logs-single        	0.9.3        	v1.16.0    	Victoria Logs Single version - high-performance...
-vm/victoria-metrics-agent      	0.17.2       	v1.113.0   	Victoria Metrics Agent - collects metrics from ...
-vm/victoria-metrics-alert      	0.15.0       	v1.113.0   	Victoria Metrics Alert - executes a list of giv...
-vm/victoria-metrics-anomaly    	1.9.0        	v1.21.0    	Victoria Metrics Anomaly Detection - a service ...
-vm/victoria-metrics-auth       	0.10.0       	v1.113.0   	Victoria Metrics Auth - is a simple auth proxy ...
-vm/victoria-metrics-cluster    	0.19.2       	v1.113.0   	Victoria Metrics Cluster version - high-perform...
-vm/victoria-metrics-common     	0.0.42       	           	Victoria Metrics Common - contains shared templ...
-vm/victoria-metrics-distributed	0.9.0        	v1.113.0   	A Helm chart for Running VMCluster on Multiple ...
-vm/victoria-metrics-gateway    	0.8.0        	v1.113.0   	Victoria Metrics Gateway - Auth & Rate-Limittin...
-vm/victoria-metrics-k8s-stack  	0.39.0       	v1.113.0   	Kubernetes monitoring on VictoriaMetrics stack....
-vm/victoria-metrics-operator   	0.43.0       	v0.54.1    	Victoria Metrics Operator                         
-vm/victoria-metrics-single     	0.15.1       	v1.113.0   	Victoria Metrics Single version - high-performa...
+NAME                                    CHART VERSION   APP VERSION     DESCRIPTION
+vm/victoria-metrics-single              0.29.0          v1.134.0        VictoriaMetrics Single version - high-performan...
+vm/victoria-metrics-agent               0.30.0          v1.134.0        VictoriaMetrics Agent - collects metrics from v...
+vm/victoria-metrics-alert               0.30.0          v1.134.0        VictoriaMetrics Alert - executes a list of give...
+vm/victoria-metrics-anomaly             1.12.9          v1.28.2         VictoriaMetrics Anomaly Detection - a service t...
+...(list continues)...
 ```
 
+## 2. Install [VictoriaMetrics single](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) from Helm Chart
 
-## 2. Install [VictoriaMetrics Single](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) from Helm Chart
+Run this command in your terminal to install [VictoriaMetrics single node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the default [namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) in your cluster:
 
-Run this command in your terminal:
-
-```text
+```shell
 helm install vmsingle vm/victoria-metrics-single -f https://docs.victoriametrics.com/guides/examples/guide-vmsingle-values.yaml
 ```
 
-Here is full file content `guide-vmsingle-values.yaml`
+Below are the key sections in the chart values file [`guide-vmsingle-values.yaml`](https://docs.victoriametrics.com/guides/examples/guide-vmsingle-values.yaml):
 
-```yaml
-server:
-  scrape:
-    enabled: true
-    configMap: ""
-    config:
-      global:
-        scrape_interval: 15s
-      scrape_configs:
-        - job_name: victoriametrics
-          static_configs:
-            - targets: [ "localhost:8428" ]
-        - job_name: "kubernetes-apiservers"
-          kubernetes_sd_configs:
-            - role: endpoints
-          scheme: https
-          tls_config:
-            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-            insecure_skip_verify: true
-          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-          relabel_configs:
-            - source_labels:
-                [
-                  __meta_kubernetes_namespace,
-                  __meta_kubernetes_service_name,
-                  __meta_kubernetes_endpoint_port_name,
-                ]
-              action: keep
-              regex: default;kubernetes;https
-        - job_name: "kubernetes-nodes"
-          scheme: https
-          tls_config:
-            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-            insecure_skip_verify: true
-          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-          kubernetes_sd_configs:
-            - role: node
-          relabel_configs:
-            - action: labelmap
-              regex: __meta_kubernetes_node_label_(.+)
-        - job_name: "kubernetes-nodes-cadvisor"
-          scheme: https
-          tls_config:
-            ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-            insecure_skip_verify: true
-          bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-          kubernetes_sd_configs:
-            - role: node
-          metrics_path: /metrics/cadvisor
-          relabel_configs:
-            - action: labelmap
-              regex: __meta_kubernetes_node_label_(.+)
-            - source_labels: [__metrics_path__]
-              target_label: metrics_path
-          metric_relabel_configs:
-            - action: replace
-              source_labels: [pod]
-              regex: '(.+)'
-              target_label: pod_name
-              replacement: '${1}'
-            - action: replace
-              source_labels: [container]
-              regex: '(.+)'
-              target_label: container_name
-              replacement: '${1}'
-            - action: replace
-              target_label: name
-              replacement: k8s_stub
-            - action: replace
-              source_labels: [id]
-              regex: '^/system\.slice/(.+)\.service$'
-              target_label: systemd_service_name
-              replacement: '${1}'
-```
+- With `scrape: enabled: true`, we enable metric autodiscovery for the Kubernetes cluster.
 
+    ```yaml
+    server:
+      scrape:
+        enabled: true
+        ...
+    ```
 
-* By running `helm install vmsingle vm/victoria-metrics-single` we install [VictoriaMetrics Single](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to default [namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) inside your cluster
-* By adding `scrape: enabled: true` we add and enable autodiscovery scraping from kubernetes cluster to [VictoriaMetrics Single](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)
-* On line 67 from [https://docs.victoriametrics.com/guides/examples/guide-vmsingle-values.yaml](https://docs.victoriametrics.com/guides/examples/guide-vmsingle-values.yaml) we added `metric_relabel_configs` section that will help us to show Kubernetes metrics on Grafana dashboard.
+- The `metric_relabel_configs` section normalizes Kubernetes metrics labels so they are shown correctly in the Grafana dashboard later on.
 
+    ```yaml
+              ...
+              metric_relabel_configs:
+                - action: replace
+                  source_labels: [pod]
+                  regex: '(.+)'
+                  target_label: pod_name
+                  replacement: '${1}'
+                - action: replace
+                  source_labels: [container]
+                  regex: '(.+)'
+                  target_label: container_name
+                  replacement: '${1}'
+                - action: replace
+                  target_label: name
+                  replacement: k8s_stub
+                - action: replace
+                  source_labels: [id]
+                  regex: '^/system\.slice/(.+)\.service$'
+                  target_label: systemd_service_name
+                  replacement: '${1}'
+              ...
+    ```
 
-As a result of the command you will see the following output:
+The `helm install vmsingle vm/victoria-metrics-single` command should result in the following output:
 
 ```text
 NAME: vmsingle
-LAST DEPLOYED: Fri Mar 21 11:50:39 2025
+LAST DEPLOYED: Wed Jan 28 13:04:36 2026
 NAMESPACE: default
 STATUS: deployed
 REVISION: 1
+DESCRIPTION: Install complete
 TEST SUITE: None
 NOTES:
 The VictoriaMetrics write api can be accessed via port 8428 on the following DNS name from within your cluster:
@@ -173,12 +118,12 @@ Metrics Ingestion:
     export POD_NAME=$(kubectl get pods --namespace default -l "app=" -o jsonpath="{.items[0].metadata.name}")
     kubectl --namespace default port-forward $POD_NAME 8428
 
-  Write URL inside the kubernetes cluster:
+  Write the URL inside the Kubernetes cluster:
     http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local.:8428/<protocol-specific-write-endpoint>
 
   All supported write endpoints can be found at https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data
 
-  E.g: for Prometheus:
+  E.g, for Prometheus:
     http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local.:8428/api/v1/write
 
 Metrics Scrape:
@@ -190,7 +135,7 @@ Metrics Scrape:
     Inside cluster:
       http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local.:8428/targets
     Outside cluster:
-      You need to port-forward service (see instructions above) and call
+      You need to port-forward the service (see instructions above) and call
       http://<service-host-port>/targets
 
 Read Data:
@@ -198,38 +143,43 @@ Read Data:
     http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local.:8428
 ```
 
-For us it’s important to remember the url for the datasource (copy lines from output).
+Take note of the Grafana datasource URL near the end of the output, as we'll use it in the next step. In the example above, this is the datasource URL:
 
-Verify that VictoriaMetrics pod is up and running by executing the following command:
+```text
+The following URL can be used as the datasource URL in Grafana::
+http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local.:8428
 
+```
+
+Verify that the VictoriaMetrics pod is up and running by executing the following command:
 
 ```shell
 kubectl get pods
 ```
 
-The expected output is:
+Wait until the STATUS is Running. The expected output is:
 
 ```text
-NAME                                                READY   STATUS    RESTARTS   AGE
+NAME                                        READY   STATUS    RESTARTS   AGE
 vmsingle-victoria-metrics-single-server-0   1/1     Running   0          68s
 ```
 
-
 ## 3. Install and connect Grafana to VictoriaMetrics with Helm
 
-Add the Grafana Helm repository. 
-
+Add the Grafana Helm repository.
 
 ```shell
-helm repo add grafana https://grafana.github.io/helm-charts
+helm repo add grafana-community https://grafana-community.github.io/helm-charts
 helm repo update
 ```
 
+> [!NOTE] Tip
+> See more information on Grafana in [ArtifactHUB](https://artifacthub.io/packages/helm/grafana-community/grafana)
 
-By installing the Chart with the release name `my-grafana`, you add the VictoriaMetrics datasource with official dashboard and kubernetes dashboard:
+Create a config file for the Grafana service. Ensure that the `url` value matches the Grafana datasource URL from the previous step:
 
 ```yaml
-cat <<EOF | helm install my-grafana grafana/grafana -f -
+cat <<EOF > grafana-single-values.yml
   datasources:
     datasources.yaml:
       apiVersion: 1
@@ -237,7 +187,8 @@ cat <<EOF | helm install my-grafana grafana/grafana -f -
         - name: victoriametrics
           type: prometheus
           orgId: 1
-          url: http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local:8428
+          # use the URL obtained from the VictoriaMetrics helm install output
+          url: http://vmsingle-victoria-metrics-single-server.default.svc.cluster.local.:8428
           access: proxy
           isDefault: true
           updateIntervalSeconds: 10
@@ -260,31 +211,71 @@ cat <<EOF | helm install my-grafana grafana/grafana -f -
     default:
       victoriametrics:
         gnetId: 10229
-        revision: 22
         datasource: victoriametrics
       kubernetes:
         gnetId: 14205
-        revision: 1
         datasource: victoriametrics
 EOF
 ```
 
+Run the following command to install Grafana with the release name `my-grafana`:
 
-By running this command we:
-* Install Grafana from Helm repository.
-* Provision VictoriaMetrics datasource with the url from the output above which we copied before.
-* Add [this dashboard](https://grafana.com/grafana/dashboards/10229) for VictoriaMetrics.
-* Add [this dashboard](https://grafana.com/grafana/dashboards/14205) to see Kubernetes cluster metrics.
+```shell
+helm install my-grafana grafana-community/grafana -f grafana-single-values.yml
+```
+
+By running this command, we:
+
+- Install Grafana from the Helm repository.
+- Configure Grafana to use the VictoriaMetrics datasource URL.
+- Add two starter dashboards:
+  - [Kubernetes Cluster Monitoring (via Prometheus)](https://grafana.com/grafana/dashboards/14205-kubernetes-cluster-monitoring-via-prometheus/) to show the Kubernetes Cluster metrics.
+  - [VictoriaMetrics - single-node](https://grafana.com/grafana/dashboards/10229-victoriametrics-single-node/) for VictoriaMetrics telemetry ingestion monitoring.
+
+Check the output log in your terminal. You should see the following output:
+
+```text
+NAME: my-grafana
+LAST DEPLOYED: Wed Jan 28 13:12:51 2026
+NAMESPACE: default
+STATUS: deployed
+REVISION: 1
+DESCRIPTION: Install complete
+NOTES:
+1. Get your 'admin' user password by running:
+
+   kubectl get secret --namespace default my-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
 
 
-Check the output log in your terminal.
-To see the password for Grafana `admin` user use the following command:
+2. The Grafana server can be accessed via port 80 on the following DNS name from within your cluster:
+
+   my-grafana.default.svc.cluster.local
+
+   Get the Grafana URL to visit by running these commands in the same shell:
+     export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=my-grafana" -o jsonpath="{.items[0].metadata.name}")
+     kubectl --namespace default port-forward $POD_NAME 3000
+
+3. Login with the password from step 1 and the username: admin
+#################################################################################
+######   WARNING: Persistence is disabled!!! You will lose your data when   #####
+######            the Grafana pod is terminated.                            #####
+#################################################################################
+```
+
+To see the password for Grafana `admin` user use the command shown in the previous output:
 
 ```shell
 kubectl get secret --namespace default my-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
 ```
 
-Expose Grafana service on `127.0.0.1:3000`:
+Wait until the Grafana pod Status is Running:
+
+```text
+NAME                                        READY   STATUS    RESTARTS   AGE
+my-grafana-bc7796cf5-ffmln                  1/1     Running   0          8m40s
+```
+
+Expose the Grafana service on `127.0.0.1:3000` with:
 
 ```shell
 export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=my-grafana" -o jsonpath="{.items[0].metadata.name}")
@@ -292,25 +283,39 @@ export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/nam
 kubectl --namespace default port-forward $POD_NAME 3000
 ```
 
-Now Grafana should be accessible on the `http://127.0.0.1:3000` address.
+Now Grafana should be accessible at `http://127.0.0.1:3000`.
 
+## 4. View the dashboards in your browser
 
-## 4. Check the obtained result in your browser
+To check that VictoriaMetrics has collected metrics from the Kubernetes cluster, open the browser to `http://127.0.0.1:3000/dashboards` and choose the `Kubernetes Cluster Monitoring (via Prometheus)` dashboard.
 
-To check that VictoriaMetrics has collects metrics from the k8s cluster open in browser `http://127.0.0.1:3000/dashboards` and choose `Kubernetes Cluster Monitoring (via Prometheus)` dashboard. Use `admin` for login and `password` that you previously obtained from kubectl. 
+Use `admin` as the username and the password you obtained earlier using `kubectl get secret ...`.
 
-![single dashboards](grafana-dashboards.webp)
+![Single and Kubernetes dashboards in Grafana](grafana-dashboards.webp)
+<figcaption style="text-align: center; font-style: italic;">List of pre-installed dashboards in Grafana</figcaption>
 
-You will see something like this:
+You should see the metrics for your Kubernetes dashboard:
 
-![k8s dashboards](grafana-k8s-dashboard.webp)
+![Kubernetes dashboard](grafana-k8s-dashboard.webp)
+<figcaption style="text-align: center; font-style: italic;">Grafana dashboard showing the Kubernetes cluster metrics</figcaption>
 
-VictoriaMetrics dashboard also available to use:
+The VictoriaMetrics dashboard shows metrics on telemetry ingestion and resource utilization:
 
-![single](grafana.webp)
+![VictoriaMetrics Single dashboard](grafana.webp)
+<figcaption style="text-align: center; font-style: italic;">Grafana dashboard for the VictoriaMetrics single-node service</figcaption>
 
 ## 5. Final thoughts
 
-* We have set up TimeSeries Database for your k8s cluster.
-* Collected metrics from all running pods,nodes, … and store them in VictoriaMetrics database.
-* Visualize resources used in Kubernetes cluster by Grafana dashboards.
+- You now have a time series database for your Kubernetes cluster.
+- VictoriaMetrics continuously collects and stores metrics from all running pods and nodes.
+- Grafana dashboards give you a visual view of cluster resources.
+
+Consider reading these resources to complete your setup:
+
+- VictoriaMetrics
+  - [Learn more about the single-node version](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)
+  - [Migrate existing metric data into VictoriaMetrics with vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/)
+  - [Setup alerts](https://docs.victoriametrics.com/victoriametrics/vmalert/)
+- Grafana
+  - [Enable persistent storage](https://grafana.com/docs/grafana/latest/setup-grafana/installation/helm/#enable-persistent-storage-recommended)
+  - [Configure private TLS authority](https://grafana.com/docs/grafana/latest/setup-grafana/installation/helm/#configure-a-private-ca-certificate-authority)

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -12,6 +12,7 @@ aliases:
   - /Cluster-VictoriaMetrics.html
   - /cluster-victoriametrics/index.html
   - /cluster-victoriametrics/
+  - /Cluster-VictoriaMetrics/
 ---
 
 VictoriaMetrics is a fast, cost-effective and scalable time series database. It can be used as a long-term remote storage for Prometheus.
@@ -139,6 +140,7 @@ Currently supported endpoints for `<suffix>` are:
 - `/prometheus/api/v1/status/tsdb`
 - `/prometheus/api/v1/export`
 - `/prometheus/api/v1/export/csv`
+- `/prometheus/api/v1/metadata`
 - `/vmui`
 
 It is allowed to explicitly specify tenant IDs via `vm_account_id` and `vm_project_id` labels in the query.
@@ -606,7 +608,9 @@ Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.c
     - `opentsdb/api/put` - for accepting [OpenTSDB HTTP /api/put requests](http://opentsdb.net/docs/build/html/api_http/put.html). This handler is disabled by default. It is exposed on a distinct TCP address set via `-opentsdbHTTPListenAddr` command-line flag. See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) for details.
 
 - URLs for [Prometheus querying API](https://prometheus.io/docs/prometheus/latest/querying/api/): `http://<vmselect>:8481/select/<accountID>/prometheus/<suffix>`, where:
-  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant)
+  - `<accountID>` is an arbitrary number identifying data namespace for the query (aka tenant). It is possible to set it as `accountID:projectID`,
+  where `projectID` is also arbitrary 32-bit integer. If `projectID` isn't set, then it equals to `0`. See [multitenancy docs](#multitenancy) for more details.
+  The `<accountID>` can be set to `multitenant` string, e.g. `http://<vmselect>:8481/select/multitenant/<suffix>` for querying over multiple tenants (see the full list of [supported multitenant read endpoints](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)).
   - `<suffix>` may have the following values:
     - `api/v1/query` - performs [PromQL instant query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query).
     - `api/v1/query_range` - performs [PromQL range query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query).
@@ -622,6 +626,7 @@ Check practical examples of [VictoriaMetrics API](https://docs.victoriametrics.c
     - `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries,
       which is returned in the response.
     - `api/v1/status/top_queries` - for listing the most frequently executed queries and queries taking the most duration.
+    - `api/v1/metadata` - for fetching [metrics metadata](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata). See [these docs](#metrics-metadata) for more details.
     - `metric-relabel-debug` - for debugging [relabeling rules](https://docs.victoriametrics.com/victoriametrics/relabeling/).
 
 - URLs for [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api): `http://<vmselect>:8481/select/<accountID>/graphite/<suffix>`, where:
@@ -1016,10 +1021,18 @@ to ensure query results consistency, even if storage layer didn't complete dedup
 Cluster version of VictoriaMetrics can store metric metadata (TYPE, HELP, UNIT) {{% available_from "v1.130.0" %}}.
 Metadata ingestion is disabled by default. To enable it, set `-enableMetadata=true` on `vminsert` and `vmagent`.
 
-The metadata is stored in memory and can use up to 1% of available memory by default. The size could be adjusted by `-storage.maxMetadataStorageSize` flag.
-Please note that metadata is lost after `vmstorage` restarts. It is ingested independently from metrics, so a metric may exist without metadata, and vice versa.
+The metadata is cached in-memory in a ring buffer and can use up to 1% of available memory by default (see `-storage.maxMetadataStorageSize` cmd-line flag).
+When in-memory size is exceeded, the least updated entries are dropped first. Entries that weren't updated for 1h are cleaned up automatically.
 
-Metadata can be queried via the `/select/0/prometheus/api/v1/metadata` endpoint, which provides a response compatible with the Prometheus [metadata API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata).
+> The following expression helps to understand if metadata cache capacity is utilized for more than 90%: `vm_metrics_metadata_storage_size_bytes / vm_metrics_metadata_storage_max_size_bytes > 0.9`.
+Setup [monitoring](https://docs.victoriametrics.com/victoriametrics/quick-start/#monitoring) and recommended alerting rules
+to get notified about cache capacity issues.
+
+Metadata is ingested independently from metrics, so a metric can exist without metadata, and vice versa.
+Metadata is expected to be ephemeral and constantly updated on ingestion. For this reason, metadata cache isn't
+persisted during storage restarts.
+
+Metadata supports [multitenancy](#multitenancy) and can be queried via the `/select/<accountID>/prometheus/api/v1/metadata` endpoint, which provides a response compatible with the Prometheus [metadata API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata).
 If multiple vmstorage nodes return metadata for the same metric family name, or if multiple tenants have metadata for the same metric family name, vmselect returns only the first matching result. 
 Duplicate metadata entries are not merged or deduplicated across storages or tenants. See [/api/v1/metadata](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1metadata) example.
 

docs/victoriametrics/FAQ.md

@@ -286,7 +286,8 @@ Source code for Victoriametrics can be found in the following locations:
 VictoriaMetrics is able to handle data from hundreds of millions of IoT sensors and industrial sensors.
 It supports [high cardinality data](https://medium.com/@valyala/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b),
 perfectly [scales up on a single node](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae)
-and scales horizontally to multiple nodes.
+and scales horizontally to multiple nodes in [cluster setup](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
+It also supports an option for reducing the index size for IoT data - see [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning-for-low-churn-rate).
 
 ## What is the difference between single-node and cluster versions of VictoriaMetrics?
 

docs/victoriametrics/MetricsQL.md

@@ -12,6 +12,7 @@ aliases:
 - /MetricsQL.html
 - /metricsql/index.html
 - /metricsql/
+- /MetricsQL/
 ---
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).

docs/victoriametrics/Quick-Start.md

@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.134.0
+docker pull victoriametrics/victoria-metrics:v1.135.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.134.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.135.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 _For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._

docs/victoriametrics/README.md

@@ -93,8 +93,10 @@ VictoriaMetrics has the following prominent features:
 * It supports metrics [relabeling](#relabeling).
 * It can deal with [high cardinality issues](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-cardinality) and
   [high churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) issues via [series limiter](#cardinality-limiter).
-* It ideally works with big amounts of time series data from APM, Kubernetes, IoT sensors, connected cars, industrial telemetry, financial data
-  and various [Enterprise workloads](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+* It ideally works for big amounts of time series with both [high churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate) (APM, Kubernetes)
+  and low churn rate (IoT sensors, connected cars, industrial telemetry, financial data - see
+  [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning-for-low-churn-rate)),
+  plus various [Enterprise workloads](https://docs.victoriametrics.com/victoriametrics/enterprise/).
 * It has an open source [cluster version](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/cluster).
 * It can store data on [NFS-based storages](https://en.wikipedia.org/wiki/Network_File_System) such as [Amazon EFS](https://aws.amazon.com/efs/)
   and [Google Filestore](https://cloud.google.com/filestore).
@@ -471,6 +473,7 @@ VictoriaMetrics supports the following handlers from [Prometheus querying API](h
 * [/api/v1/label/.../values](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues)
 * [/api/v1/status/tsdb](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats). See [these docs](#tsdb-stats) for details.
 * [/api/v1/targets](https://prometheus.io/docs/prometheus/latest/querying/api/#targets) - see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter) for more details.
+* [/api/v1/metadata](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata) - see [these docs](#metrics-metadata) for more details.
 * [/federate](https://prometheus.io/docs/prometheus/latest/federation/) - see [these docs](#federation) for more details.
 
 These handlers can be queried from Prometheus-compatible clients such as Grafana or curl.
@@ -1378,11 +1381,19 @@ see [these docs](https://docs.victoriametrics.com/victoriametrics/stream-aggrega
 
 ## Metrics Metadata
 
-Single-node VictoriaMetrics can store metric metadata (TYPE, HELP, UNIT) {{% available_from "v1.130.0" %}}.
+Single-node VictoriaMetrics can store metric metadata (`TYPE`, `HELP`, `UNIT`) {{% available_from "v1.130.0" %}}.
 Metadata ingestion and querying are disabled by default. To enable them, set `-enableMetadata=true`.
 
-The metadata is stored in memory and can use up to 1% of available memory by default. The size could be adjusted by `-storage.maxMetadataStorageSize` flag.
-Please note that metadata is lost after restarts. It is ingested independently from metrics, so a metric may exist without metadata, and vice versa.
+The metadata is cached in-memory in a ring buffer and can use up to 1% of available memory by default (see `-storage.maxMetadataStorageSize` cmd-line flag).
+When in-memory size is exceeded, the least updated entries are dropped first. Entries that weren't updated for 1h are cleaned up automatically.
+
+> The following expression helps to understand if metadata cache capacity is utilized for more than 90%: `vm_metrics_metadata_storage_size_bytes / vm_metrics_metadata_storage_max_size_bytes > 0.9`.
+Setup [monitoring](https://docs.victoriametrics.com/victoriametrics/quick-start/#monitoring) and recommended alerting rules
+to get notified about cache capacity issues.
+
+Metadata is ingested independently from metrics, so a metric can exist without metadata, and vice versa.
+Metadata is expected to be ephemeral and constantly updated on ingestion. For this reason, metadata cache isn't 
+persisted during restarts.
 
 Metadata can be queried via the `/api/v1/metadata` endpoint, which provides a response compatible with the Prometheus [metadata API](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata).
 See [/api/v1/metadata](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1metadata) example.
@@ -1671,6 +1682,9 @@ See [Why IndexDB size is so large?](https://docs.victoriametrics.com/victoriamet
 Downsampling is performed during [background merges](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#storage).
 It cannot be performed if there is not enough of free disk space or if vmstorage is in [read-only mode](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#readonly-mode).
 
+Downsampling period changes `/api/v1/export` API output. During query requests, if export `start` period is not specified and `reduce_mem_usage` param is omitted, the biggest `downsampling.period` is applied.
+As an example, export request  `/api/v1/export?match[]=series` with `-downsampling.period=30d:1h,180d:24h` will return samples downsampled with `24h` interval.
+
 It's expected that resource usage will temporarily increase when **downsampling with filters** is applied.
 This is because additional operations are required to read historical data, downsample, and persist it back,
 which will cost extra CPU and memory.

docs/victoriametrics/Single-server-VictoriaMetrics.md

@@ -13,5 +13,6 @@ aliases:
   - /Single-server-VictoriaMetrics.html
   - /single-server-victoriametrics/index.html
   - /single-server-victoriametrics/
+  - /VictoriaMetrics.html
 ---
 {{% content "README.md" %}}

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,21 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* SECURITY: upgrade Go builder from Go1.25.6 to Go1.25.7. See [the list of issues addressed in Go1.25.7](https://github.com/golang/go/issues?q=milestone%3AGo1.25.7%20label%3ACherryPickApproved).
+* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.9-3.21.6-3.22.3-3.23.3-released.html).
+
+* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229), [dashboards/cluster](https://grafana.com/grafana/dashboards/11176), [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add clickable source code links to the `Logging rate` panel in `Drilldown`. Users can use it to navigate directly to the source code location that generated those logs, making debugging and code exploration easier. See [#10406](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10406).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `Queries with most memory to execute` section in `Top Queries` page of `vmui`. It can help users to find queries that consume most memory and potentially cause OOM. See [#9330](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9330).
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): make label value autocomplete context-aware by suggesting values only from series matching already selected label filters. See [#9269](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9269).
+
+* BUGFIX: all VictoriaMetrics components: respect default http client proxy env variables (HTTP_PROXY,HTTPS_PROXY,NO_PROXY). See [#10385](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10385). Thanks to @zane-deg for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly expose `kubernetes_sd` discovery network dialer metrics `vm_promscrape_discovery_kubernetes_conn_*`. See [#10382](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10382).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): slightly reduce memory usage for [metrics-metadata](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#metrics-metadata) ingestion. See [#10392](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10392).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce time‑based manual offset commit for kafka consumer to fix performance degradation with enabled manual commit. After this change, it will commit partition offsets in batch per second to avoid high commit QPS on the Kafka broker. It's no longer recommended to set `enable.auto.commit=true` in `-kafka.consumer.topic.options`, as `vmagent` will automatically manage it. See [#10395](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10395).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix `vm_deduplicated_samples_total{type="select"}` to correctly count deduplicated identical samples with the same timestamp and value. See [#10400](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10400).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): previously the [Graphite render API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#render-api) used a fixed process timeout that could expire before long queries completed; now the timeout follows the query deadline, so users can extend it via `-search.maxQueryDuration` or the `timeout` [argument in the query](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#query-data). See [#8484](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8484).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid slow-path deduplication for time series with [staleness markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). See [#10384](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10384).
+
 ## [v1.135.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.135.0)
 
 Released at 2026-01-30
@@ -84,7 +99,7 @@ Released at 2026-01-16
 Released at 2026-01-02
 
 **Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Upgrading to per-partition index requires registering all active time series. Expect slow down of data ingestion and queries during upgrade roll-out. This is a one-time operation. Additionally, for users with retention periods shorter than 1 month the disk usage may increase.
-**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Running this version in deployments with big datasets may cause high CPU utilization. See [10297](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10297).
+**Update Note 2:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Certain data and query patterns may cause high CPU utilization due to [10154](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10154). See [10297](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10297).
 **Update Note 3:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): lock contention in the ingestion path may cause frequent context switches and storage connection saturation spikes. See [10367](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10367). Addressed in `v1.135.0`.
 **Update Note 4:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): TSDB status may be empty if the partition index does not have records for the requested date. See [10315](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10315). Addressed in `v1.135.0`.
 **Update Note 5:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `indexdb/tagFiltersToMetricIDs`, `indexdb/metricID` and `indexdb/date_metricID` cache metrics are not reported properly. See [10275](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10275). Addressed in `v1.135.0`.
@@ -153,6 +168,22 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
 
+## [v1.122.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.14)
+
+Released at 2026-01-30
+
+**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
+
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): stop backend health checks for URL prefixes defined in `url_map` during configuration reloads. Previously, stale backends kept being health-checked and produced repeated warning logs after reloads. See [#10334](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10334).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly return tenants results for `/admin/tenants` when `start` or `end` are specified. See [#10312](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10312)
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix "Percentage from total" calculation on the Cardinality Explorer page when multiple metrics match the filter. See [#10323](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10323). Thanks to @PleasingFungus for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): apply `-promscrape.maxScrapeSize` check to decompressed data instead of compressed data. See [#9481](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9481).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix [alert restore](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerts-state-on-restarts) when a group contains many rules and is slow to complete evaluation. Previously, the restore process might not retrieve the correct previous alert state. See [#10335](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix `changes()` function when gaps between samples exceed the lookbehind window. Previously, it could yield a non-zero value even when the sample value remained unchanged. See [#10280](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fix an issue where canceling a client request (closing a browser tab or timeout) incorrectly marked all backends as unavailable for `-failTimeout` duration (3s by default), even though backends were healthy. See [#10318](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10318).
+
 ## [v1.122.13](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.13)
 
 Released at 2026-01-16
@@ -268,6 +299,21 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11110)
 
+## [v1.110.29](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.29)
+
+Released at 2026-01-30
+
+**v1.110.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11100) release**
+
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix `changes()` function when gaps between samples exceed the lookbehind window. Previously, it could yield a non-zero value even when the sample value remained unchanged. See [#10280](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10280).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix [alert restore](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerts-state-on-restarts) when a group contains many rules and is slow to complete evaluation. Previously, the restore process might not retrieve the correct previous alert state. See [#10335](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10335).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fix an issue where canceling a client request (closing a browser tab or timeout) incorrectly marked all backends as unavailable for `-failTimeout` duration (3s by default), even though backends were healthy. See [#10318](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10318).
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): stop backend health checks for URL prefixes defined in `url_map` during configuration reloads. Previously, stale backends kept being health-checked and produced repeated warning logs after reloads. See [#10334](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10334).
+* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly return tenants results for `/admin/tenants` when `start` or `end` are specified. See [#10312](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10312)
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix "Percentage from total" calculation on the Cardinality Explorer page when multiple metrics match the filter. See [#10323](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10323). Thanks to @PleasingFungus for the contribution.
+
 ## [v1.110.28](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.28)
 
 Released at 2026-01-16

docs/victoriametrics/changelog/CHANGELOG_2025.md

@@ -78,6 +78,8 @@ Read [VictoriaLogs#869](https://github.com/VictoriaMetrics/VictoriaLogs/issues/8
 
 **Known issue: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): may shutdown ungracefully (data, indexes, and caches are not flushed to disk) in Kubernetes deployments if the number of `vminserts` is > 1 and `terminationGracePeriodSeconds` < 60s (30s by default). The issue was introduced in [9487](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9487) and [10136](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10136) and was fixed in [10224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10224). The fix will be included into `v1.133.0`. Even with this fix, ungraceful shutdowns are possible if flushing in-memory contents takes more than 5s. In this case decrease `-storage.vminsertConnsShutdownDuration` and/or increase [terminationGracePeriodSeconds](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination-flow).**
 
+**Known issue:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Certain data and query patterns may cause high CPU utilization due to [10154](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10154). See [10297](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10297).
+
 * SECURITY: upgrade Go builder from Go1.25.4 to Go1.25.5. See [the list of issues addressed in Go1.25.5](https://github.com/golang/go/issues?q=milestone%3AGo1.25.5%20label%3ACherryPickApproved).
 
 * FEATURE: [dashboards/operator](https://grafana.com/grafana/dashboards/17869-victoriametrics-operator/): add panels for flags and configuration parameters values. See [operator#1341](https://github.com/VictoriaMetrics/operator/issues/1341).
@@ -2257,4 +2259,4 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 ## Previous releases
 
-See changes for older releases [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2024/).
+See changes for older releases [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2024/).

docs/victoriametrics/enterprise.md

@@ -117,7 +117,7 @@ It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [
 
 Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
 and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.134.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.135.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -135,8 +135,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.134.0/victoria-metrics-linux-amd64-v1.134.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.134.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.135.0/victoria-metrics-linux-amd64-v1.135.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.135.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -151,7 +151,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.134.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.135.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
 flag as described in the [binary-releases](#binary-releases) section.
@@ -161,13 +161,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.134.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.135.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.134.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.135.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -177,7 +177,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.134.0
+    image: victoriametrics/victoria-metrics:v1.135.0
     ports:
       - 8428:8428
     volumes:
@@ -209,7 +209,7 @@ is used to provide the license key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.134.0-enterprise
+    tag: v1.135.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -220,7 +220,7 @@ In order to provide the license key via existing secret, the following values fi
 ```yaml
 server:
   image:
-    tag: v1.134.0-enterprise
+    tag: v1.135.0-enterprise
 
 license:
   secret:
@@ -270,7 +270,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.134.0-enterprise
+    tag: v1.135.0-enterprise
 ```
 
 In order to provide the license key via an existing secret, the following custom resource is used:
@@ -287,7 +287,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.134.0-enterprise
+    tag: v1.135.0-enterprise
 ```
 
 Example secret with license key:
@@ -338,7 +338,7 @@ Builds are available for amd64 and arm64 architectures.
 
 Example archive:
 
-`victoria-metrics-linux-amd64-v1.134.0-enterprise.tar.gz`
+`victoria-metrics-linux-amd64-v1.135.0-enterprise.tar.gz`
 
 Includes:
 
@@ -347,7 +347,7 @@ Includes:
 
 Example Docker image:
 
-`victoriametrics/victoria-metrics:v1.134.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
+`victoriametrics/victoria-metrics:v1.135.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
 
 ## Monitoring license expiration
 

docs/victoriametrics/integrations/zabbixconnector.md

@@ -5,6 +5,8 @@ menu:
   docs:
     parent: "integrations-vm"
     weight: 10
+aliases:
+  - /victoriametrics/integrations/zabbix/
 ---
 
 VictoriaMetrics components like **vmagent**, **vminsert** or **single-node** can receive data from 
@@ -61,4 +63,4 @@ curl http://localhost:8428/api/v1/export -d 'match={host="Zabbix server"}'
 {"metric":{"__name__":"item_1","host":"ZabbixServer","hostname":"ZabbixServer","group_servers":"exists","tag_foo":"exists"},"values":[0],"timestamps":[1673454303800]}
 {"metric":{"__name__":"item_2","host":"ZabbixServer","hostname":"ZabbixServer","group_servers":"exists","tag_foo":"test,exists"},"values":[1],"timestamps":[1673454303832]}
 {"metric":{"__name__":"item_3","host":"ZabbixServer","hostname":"ZabbixServer","group_servers":"exists","tag_bar":"test"},"values":[123],"timestamps":[1673454303867]}
-```
+```

docs/victoriametrics/query-stats.md

@@ -26,7 +26,7 @@ Here's how `<duration>` works:
 
 * `-search.logSlowQueryStats=5s` logs statistics for queries that take longer than `5s`;
 * `-search.logSlowQueryStats=1us` logs statistics for **all queries**;
-* `-search.logSlowQueryStats=0` turns off query stats logging (this is the default).
+* `-search.logSlowQueryStats=0` turns off query stats logging.
 
 **Example of a query statistics log:**
 

docs/victoriametrics/scrape_config_examples.md

@@ -35,8 +35,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.134.0/victoria-metrics-linux-amd64-v1.134.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.134.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.135.0/victoria-metrics-linux-amd64-v1.135.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.135.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.134.0/victoria-metrics-linux-amd64-v1.134.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.134.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.135.0/victoria-metrics-linux-amd64-v1.135.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.135.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/vmagent.md

@@ -114,6 +114,8 @@ The maximum on-disk size for the buffered metrics can be limited with `-remoteWr
 
 `vmagent` can save on network bandwidth usage costs by using [VictoriaMetrics remote write protocol](#victoriametrics-remote-write-protocol).
 
+See [how to optimize index size at VictoriaMetrics for IoT and industrial monitoring](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning-for-low-churn-rate).
+
 ### Drop-in replacement for Prometheus
 
 If you use Prometheus only for scraping metrics from various targets and forwarding these metrics to remote storage
@@ -284,15 +286,15 @@ flowchart TB
       %% Left branch
       G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
       H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
-      H2 --> H3[per-url extra labels<br><b>-remoteWrite.label</b>]
-      H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
+      H2 --> H3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
+      H3 --> H4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
       H4 --> H5[[push to <b>-remoteWrite.url</b>]]
 
       %% Right branch
       G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
       R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
-      R2 --> R3[per-url extra labels<br><b>-remoteWrite.label</b>]
-      R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
+      R2 --> R3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
+      R3 --> R4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
       R4 --> R5[[push to <b>-remoteWrite.url</b>]]
 ```
 
@@ -509,15 +511,20 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
 
 * The `global -> external_labels` section in `-promscrape.config` file. These labels are added only to metrics scraped from targets configured
   in the `-promscrape.config` file. They aren't added to metrics collected via other [data ingestion protocols](#how-to-push-data-to-vmagent).
-* The `-remoteWrite.label` command-line flag. These labels are added to all the collected metrics before sending them to `-remoteWrite.url`.
+* The `-remoteWrite.label` command-line flag. These labels are added **to all the collected metrics** before sending them **to all configured `-remoteWrite.url`**.
   For example, the following command starts `vmagent`, which adds `{datacenter="foobar"}` label to all the metrics pushed
-  to all the configured remote storage systems (all the `-remoteWrite.url` flag values):
+  to all the configured `-remoteWrite.url` destinations:
 
   ```sh
   /path/to/vmagent -remoteWrite.label=datacenter=foobar ...
   ```
 
-* Via relabeling. See [Relabeling Cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/).
+* Via relabeling. Relabeling can be applied globally and per each configured `-remoteWrite.url` destination. See [Relabeling Cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/).
+* Add `extra_label` GET param to `-remoteWrite.url` address (only works when sending data to VictoriaMetrics components):
+
+  ```sh
+  /path/to/vmagent -remoteWrite.url=http://127.0.0.1:8428/api/v1/write?extra_label="env=prod"
+  ```
 
 ## Automatically generated metrics
 

docs/victoriametrics/vmagent_common_flags.md

@@ -412,7 +412,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
   -remoteWrite.keepDanglingQueues
      Keep persistent queues contents at -remoteWrite.tmpDataPath in case there are no matching -remoteWrite.url. Useful when -remoteWrite.url is changed temporarily and persistent queue files will be needed later on.
   -remoteWrite.label array
-     Optional label in the form 'name=value' to add to all the metrics before sending them to -remoteWrite.url. Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage
+     Optional label in the form 'name=value' to add to all the metrics before sending them to all -remoteWrite.url.
      Supports an array of values separated by comma or specified via multiple flags.
      Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
   -remoteWrite.maxBlockSize size

docs/victoriametrics/vmctl/vmctl.md

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
 
 Download and unpack vmctl:
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.134.0/vmutils-darwin-arm64-v1.134.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.135.0/vmutils-darwin-arm64-v1.135.0.tar.gz
 
-tar xzf vmutils-darwin-arm64-v1.134.0.tar.gz
+tar xzf vmutils-darwin-arm64-v1.135.0.tar.gz
 ```
 
 Once binary is unpacked, see the full list of supported modes by running the following command:

go.mod

@@ -1,6 +1,6 @@
 module github.com/VictoriaMetrics/VictoriaMetrics
 
-go 1.25.6
+go 1.25.7
 
 require (
 	cloud.google.com/go/storage v1.59.1

lib/backup/actions/restore.go

@@ -112,6 +112,9 @@ func (r *Restore) Run(ctx context.Context) error {
 		}
 		offset += p.Size
 	}
+	if offset != pOld.FileSize {
+		return fmt.Errorf("invalid size for %q; got %d; want %d", path, offset, pOld.FileSize)
+	}
 
 	partsToDelete := common.PartsDifference(dstParts, srcParts)
 	deleteSize := uint64(0)

lib/fasttime/fasttime_normal.go

@@ -1,4 +1,4 @@
-//go:build !goexperiment.synctest
+//go:build !synctest
 
 package fasttime
 

lib/fasttime/fasttime_synctest.go

@@ -1,4 +1,4 @@
-//go:build goexperiment.synctest
+//go:build synctest
 
 package fasttime
 

lib/jwt/algo.go

@@ -0,0 +1,58 @@
+package jwt
+
+import (
+	"crypto"
+	_ "crypto/sha256" // to register a hash
+	_ "crypto/sha512" // to register a hash
+	"errors"
+)
+
+// Verifier is used to verify tokens.
+type Verifier interface {
+	Verify(token *Token) error
+}
+
+// Algorithm for signing and verifying.
+type Algorithm string
+
+func (a Algorithm) String() string { return string(a) }
+
+// Algorithm names for signing and verifying.
+const (
+	RS256 Algorithm = "RS256"
+	RS384 Algorithm = "RS384"
+	RS512 Algorithm = "RS512"
+
+	ES256 Algorithm = "ES256"
+	ES384 Algorithm = "ES384"
+	ES512 Algorithm = "ES512"
+
+	PS256 Algorithm = "PS256"
+	PS384 Algorithm = "PS384"
+	PS512 Algorithm = "PS512"
+)
+
+// JWT sign, verify, build and parse errors.
+var (
+	// ErrNilKey indicates that key is nil.
+	ErrNilKey = errors.New("key is nil")
+
+	// ErrInvalidKey indicates that key is not valid.
+	ErrInvalidKey = errors.New("key is not valid")
+
+	// ErrUnsupportedAlg indicates that given algorithm is not supported.
+	ErrUnsupportedAlg = errors.New("algorithm is not supported")
+
+	// ErrInvalidSignature indicates that signature is not valid.
+	ErrInvalidSignature = errors.New("signature is not valid")
+)
+
+func hashPayload(hash crypto.Hash, payload []byte) ([]byte, error) {
+	hasher := hash.New()
+
+	if _, err := hasher.Write(payload); err != nil {
+		return nil, err
+	}
+	signed := hasher.Sum(nil)
+	return signed, nil
+}

lib/jwt/algo_es.go

@@ -0,0 +1,104 @@
+package jwt
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"math/big"
+)
+
+// newVerifierES returns a new ECDSA-based verifier.
+func newVerifierES(alg Algorithm, key *ecdsa.PublicKey) (*esAlg, error) {
+	if key == nil {
+		return nil, ErrNilKey
+	}
+	hash, err := getParamsES(alg, roundBytes(key.Params().BitSize)*2)
+	if err != nil {
+		return nil, err
+	}
+	return &esAlg{
+		alg:       alg,
+		hash:      hash,
+		publicKey: key,
+		signSize:  signSize(key),
+	}, nil
+}
+
+func signSize(key *ecdsa.PublicKey) int {
+	return roundBytes(key.Params().BitSize) * 2
+}
+
+func getAlgorithmForKey(key *ecdsa.PublicKey) Algorithm {
+	switch signSize(key) {
+	case 64:
+		return ES256
+	case 96:
+		return ES384
+	case 132:
+		return ES512
+	}
+	return ""
+}
+
+func getParamsES(alg Algorithm, size int) (crypto.Hash, error) {
+	var hash crypto.Hash
+	var keySize int
+
+	switch alg {
+	case ES256:
+		hash, keySize = crypto.SHA256, 64
+	case ES384:
+		hash, keySize = crypto.SHA384, 96
+	case ES512:
+		hash, keySize = crypto.SHA512, 132
+	default:
+		return 0, ErrUnsupportedAlg
+	}
+
+	if keySize != size {
+		return 0, ErrInvalidKey
+	}
+	return hash, nil
+}
+
+type esAlg struct {
+	alg       Algorithm
+	hash      crypto.Hash
+	publicKey *ecdsa.PublicKey
+	signSize  int
+}
+
+func (es *esAlg) SignSize() int {
+	return es.signSize
+}
+
+func (es *esAlg) Verify(token *Token) error {
+	return es.verify(token.payload, token.signature)
+}
+
+func (es *esAlg) verify(payload, signature []byte) error {
+	if len(signature) != es.SignSize() {
+		return ErrInvalidSignature
+	}
+
+	digest, err := hashPayload(es.hash, payload)
+	if err != nil {
+		return err
+	}
+
+	pivot := es.SignSize() / 2
+	r := big.NewInt(0).SetBytes(signature[:pivot])
+	s := big.NewInt(0).SetBytes(signature[pivot:])
+
+	if !ecdsa.Verify(es.publicKey, digest, r, s) {
+		return ErrInvalidSignature
+	}
+	return nil
+}
+
+func roundBytes(n int) int {
+	res := n / 8
+	if n%8 > 0 {
+		return res + 1
+	}
+	return res
+}

lib/jwt/algo_ps.go

@@ -0,0 +1,77 @@
+package jwt
+
+import (
+	"crypto"
+	"crypto/rsa"
+)
+
+// newVerifierPS returns a new RSA-PSS-based verifier.
+func newVerifierPS(alg Algorithm, key *rsa.PublicKey) (*psAlg, error) {
+	if key == nil {
+		return nil, ErrNilKey
+	}
+	hash, opts, err := getParamsPS(alg)
+	if err != nil {
+		return nil, err
+	}
+	return &psAlg{
+		alg:       alg,
+		hash:      hash,
+		publicKey: key,
+		opts:      opts,
+	}, nil
+}
+
+func getParamsPS(alg Algorithm) (crypto.Hash, *rsa.PSSOptions, error) {
+	switch alg {
+	case PS256:
+		return crypto.SHA256, optsPS256, nil
+	case PS384:
+		return crypto.SHA384, optsPS384, nil
+	case PS512:
+		return crypto.SHA512, optsPS512, nil
+	default:
+		return 0, nil, ErrUnsupportedAlg
+	}
+}
+
+var (
+	optsPS256 = &rsa.PSSOptions{
+		SaltLength: rsa.PSSSaltLengthAuto,
+		Hash:       crypto.SHA256,
+	}
+
+	optsPS384 = &rsa.PSSOptions{
+		SaltLength: rsa.PSSSaltLengthAuto,
+		Hash:       crypto.SHA384,
+	}
+
+	optsPS512 = &rsa.PSSOptions{
+		SaltLength: rsa.PSSSaltLengthAuto,
+		Hash:       crypto.SHA512,
+	}
+)
+
+type psAlg struct {
+	alg       Algorithm
+	hash      crypto.Hash
+	publicKey *rsa.PublicKey
+	opts      *rsa.PSSOptions
+}
+
+func (ps *psAlg) Verify(token *Token) error {
+	return ps.verify(token.payload, token.signature)
+}
+
+func (ps *psAlg) verify(payload, signature []byte) error {
+	digest, err := hashPayload(ps.hash, payload)
+	if err != nil {
+		return err
+	}
+
+	errVerify := rsa.VerifyPSS(ps.publicKey, ps.hash, digest, signature, ps.opts)
+	if errVerify != nil {
+		return ErrInvalidSignature
+	}
+	return nil
+}

lib/jwt/algo_rs.go

@@ -0,0 +1,60 @@
+package jwt
+
+import (
+	"crypto"
+	"crypto/rsa"
+)
+
+// newVerifierRS returns a new RSA-based verifier.
+func newVerifierRS(alg Algorithm, key *rsa.PublicKey) (*rsAlg, error) {
+	if key == nil {
+		return nil, ErrNilKey
+	}
+	hash, err := getHashRS(alg)
+	if err != nil {
+		return nil, err
+	}
+	return &rsAlg{
+		alg:       alg,
+		hash:      hash,
+		publicKey: key,
+	}, nil
+}
+
+func getHashRS(alg Algorithm) (crypto.Hash, error) {
+	var hash crypto.Hash
+	switch alg {
+	case RS256:
+		hash = crypto.SHA256
+	case RS384:
+		hash = crypto.SHA384
+	case RS512:
+		hash = crypto.SHA512
+	default:
+		return 0, ErrUnsupportedAlg
+	}
+	return hash, nil
+}
+
+type rsAlg struct {
+	alg       Algorithm
+	hash      crypto.Hash
+	publicKey *rsa.PublicKey
+}
+
+func (rs *rsAlg) Verify(token *Token) error {
+	return rs.verify(token.payload, token.signature)
+}
+
+func (rs *rsAlg) verify(payload, signature []byte) error {
+	digest, err := hashPayload(rs.hash, payload)
+	if err != nil {
+		return err
+	}
+
+	errVerify := rsa.VerifyPKCS1v15(rs.publicKey, rs.hash, digest, signature)
+	if errVerify != nil {
+		return ErrInvalidSignature
+	}
+	return nil
+}

lib/jwt/jwt.go

@@ -0,0 +1,289 @@
+package jwt
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	prefix = "Bearer "
+)
+
+const (
+	read = 1 << iota
+	write
+)
+
+var (
+	// ErrHeaderMissing missing header.
+	ErrHeaderMissing = fmt.Errorf("jwt authorization header is missing")
+	// ErrVMAccessFieldMissing missing vm_access field.
+	ErrVMAccessFieldMissing = fmt.Errorf("missing `vm_access` claim")
+	// ErrBadTokenFormat incorrect format for token
+	ErrBadTokenFormat = fmt.Errorf("bad token format, must be jwt")
+)
+
+// Token represents jwt token
+// https://auth0.com/docs/tokens/json-web-tokens
+type Token struct {
+	header             *header
+	body               *body
+	payload, signature []byte
+}
+
+type header struct {
+	Alg string `json:"alg"`
+	Typ string `json:"typ"`
+	Kid string `json:"kid"`
+}
+
+type body struct {
+	// expired at time unix_ts
+	Exp int64 `json:"exp"`
+	// issued at time unix_ts
+	Iat      int64   `json:"iat"`
+	Jti      string  `json:"jti,omitempty"`
+	Scope    string  `json:"scope,omitempty"`
+	VMAccess *access `json:"vm_access"`
+}
+
+// Labels defines labels added to filters or incoming time series.
+type Labels map[string]string
+
+// AsExtraLabels - converts labels to label=value pairs.
+func (l Labels) AsExtraLabels() []string {
+	if len(l) == 0 {
+		return nil
+	}
+	res := make([]string, 0, len(l))
+	for k, v := range l {
+		res = append(res, k+"="+v)
+	}
+	// sort for consistent uri.
+	sort.Slice(res, func(i, j int) bool {
+		return res[i] < res[j]
+	})
+	return res
+}
+
+type access struct {
+	Tenant TenantID `json:"tenant_id,omitempty"`
+	Labels Labels   `json:"extra_labels,omitempty"`
+	// promql filters applied to each select query
+	ExtraFilters []string `json:"extra_filters,omitempty"`
+	// role can be denied as 1 = read, 2 = write, 3 = read and write
+	// 0 = unconfigured - read and write
+	Mode int `json:"mode,omitempty"`
+}
+
+// TenantID represents tenantID.
+type TenantID struct {
+	ProjectID int32 `json:"project_id"`
+	AccountID int32 `json:"account_id"`
+}
+
+// String implements interface.
+func (tid TenantID) String() string {
+	return fmt.Sprintf("%d:%d", tid.AccountID, tid.ProjectID)
+}
+
+// NewToken creates token from raw string.
+func NewToken(auth string, enforceAuthPrefix bool) (*Token, error) {
+	if enforceAuthPrefix && (len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix)) {
+		return nil, fmt.Errorf("wrong format, prefix: %s is missing", prefix)
+	}
+
+	// While https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 states that only Bearer prefix is allowed,
+	// it claims to be conformant to the generic syntax defined in https://datatracker.ietf.org/doc/html/rfc2617#section-1.2
+	// which permits case-insensitive auth scheme.
+	// So we should be tolerant to different cases of "Bearer" prefix.
+	if len(auth) >= len(prefix) && strings.EqualFold(auth[:len(prefix)], prefix) {
+		auth = auth[len(prefix):]
+	}
+
+	jwt := strings.SplitN(auth, ".", 3)
+	if len(jwt) != 3 {
+		return nil, ErrBadTokenFormat
+	}
+	var t Token
+	return t.parse(jwt[0], jwt[1], jwt[2])
+}
+
+// NewTokenFromRequestWithCustomHeader return new jwt token from request by provided header
+func NewTokenFromRequestWithCustomHeader(r *http.Request, headerName string, enforceAuthPrefix bool) (*Token, error) {
+	auth := r.Header.Get(headerName)
+	if len(auth) == 0 {
+		return nil, ErrHeaderMissing
+	}
+	return NewToken(auth, enforceAuthPrefix)
+}
+
+func (t *Token) parse(header, body, signature string) (*Token, error) {
+	b, err := parseJWTBody(body)
+	if err != nil {
+		return nil, err
+	}
+	if b.VMAccess == nil {
+		return nil, ErrVMAccessFieldMissing
+	}
+	t.body = b
+	h, err := parseJWTHeader(header)
+	if err != nil {
+		return nil, err
+	}
+	t.header = h
+
+	t.payload = []byte(header + "." + body)
+	t.signature, err = decodeB64([]byte(signature))
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode signature as b64: %w", err)
+	}
+
+	return t, nil
+}
+
+// IsExpired checks if jwt token is expired.
+func (t *Token) IsExpired(currentTime time.Time) bool {
+	return currentTime.Unix() > t.body.Exp
+}
+
+// CanWrite checks if token has write permissions.
+func (t *Token) CanWrite() bool {
+	// unconfigured
+	if t.body.VMAccess.Mode == 0 {
+		return true
+	}
+	if write&t.body.VMAccess.Mode > 0 {
+		return true
+	}
+	return false
+}
+
+// CanRead check if token has read permissions.
+func (t *Token) CanRead() bool {
+	// unconfigured
+	if t.body.VMAccess.Mode == 0 {
+		return true
+	}
+	if read&t.body.VMAccess.Mode > 0 {
+		return true
+	}
+	return false
+}
+
+// AccessLabels returns access labels for given JWT token,
+// in key=value format.
+func (t *Token) AccessLabels() []string {
+	return t.body.VMAccess.Labels.AsExtraLabels()
+}
+
+// Tenant returns tenantID for token.
+func (t *Token) Tenant() TenantID {
+	return t.body.VMAccess.Tenant
+}
+
+// ExtraFilters metricsql filters for select queries
+func (t *Token) ExtraFilters() []string {
+	return t.body.VMAccess.ExtraFilters
+}
+
+func parseJWTHeader(data string) (*header, error) {
+	var jh header
+	decoded, err := decodeB64([]byte(data))
+	if err != nil {
+		return nil, fmt.Errorf("cannot decode jwt header as b64: %w", err)
+	}
+	if err := json.Unmarshal(decoded, &jh); err != nil {
+		return nil, fmt.Errorf("cannot parse jwt header: %w", err)
+	}
+	return &jh, nil
+}
+
+func parseJWTBody(data string) (*body, error) {
+	type tbody struct {
+		// expired at time unix_ts
+		Exp int64 `json:"exp"`
+		// issued at time unix_ts
+		Iat   int64           `json:"iat"`
+		Jti   string          `json:"jti,omitempty"`
+		Scope json.RawMessage `json:"scope,omitempty"`
+		// store as raw message to support different types
+		VMAccess *json.RawMessage `json:"vm_access"`
+	}
+	var tb tbody
+
+	decoded, err := decodeB64([]byte(data))
+	if err != nil {
+		return nil, fmt.Errorf("cannot decode jwt body as b64: %w", err)
+	}
+	if err := json.Unmarshal(decoded, &tb); err != nil {
+		return nil, fmt.Errorf("cannot parse jwt body: %w", err)
+	}
+
+	if tb.VMAccess == nil {
+		return nil, ErrVMAccessFieldMissing
+	}
+
+	// some IDPs encode custom claims as a string
+	// try parsing as an object and fallback to a string
+	var a access
+	if err := json.Unmarshal(*tb.VMAccess, &a); err != nil {
+		var s string
+		if err := json.Unmarshal(*tb.VMAccess, &s); err != nil {
+			return nil, fmt.Errorf("cannot parse jwt body vm_access: %w", err)
+		}
+
+		if err := json.Unmarshal([]byte(s), &a); err != nil {
+			return nil, fmt.Errorf("cannot parse jwt body vm_access: %w", err)
+		}
+	}
+
+	// some IDPs encode scope as a string and some as an array
+	var scope string
+	if tb.Scope != nil {
+		if err := json.Unmarshal(tb.Scope, &scope); err != nil {
+			var scopeSlice []string
+			if err := json.Unmarshal(tb.Scope, &scopeSlice); err != nil {
+				return nil, fmt.Errorf("cannot parse jwt body scope: %w", err)
+			}
+			scope = strings.Join(scopeSlice, " ")
+		}
+	}
+
+	parsedBody := &body{
+		Exp:      tb.Exp,
+		Iat:      tb.Iat,
+		Jti:      tb.Jti,
+		Scope:    scope,
+		VMAccess: &a,
+	}
+	return parsedBody, nil
+}
+
+func decodeB64(data []byte) ([]byte, error) {
+	idx := bytes.IndexAny(data, "+/")
+	// slow path, std base64 encoding convert it to url encoding
+	if idx >= 0 {
+		for idx, c := range data {
+			switch c {
+			case '+':
+				data[idx] = '-'
+			case '/':
+				data[idx] = '_'
+			}
+		}
+
+	}
+	dst := make([]byte, base64.RawURLEncoding.DecodedLen(len(data)))
+	_, err := base64.RawURLEncoding.Decode(dst, data)
+	if err != nil {
+		return nil, fmt.Errorf("cannot decode jwt body as b64: %w", err)
+	}
+	return dst, nil
+}

lib/jwt/jwt_test.go

@@ -0,0 +1,661 @@
+package jwt
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+func TestParseJWTHeader_Failure(t *testing.T) {
+	f := func(data, expectedErr string, encode bool) {
+		t.Helper()
+		if encode {
+			encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
+			encoded := make([]byte, encodedLen)
+			base64.RawURLEncoding.Encode(encoded, []byte(data))
+			data = string(encoded)
+		}
+		if _, err := parseJWTHeader(data); err != nil {
+			if err.Error() != expectedErr {
+				t.Errorf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
+			}
+		} else {
+			t.Errorf("expecting non-nil error")
+		}
+	}
+
+	// invalid input
+	f(
+		`bad input`,
+		`cannot decode jwt header as b64: cannot decode jwt body as b64: illegal base64 data at input byte 3`,
+		false,
+	)
+
+	// invalid b644
+	f(
+		`YmFk`,
+		`cannot parse jwt header: invalid character 'b' looking for beginning of value`,
+		false,
+	)
+
+	// invalid header json
+	f(`{]`,
+		`cannot parse jwt header: invalid character ']' looking for beginning of object key string`,
+		true,
+	)
+
+	// invalid header type json
+	f(`[]`,
+		`cannot parse jwt header: json: cannot unmarshal array into Go value of type jwt.header`,
+		true,
+	)
+}
+
+func TestParseJWTHeader_Success(t *testing.T) {
+	f := func(data string, expected *header) {
+		t.Helper()
+		encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
+		encoded := make([]byte, encodedLen)
+		base64.RawURLEncoding.Encode(encoded, []byte(data))
+		header, err := parseJWTHeader(string(encoded))
+		if err != nil {
+			t.Fatalf("parseJWTHeader() error: %s", err)
+		}
+		if !reflect.DeepEqual(header, expected) {
+			t.Fatalf("unexpected token header;\ngot\n%v\nwant\n%v", header, expected)
+		}
+	}
+
+	// parse ok supported algorithms
+	supportedAlgorithms := []string{
+		"RS256", "RS384", "RS512", "ES256", "ES384", "ES512", "PS256", "PS384", "PS512",
+	}
+	for i := range supportedAlgorithms {
+		f(fmt.Sprintf(`{
+			"alg": %q,
+			"kid": "test"
+		}`, supportedAlgorithms[i]),
+			&header{
+				Alg: supportedAlgorithms[i],
+				Kid: "test",
+			},
+		)
+	}
+}
+
+func TestParseJWTBody_Failure(t *testing.T) {
+	f := func(data, expectedErr string, encode bool) {
+		t.Helper()
+		if encode {
+			encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
+			encoded := make([]byte, encodedLen)
+			base64.RawURLEncoding.Encode(encoded, []byte(data))
+			data = string(encoded)
+		}
+		if _, err := parseJWTBody(data); err != nil {
+			if err.Error() != expectedErr {
+				t.Errorf("unexpected error message: \ngot\n%s\nwant\n%s", err.Error(), expectedErr)
+			}
+		} else {
+			t.Errorf("expecting non-nil error")
+		}
+	}
+
+	// invalid input
+	f(
+		`bad input`,
+		`cannot decode jwt body as b64: cannot decode jwt body as b64: illegal base64 data at input byte 3`,
+		false,
+	)
+
+	// invalid b644
+	f(
+		`YmFk`,
+		`cannot parse jwt body: invalid character 'b' looking for beginning of value`,
+		false,
+	)
+
+	// invalid body json
+	f(
+		`{]`,
+		`cannot parse jwt body: invalid character ']' looking for beginning of object key string`,
+		true,
+	)
+
+	// invalid body type json
+	f(
+		`[]`,
+		`cannot parse jwt body: json: cannot unmarshal array into Go value of type jwt.tbody`,
+		true,
+	)
+
+	// missing vm_access claim
+	f(
+		`{}`,
+		"missing `vm_access` claim",
+		true,
+	)
+
+	// vm_access claim invalid type
+	f(
+		`{"vm_access": 123}`,
+		"cannot parse jwt body vm_access: json: cannot unmarshal number into Go value of type string",
+		true,
+	)
+
+	// vm_access claim null
+	f(
+		`{"vm_access": null}`,
+		"missing `vm_access` claim",
+		true,
+	)
+
+	// invalid vm_access: account_id type mismatch
+	f(
+		`{"vm_access": {"tenant_id": {"account_id": "1", "project_id": 5}}}`,
+		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		true,
+	)
+
+	// invalid vm_access: project_id type mismatch
+	f(
+		`{"vm_access": {"tenant_id": {"account_id": 1, "project_id": "5"}}}`,
+		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		true,
+	)
+
+	// invalid vm_access: extra_label type mismatch
+	f(`
+{
+	"vm_access": {
+		"extra_labels": [{
+			"project": "dev",
+			"team": "mobile"
+		}],
+		"tenant_id": {
+			"account_id": 1,
+			"project_id": 5
+		}
+	}
+}`,
+		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		true,
+	)
+
+	// invalid vm_access: extra_filters type mismatch
+	f(`
+{
+	"vm_access": {
+		"extra_filters": [{}],
+		"tenant_id": {
+			"account_id": 1,
+			"project_id": 5
+		}
+	}
+}`,
+		`cannot parse jwt body vm_access: json: cannot unmarshal object into Go value of type string`,
+		true,
+	)
+
+	// invalid exp claim value type
+	f(
+		`{"exp": "1610976189", "vm_access": {}}`,
+		`cannot parse jwt body: json: cannot unmarshal string into Go struct field tbody.exp of type int64`,
+		true,
+	)
+}
+
+func TestParseJWTBody_Success(t *testing.T) {
+	f := func(data string, resultExpected *body) {
+		t.Helper()
+
+		encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
+		encoded := make([]byte, encodedLen)
+		base64.RawURLEncoding.Encode(encoded, []byte(data))
+
+		result, err := parseJWTBody(string(encoded))
+		if err != nil {
+			t.Fatalf("parseJWTBody() error: %s", err)
+		}
+		if result.Exp != resultExpected.Exp {
+			t.Fatalf("unexpected Exp; got %d; want %d", result.Exp, resultExpected.Exp)
+		}
+		if result.Iat != resultExpected.Iat {
+			t.Fatalf("unexpected Iat; got %d; want %d", result.Iat, resultExpected.Iat)
+		}
+		if result.Scope != resultExpected.Scope {
+			t.Fatalf("unexpected scope; got %q; want %q", result.Scope, resultExpected.Scope)
+		}
+		if result.Jti != resultExpected.Jti {
+			t.Fatalf("unexpected jti; got %q; want %q", result.Jti, resultExpected.Jti)
+		}
+		if !reflect.DeepEqual(result.VMAccess.Tenant, resultExpected.VMAccess.Tenant) {
+			t.Fatalf("unexpected tenant; got %v; want %v", result.VMAccess.Tenant, resultExpected.VMAccess.Tenant)
+		}
+		if !reflect.DeepEqual(result.VMAccess.Labels, resultExpected.VMAccess.Labels) {
+			t.Fatalf("unexpected labels; got %v; want %v", result.VMAccess.Labels, resultExpected.VMAccess.Labels)
+		}
+		if !reflect.DeepEqual(result.VMAccess.ExtraFilters, resultExpected.VMAccess.ExtraFilters) {
+			t.Fatalf("unexpected extra_filters; got %v; want %v", result.VMAccess.ExtraFilters, resultExpected.VMAccess.ExtraFilters)
+		}
+	}
+
+	f(`{"vm_access": {}}`, &body{
+		VMAccess: &access{},
+	})
+	f(`{"vm_access": {"tenant_id": {}}}`, &body{
+		VMAccess: &access{},
+	})
+
+	f(
+		`
+{
+    "vm_access": {
+        "tenant_id": {
+            "project_id": 5,
+            "account_id": 1
+        }
+    }
+}`,
+		&body{
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+			},
+		},
+	)
+
+	f(
+		`
+{
+    "vm_access": {
+        "extra_labels": {
+            "project": "dev",
+            "team": "mobile"
+        }
+    }
+}`,
+		&body{
+			VMAccess: &access{
+				Labels: Labels{
+					"project": "dev",
+					"team":    "mobile",
+				},
+			},
+		},
+	)
+
+	f(
+		`
+{
+    "vm_access": {
+        "extra_filters": [
+             "{project=\"dev\"}",
+             "{team=~\"mobile\"}"
+         ]
+    }
+}`,
+		&body{
+			VMAccess: &access{
+				ExtraFilters: []string{
+					`{project="dev"}`,
+					`{team=~"mobile"}`,
+				},
+			},
+		},
+	)
+
+	f(
+		`
+{
+    "vm_access": {
+        "tenant_id": {
+            "project_id": 5,
+            "account_id": 1
+        },
+        "extra_labels": {
+            "project": "dev",
+            "team": "mobile"
+        },
+        "extra_filters": [
+             "{project=\"dev\"}",
+             "{team=~\"mobile\"}"
+         ]
+    }
+}`,
+		&body{
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+				Labels: Labels{
+					"project": "dev",
+					"team":    "mobile",
+				},
+				ExtraFilters: []string{
+					`{project="dev"}`,
+					`{team=~"mobile"}`,
+				},
+			},
+		},
+	)
+
+	f(
+		`
+{
+    "exp": 1610976189,
+    "iat": 1610975889,
+    "jti": "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+    "scope": "openid email profile",
+    "vm_access": {}
+}`,
+		&body{
+			Exp:      1610976189,
+			Iat:      1610975889,
+			Jti:      "9b194187-6bb7-4244-9d1b-559eab2ef7f3",
+			Scope:    "openid email profile",
+			VMAccess: &access{},
+		},
+	)
+}
+
+func TestNewTokenFromRequest_Failure(t *testing.T) {
+	f := func(r *http.Request) {
+		t.Helper()
+
+		_, err := NewTokenFromRequestWithCustomHeader(r, "Authorization", false)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	// missing header
+	f(&http.Request{})
+
+	// bad input
+	f(&http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer fsfFSF",
+			},
+		},
+	})
+
+	// bad input malformed
+	r := &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhQVpvQ0d2dUdiRm9mdFdIeFFaeVJTUWVuM3lYNFUwR1BsUDVvWk9RU3djIn0.eyJleHAiOjE2MTA4ODkyNjYsImlhdCI6MTYxMDg4ODk2NiwiYXV0aF90aW1lIjoxNjEwODg4MDQ0LCJqdGkiOiIwOWEwNThhMi0wNzUyLTRlY2QtYTRlOS1iNjVlODVhZjQyM2YiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIzZGRjODc0OS1lZTI2LTQ2ODEtOWNlYy03M2U5YmIyZmRkOGUiLCJhY3IiOiIwIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bS1hY2Nlc3MiOnsibGFiZWxzIjp7InByb2plY3QiOiJkZXYiLCJ0ZWFtIjoibW9iaWxlIn0sInRlbmFudElEIjp7ImFjY291bnRJRCI6MSwicHJvamVjdElEIjo1fX0sImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6InRnIHRnIiwicHJvamVjdCI6Im1vYmlsZSIsInByZWZlcnJlZF91c2VybmFtZSI6InRnIiwidGVhbSI6ImRldiIsImdpdmVuX25hbWUiOiJ0ZyIsImZhbWlseV9uYW1lIjoidGciLCJlbWFpbCI6InRnQGZnaHQubmV0In0",
+			},
+		},
+	}
+	f(r)
+}
+
+func TestNewTokenFromRequest_Success(t *testing.T) {
+	f := func(r *http.Request, resultExpected *Token, enforcePrefix bool) {
+		t.Helper()
+
+		result, err := NewTokenFromRequestWithCustomHeader(r, "Authorization", enforcePrefix)
+		if err != nil {
+			t.Fatalf("NewTokenFromRequest() error: %s", err)
+		}
+		if !reflect.DeepEqual(result.body.VMAccess, resultExpected.body.VMAccess) {
+			t.Fatalf("unexpected token body VMAccess;\ngot\n%v\nwant\n%v", result.body.VMAccess, resultExpected.body.VMAccess)
+		}
+		if !reflect.DeepEqual(result.header, resultExpected.header) {
+			t.Fatalf("unexpected token header\ngot\n%v\nwant\n%v", result.header, resultExpected.header)
+		}
+	}
+
+	// parse ok
+	r := &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhQVpvQ0d2dUdiRm9mdFdIeFFaeVJTUWVuM3lYNFUwR1BsUDVvWk9RU3djIn0.eyJleHAiOjE2MTA5NzYxODksImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsiZXh0cmFfbGFiZWxzIjp7InByb2plY3QiOiJkZXYiLCJ0ZWFtIjoibW9iaWxlIn0sInRlbmFudF9pZCI6eyJhY2NvdW50X2lkIjoxLCJwcm9qZWN0X2lkIjo1fX0sImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6InRnIHRnIiwicHJvamVjdCI6Im1vYmlsZSIsInByZWZlcnJlZF91c2VybmFtZSI6InRnIiwidGVhbSI6ImRldiIsImdpdmVuX25hbWUiOiJ0ZyIsImZhbWlseV9uYW1lIjoidGciLCJlbWFpbCI6InRnQGZnaHQubmV0In0.XErPkz-qL-EV8BBAR17MoFytc5ajYRz71f9_GOuG1AVcMnUsD6D3x4z5jL1dLyoGGm8OUW_RIVrjMpZf_xOfgQKRVHAMaJi64UtpwS8EF50mlOCDAdKl6wlzAS4laV3dW9W9QrTH7TMetG33WVsJGaD-MIwSYJ5peh6u__oniezsRavw8Qw3nLpZCQPb-NatT3Q1raj1ymLJErJPtUBSk3ieWCVpTMo4ZYKFIQt2wjHeOVOF_3suhPfhgEgXlN6aUq3xeYJ1aAtl_5Ao3pB2pto46kDSXIulQQuGdttsw7bSDOYqZ-tx3y7DBWNdIcghsO_iMvrA805j5hG4Nu84Sw",
+			},
+		},
+	}
+	resultExpected := &Token{
+		body: &body{
+			Exp:   1610889266,
+			Iat:   1610888966,
+			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
+			Scope: "openid email profile",
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+				Labels: map[string]string{
+					"project": "dev",
+					"team":    "mobile",
+				},
+			},
+		},
+		header: &header{
+			Alg: "RS256",
+			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, true)
+
+	// parse ok with non-standard "BEARER" prefix
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"BEARER eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhQVpvQ0d2dUdiRm9mdFdIeFFaeVJTUWVuM3lYNFUwR1BsUDVvWk9RU3djIn0.eyJleHAiOjE2MTA5NzYxODksImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsiZXh0cmFfbGFiZWxzIjp7InByb2plY3QiOiJkZXYiLCJ0ZWFtIjoibW9iaWxlIn0sInRlbmFudF9pZCI6eyJhY2NvdW50X2lkIjoxLCJwcm9qZWN0X2lkIjo1fX0sImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6InRnIHRnIiwicHJvamVjdCI6Im1vYmlsZSIsInByZWZlcnJlZF91c2VybmFtZSI6InRnIiwidGVhbSI6ImRldiIsImdpdmVuX25hbWUiOiJ0ZyIsImZhbWlseV9uYW1lIjoidGciLCJlbWFpbCI6InRnQGZnaHQubmV0In0.XErPkz-qL-EV8BBAR17MoFytc5ajYRz71f9_GOuG1AVcMnUsD6D3x4z5jL1dLyoGGm8OUW_RIVrjMpZf_xOfgQKRVHAMaJi64UtpwS8EF50mlOCDAdKl6wlzAS4laV3dW9W9QrTH7TMetG33WVsJGaD-MIwSYJ5peh6u__oniezsRavw8Qw3nLpZCQPb-NatT3Q1raj1ymLJErJPtUBSk3ieWCVpTMo4ZYKFIQt2wjHeOVOF_3suhPfhgEgXlN6aUq3xeYJ1aAtl_5Ao3pB2pto46kDSXIulQQuGdttsw7bSDOYqZ-tx3y7DBWNdIcghsO_iMvrA805j5hG4Nu84Sw",
+			},
+		},
+	}
+	resultExpected = &Token{
+		body: &body{
+			Exp:   1610889266,
+			Iat:   1610888966,
+			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
+			Scope: "openid email profile",
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+				Labels: map[string]string{
+					"project": "dev",
+					"team":    "mobile",
+				},
+			},
+		},
+		header: &header{
+			Alg: "RS256",
+			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, true)
+
+	// go-jwt
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2NDU1MzY3NTgsImlhdCI6MTY0NTUzNjYzOCwidm1fYWNjZXNzIjp7ImV4dHJhX2ZpbHRlcnMiOlsie25hbWVzcGFjZT1-XCJlaWZ0ZGkxLXRlc3RcIn0iXSwibW9kZSI6MSwidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjB9fX0.4r3zE487ochfj_GgYRpbjmid5ktlBH0bKfz3Ut45Foc",
+			},
+		},
+	}
+	resultExpected = &Token{
+		body: &body{
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 0,
+					AccountID: 1,
+				},
+				ExtraFilters: []string{
+					`{namespace=~"eiftdi1-test"}`,
+				},
+				Mode: 1,
+			},
+		},
+		header: &header{
+			Alg: "HS256",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, true)
+
+	// jwt-with-std-b64
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2NDU2MDY5OTgsImlhdCI6MTY0NTYwNjg3OCwidm1fYWNjZXNzIjp7ImV4dHJhX2ZpbHRlcnMiOlsie25hbWVzcGFjZT1+XCJlaWZ0ZGkxLXRlc3RcIn0iXSwibW9kZSI6MSwidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjB9fX0.oAYJdff8DK4+P1oR6tBE1l2mq79p3eJ5crXlkO+CxcA",
+			},
+		},
+	}
+	resultExpected = &Token{
+		body: &body{
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 0,
+					AccountID: 1,
+				},
+				ExtraFilters: []string{
+					`{namespace=~"eiftdi1-test"}`,
+				},
+				Mode: 1,
+			},
+		},
+		header: &header{
+			Alg: "HS256",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, true)
+
+	// parse ok with filters
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxODksImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsiZXh0cmFfbGFiZWxzIjp7InByb2plY3QiOiJkZXYiLCJ0ZWFtIjoibW9iaWxlIn0sImV4dHJhX2ZpbHRlcnMiOlsie2Vudj1+XCJwcm9kfGRldlwifSIsInt0ZWFtIT1cInRlc3RcIn0iXSwidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9fSwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoidGcgdGciLCJwcm9qZWN0IjoibW9iaWxlIiwicHJlZmVycmVkX3VzZXJuYW1lIjoidGciLCJ0ZWFtIjoiZGV2IiwiZ2l2ZW5fbmFtZSI6InRnIiwiZmFtaWx5X25hbWUiOiJ0ZyJ9.Nx9An-sqto8ClmNah8Mi6y16mjB6jk-I1kxQdtP0j0c",
+			},
+		},
+	}
+	resultExpected = &Token{
+		body: &body{
+			Exp:   1610889266,
+			Iat:   1610888966,
+			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
+			Scope: "openid email profile",
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+				Labels: map[string]string{
+					"project": "dev",
+					"team":    "mobile",
+				},
+				ExtraFilters: []string{`{env=~"prod|dev"}`, `{team!="test"}`},
+			},
+		},
+		header: &header{
+			Alg: "HS256",
+			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, true)
+
+	// parse ok without prefix
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxODksImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsiZXh0cmFfbGFiZWxzIjp7InByb2plY3QiOiJkZXYiLCJ0ZWFtIjoibW9iaWxlIn0sImV4dHJhX2ZpbHRlcnMiOlsie2Vudj1+XCJwcm9kfGRldlwifSIsInt0ZWFtIT1cInRlc3RcIn0iXSwidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9fSwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoidGcgdGciLCJwcm9qZWN0IjoibW9iaWxlIiwicHJlZmVycmVkX3VzZXJuYW1lIjoidGciLCJ0ZWFtIjoiZGV2IiwiZ2l2ZW5fbmFtZSI6InRnIiwiZmFtaWx5X25hbWUiOiJ0ZyJ9.Nx9An-sqto8ClmNah8Mi6y16mjB6jk-I1kxQdtP0j0c",
+			},
+		},
+	}
+	resultExpected = &Token{
+		body: &body{
+			Exp:   1610889266,
+			Iat:   1610888966,
+			Jti:   "09a058a2-0752-4ecd-a4e9-b65e85af423f",
+			Scope: "openid email profile",
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+				Labels: map[string]string{
+					"project": "dev",
+					"team":    "mobile",
+				},
+				ExtraFilters: []string{`{env=~"prod|dev"}`, `{team!="test"}`},
+			},
+		},
+		header: &header{
+			Alg: "HS256",
+			Kid: "aAZoCGvuGbFoftWHxQZyRSQen3yX4U0GPlP5oZOQSwc",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, false)
+
+	// parse ok with string vm_access
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyJ9.eyJhdWQiOiI3YTczMTFlNy1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLCJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzL3YyLjAiLCJpYXQiOjE3MjU2MjUzMzIsIm5iZiI6MTcyNTYyNTMzMiwiZXhwIjoxNzI1NjI5MjMyLCJuYW1lIjoiWmFraGFyIEJlc3NhcmFiIiwib2lkIjoiOGI5ZWY2YjMtMWMwMS00YjczLTg0ODItMjRkNmI2NTE1Y2U0IiwicHJlZmVycmVkX3VzZXJuYW1lIjoiei5iZXNzYXJhYkB2aWN0b3JpYW1ldHJpY3MuY29tIiwicmgiOiIwLkFXTUJ6aDdhSlpGbWFFaW5leHNQbTc1ZlEtY1JjM3AtdWw1Sm1WNnZVMDV6d1JCakFaby4iLCJzdWIiOiJXRld3QTlYZjZpZXUxLUgwNDBuU0QxRVo3UWxOLTVHbWxob2p4czdMUFJRIiwidGlkIjoiMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzIiwidXRpIjoidlo1MjQySmhNVWFUUktaYVFCRjhBQSIsInZlciI6IjIuMCIsInZtX2FjY2VzcyI6IntcInRlbmFudF9pZFwiOntcInByb2plY3RfaWRcIjogNSwgXCJhY2NvdW50X2lkXCI6IDF9fSJ9.E0pEjbazG1QP5nT7fk3GZ9QjIchxOegBQGWnRN8-xFVSJ61v9-FZ-0fNHCYuMVpWvCLqlAHscITB1EYOt4ezvVdwNhO-TXTFAXGznXD4WRsK_G5KGk1QuV-kYwhvidZsPGQe39KlAJm5BPx1fnoHr4yakD647aspd4p9SAsM_H0l4agVZeAhfBqKHI0-cnLcbGb7mC-pZUB1fJBvwc9OT2gzjmA-2T2Vmv4C33I70oDt-wTYmMyHQ4uItTVkj6JXo6gc4V1APJvtA6fB8iq75J-NZ51MiptVIoocX3fYHuC-FwHpi9AFH-1o06tHN0N_A4Hjf8cyzsG8GBaLLGQblw",
+			},
+		},
+	}
+
+	resultExpected = &Token{
+		body: &body{
+			Exp: 1725629232,
+			Iat: 1725625332,
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+			},
+		},
+		header: &header{
+			Alg: "RS256",
+			Kid: "H9nj5AOSswMphg1SFx7jaV-lB9w",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, false)
+
+	// parse ok with scope being slice of strings
+	r = &http.Request{
+		Header: map[string][]string{
+			"Authorization": {
+				"Bearer ewogICJ0eXAiOiJKV1QiLAogICJhbGciOiJSUzI1NiIsCiAgImtpZCI6Ikg5bmo1QU9Tc3dNcGhnMVNGeDdqYVYtbEI5dyIKfQ.ewogICJhdWQiOiI3YTczMTFlNy1iYTdlLTQ5NWUtOTk1ZS1hZjUzNGU3M2MxMTAiLAogICJpc3MiOiJodHRwczovL2xvZ2luLm1pY3Jvc29mdG9ubGluZS5jb20vMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzL3YyLjAiLAogICJpYXQiOjE3MjU2MjUzMzIsCiAgIm5iZiI6MTcyNTYyNTMzMiwKICAiZXhwIjoxNzI1NjI5MjMyLAogICJuYW1lIjoiWmFraGFyIEJlc3NhcmFiIiwKICAib2lkIjoiOGI5ZWY2YjMtMWMwMS00YjczLTg0ODItMjRkNmI2NTE1Y2U0IiwKICAicHJlZmVycmVkX3VzZXJuYW1lIjoiei5iZXNzYXJhYkB2aWN0b3JpYW1ldHJpY3MuY29tIiwKICAicmgiOiIwLkFXTUJ6aDdhSlpGbWFFaW5leHNQbTc1ZlEtY1JjM3AtdWw1Sm1WNnZVMDV6d1JCakFaby4iLAogICJzdWIiOiJXRld3QTlYZjZpZXUxLUgwNDBuU0QxRVo3UWxOLTVHbWxob2p4czdMUFJRIiwKICAidGlkIjoiMjVkYTFlY2UtNjY5MS00ODY4LWE3N2ItMWIwZjliYmU1ZjQzIiwKICAidXRpIjoidlo1MjQySmhNVWFUUktaYVFCRjhBQSIsCiAgInZlciI6IjIuMCIsCiAgInZtX2FjY2VzcyI6IntcInRlbmFudF9pZFwiOntcInByb2plY3RfaWRcIjogNSwgXCJhY2NvdW50X2lkXCI6IDF9fSIsCiAgInNjb3BlIjogWyJvcGVuaWQiLCAidm0iXQp9.ZXdvZ0lDSjBlWEFpT2lKS1YxUWlMQW9nSUNKaGJHY2lPaUpTVXpJMU5pSXNDaUFnSW10cFpDSTZJa2c1Ym1vMVFVOVRjM2ROY0dobk1WTkdlRGRxWVZZdGJFSTVkeUlLZlEuLktrUG9qNWJoaDNWcnRyY3RVb0lHaE5vN2hNc2VGT3hESGVEQ2g3MFViV2l2LU5pb1Zia2duZk1CMkhacHN6WGU5WmNmX2FIaURJSVNTYkNTaDlvQnF1aS02OEJDcmplNFJWRkpGZFV6R3V1SmdOTS11YVpBcFJqSFNNZDUxb2RvbHFoUGFHS09URnJXVmlIWlpfVDdXaVNUcV84U3Y1a2x1Y2xMb0hEcU82MU5Na2w0TmRCVnQxM1hjRTBfM243U3VxTDdpaks2dGMwZ2NzcmJ5c3JNdl9jd2VRamZsLU5fV0N0SG40NnhadEhvX0RpZERabzc2TjV1NE52Uk1OZUxNcXZ0YTgzUzhPdzNyUUlhaUFjUUNHYjBqUU5hV2VEQlFzZUZ6SjRyR0h6RjAwZDlqVkNCSHVWRmI5eHNnSnJVUDZ0S05iT2hTeEY1RzBocElVYk5OUQ",
+			},
+		},
+	}
+
+	resultExpected = &Token{
+		body: &body{
+			Exp: 1725629232,
+			Iat: 1725625332,
+			VMAccess: &access{
+				Tenant: TenantID{
+					ProjectID: 5,
+					AccountID: 1,
+				},
+			},
+		},
+		header: &header{
+			Alg: "RS256",
+			Kid: "H9nj5AOSswMphg1SFx7jaV-lB9w",
+			Typ: "JWT",
+		},
+	}
+	f(r, resultExpected, false)
+}

lib/jwt/key.go

@@ -0,0 +1,23 @@
+package jwt
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+)
+
+// ParseKey parses key in PEM format.
+// It returns a *rsa.PublicKey, *dsa.PublicKey, *ecdsa.PublicKey, or ed25519.PublicKey.
+func ParseKey(key []byte) (any, error) {
+	b, _ := pem.Decode(key)
+	if b == nil {
+		return nil, fmt.Errorf("failed to parse key %q: failed to decode PEM block containing public key", key)
+	}
+
+	k, err := x509.ParsePKIXPublicKey(b.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse key %q: %v", key, err)
+	}
+
+	return k, nil
+}

lib/jwt/key_test.go

@@ -0,0 +1,31 @@
+package jwt
+
+import "testing"
+
+func TestParseKey_Failure(t *testing.T) {
+	f := func(key string) {
+		t.Helper()
+
+		_, err := ParseKey([]byte(key))
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	}
+
+	key := "invalid_key-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(key)
+}
+
+func TestParseKey_Success(t *testing.T) {
+	f := func(key string) {
+		t.Helper()
+
+		_, err := ParseKey([]byte(key))
+		if err != nil {
+			t.Fatalf("ParseKey() error: %s", err)
+		}
+	}
+
+	key := "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(key)
+}

lib/jwt/verifier_pool.go

@@ -0,0 +1,135 @@
+package jwt
+
+import (
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"fmt"
+)
+
+var (
+	// ErrSignatureVerificationFailed token signature verification failed
+	ErrSignatureVerificationFailed = fmt.Errorf("failed to verify token signature")
+	// ErrSignatureAlgorithmNotSupported signature algorithm not supported
+	ErrSignatureAlgorithmNotSupported = fmt.Errorf("signature algorithm verification not supported, supported algorithms: RS256, RS384, RS512, PS256, PS384, PS512, ES256, ES384, ES512")
+)
+
+// VerifierPool is a pool of verifiers for different algorithms
+type VerifierPool struct {
+	rsaVerifiers map[string][]Verifier
+	psVerifiers  map[string][]Verifier
+	esVerifiers  map[string][]Verifier
+}
+
+// NewVerifierPool creates a new verifier pool for a set of keys
+func NewVerifierPool(keys []any) (*VerifierPool, error) {
+	rsaVerifiers := make(map[string][]Verifier)
+	psVerifiers := make(map[string][]Verifier)
+	esVerifiers := make(map[string][]Verifier)
+
+	rsaAlgs := []string{"RS256", "RS384", "RS512"}
+	psAlgs := []string{"PS256", "PS384", "PS512"}
+
+	for _, key := range keys {
+		switch k := key.(type) {
+		case *rsa.PublicKey:
+			for _, alg := range rsaAlgs {
+				verifier, err := newVerifierRS(Algorithm(alg), k)
+				if err != nil {
+					return nil, fmt.Errorf("failed to create RSA verifier for algorithm %s: %w", alg, err)
+				}
+				rsaVerifiers[alg] = append(rsaVerifiers[alg], verifier)
+			}
+
+			for _, alg := range psAlgs {
+				verifier, err := newVerifierPS(Algorithm(alg), k)
+				if err != nil {
+					return nil, fmt.Errorf("failed to create RSA-PSS verifier for algorithm %s: %w", alg, err)
+				}
+				psVerifiers[alg] = append(psVerifiers[alg], verifier)
+			}
+
+		case *ecdsa.PublicKey:
+			alg := getAlgorithmForKey(k)
+			if alg == "" {
+				return nil, fmt.Errorf("failed to create ECDSA verifier: unsupported key")
+			}
+
+			verifier, err := newVerifierES(alg, k)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create ES verifier for algorithm %s: %w", alg, err)
+			}
+			esVerifiers[string(alg)] = append(esVerifiers[string(alg)], verifier)
+
+		default:
+			return nil, fmt.Errorf("unknown key type: %T", key)
+		}
+	}
+
+	return &VerifierPool{
+		rsaVerifiers: rsaVerifiers,
+		psVerifiers:  psVerifiers,
+		esVerifiers:  esVerifiers,
+	}, nil
+}
+
+func (vp *VerifierPool) getVerifiers(alg string) []Verifier {
+	if len(alg) < 2 {
+		return nil
+	}
+
+	switch alg[:2] {
+	case "RS":
+		return vp.getRSVerifiers(alg)
+	case "PS":
+		return vp.getPSVerifiers(alg)
+	case "ES":
+		return vp.getESVerifiers(alg)
+	default:
+		return nil
+	}
+}
+
+// Verify verifies a token signature by using keys provided to verifier pool
+func (vp *VerifierPool) Verify(token *Token) error {
+	verifiers := vp.getVerifiers(token.header.Alg)
+	if verifiers == nil {
+		return ErrSignatureAlgorithmNotSupported
+	}
+
+	for _, verifier := range verifiers {
+		err := verifier.Verify(token)
+		if err == nil {
+			// Token verified, returning success immediately
+			return nil
+		}
+	}
+
+	return ErrSignatureVerificationFailed
+}
+
+func (vp *VerifierPool) getRSVerifiers(alg string) []Verifier {
+	v, ok := vp.rsaVerifiers[alg]
+	if ok {
+		return v
+	}
+
+	return nil
+}
+
+func (vp *VerifierPool) getPSVerifiers(alg string) []Verifier {
+	v, ok := vp.psVerifiers[alg]
+	if ok {
+		return v
+	}
+
+	return nil
+}
+
+func (vp *VerifierPool) getESVerifiers(alg string) []Verifier {
+	v, ok := vp.esVerifiers[alg]
+	if ok {
+		return v
+	}
+
+	return nil
+}

lib/jwt/verifier_pool_test.go

@@ -0,0 +1,148 @@
+package jwt
+
+import (
+	"testing"
+)
+
+func TestTokenVerify_Failure(t *testing.T) {
+	f := func(auth, key string) {
+		t.Helper()
+
+		token, err := NewToken(auth, true)
+		if err != nil {
+			t.Fatalf("unexpected error parsing token: %s", err)
+		}
+
+		k, err := ParseKey([]byte(key))
+		if err != nil {
+			return
+		}
+		vp, err := NewVerifierPool([]any{k})
+		if err != nil {
+			return
+		}
+		if err := vp.Verify(token); err != nil {
+			return
+		}
+
+		t.Fatalf("expecting some error")
+	}
+
+	// RS256 - fail
+	auth := "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.oa8LNeYBQz7HLvUEBvId2hDEbTZrHUBp1cnjqvJee45zAZ4Yf_UDMnpLIllJ5wsyyveDoIIJN1a6XbzZFbxCycBbmBycVwpow1k4y8JRvV8otjZTLSWeTChUFSGUnYWIuVrLEAX6DIA_N4QgTQj2rZ9Zjj9HgWhTvkCiuhFqg2Mvw9EjH8b2hEqwc50XubPvm34hkfy2ETK9sLvYyOwlMRwpAevikTVQ5TODgqhWp1NlagSHP7Dyw_Fx2zfrWWbS56k8fO_pcWvGxCaffYfclDLxNBfxypdl_CzZpXu3XoIlzSY7s2XZHlv6aqGom2ppyDGsdmnac5QCNzQzmhMvdA_fake"
+	key := "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// RS384 - fail
+	auth = "Bearer eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.fZPteCKEsuGKsrKQQZvXU7pAykIYCPW23TTXDZtellqc5JwtXwFNy7T6uwYXklqUmcAJouoFqePjET2MNUHug6PdA_4WdP2v287LrOS26Eto0ddWixdR-w69CbKC0VsMRpCFqvZOYBWUuBgZp6IiDyIocaJ8gfH3Cigsj9simd_aKGIPzd5bu_TNzZeLfisw_HYxP-v0qEIeU84HwfszTHzJszTFg_32FdvqyUxZwFXPoLMMpHZlkScDrgpl4ja9krLlmj8kWPMA0kJ0O_5FSWtly7wvHe_L31lHqGRAqeykGXDEsuMccgE-1sCfqiDu6EqeorlXBOXIkUiuIPan9w_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// RS512 - fail
+	auth = "Bearer eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.DXQYSz2jx63JROB7Fs9ZD2Sp7b95kzY-vj0VOZczyOSm_j4By42bwkWydxkyfc2jgE4Qmj2Mxq-mNm0BEIChP2Klh4TrmziXQZr1ZzD1KhzoZRqANg4Ngck0_Wm-qynFE8gzPiKhPlgH26PsdCn98-L8oqvqjL40AI92ZgGTDQAQavE0tfFjOamz_LHYXwAd7sJsjaJTCuLbAfzH4F-zKx1WH9fQLEp2cB1kZz6Zaa85xzDk0ndLOtLuVF9HbrMTsvx2xT-a-6sPQ-nwuBcIky4Q2A4_KxOzmfPBOqePiwVJa0Y-icq9xvAwb54WvEk8E_QECwKgrje4c2GjWG2CKA_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// ES256 - fail
+	auth = "Bearer eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.llgN8-tHQfakAfsTnPBFqmaPRYC5axXEclk_Ajzp6e_61ASwxOwZvgKDvV4po0HKYLqI3sy89M4ZV9_0SEGuKQ_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEVs/o5+uQbTjL3chynL4wXgUg2R9\nq9UU8I5mEovUf86QZ7kOBIjJwqnzD1omageEHWwHdBO6B+dFabmdT9POxg==\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// ES384 - fail
+	auth = "Bearer eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.qbA54n3OJ7TZJ4XFq2AwdRFDLe2aZwfR04j0NvE1TbVGNipMCana5NY-yacgNRJKK3RjR2OsoWQGAHg9_R6VhTm3X7VNmwcR9u7L-BCAm1cFBU35X5a-PCuke9mbaY_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEC1uWSXj2czCDwMTLWV5BFmwxdM6PX9p+\nPk9Yf9rIf374m5XP1U8q79dBhLSIuaojsvOT39UUcPJROSD1FqYLued0rXiooIii\n1D3jaW6pmGVJFhodzC31cy5sfOYotrzF\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// ES512 - fail
+	auth = "Bearer eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.AVVk7ksNokv6xP0Q1JH6Hbj2AnH3z_o-r56fFASSUFw1Rc1YIYHH_UuMwJi9YkEACKZc0lM8VPfp7h3DXaPGuGJSAKfA7Bezou7Kf1LO5zX0uc2mK-GTyxTiZQysU1JRrQ1bX_Ul7ujtI6FxQjhZw7tPAAssJPUnsB2WpavZgXnsPX_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBgc4HZz+/fBbC7lmEww0AO3NK9wVZ\nPDZ0VEnsaUFLEYpTzb90nITtJUcPUbvOsdZIZ1Q8fnbquAYgxXL5UgHMoywAib47\n6MkyyYgPk0BXZq3mq4zImTRNuaU9slj9TVJ3ScT3L1bXwVuPJDzpr5GOFpaj+WwM\nAl8G7CqwoJOsW7Kddns=\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// PS256 - fail
+	auth = "Bearer eyJhbGciOiJQUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.NXHoXwOduNa97qqsLsboHSuZFLT3WZ35A0opO9F_VZUCeLlViVxTwIgTmscaL8qjIqJvFqBn6Yb7PFaaZTfKHvd8q5aOUkN4or2aPajJv7kGADvH6emYmId6wqJU48oh0yhMQZBEB5trWyboUXM6yAsvdU3kFfyHgw4cNDmCY7oYv0LB2T3wiJ6Xi5q8ZjXt_vrilYTDzgwfI0t2Vu01fkXGuezXIyTzLia2saOFRbs6BedyrHXxGJyOvcXCPX2bqh5AQEegp9Q32_p7k6I1X67Dzd2J-2yUVsxQpRWS5ruOVZr4oE5yZBzRpLi4hr5Hwi9rHcSY7AANu-1h3eweqg_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// PS384 - fail
+	auth = "Bearer eyJhbGciOiJQUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.Wj61agr500y4fB0lrd7AJuW8DXoXF09bMZ1yCiqZl8VKbh9me2RHzEAt0orbPV_rSDdqsgvRn4V2asaodyJjf9A6E8cfNFBRJNMqi5-lpdMP7fs2YF5IVGYzGAkalGVB4QPFGf6xDpNAGk-AsAGGu1Tqpo3jawy2r9IAzoVUs9q8PlNpjKhNK06Ugvy9sxeiEQmYO5gwOrT768mU8_au4InnzXkQdQW_BdYnEZ48udtBDe1r_QQ-dLbHUTW8WnApo6xlONuQErCB7HlpTeDLTcEyAcl_8HmOGupQpMUJCJW1p8D7PCjNSDQFAjr-OOHG7eqH0VGFcglRu95nPdJtHg_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// PS512 - fail
+	auth = "Bearer eyJhbGciOiJQUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.JoPjCp97dGl4GGEvvaoVt1-SjQr9Qw29nGAl7NGXm9CCBt56dDTaoW_e42sF9XirSnzCztLT4841MbH-MOr7cWiEV-XHCLGNEpeSSZRcNAgnEmqTZSOJiMOHeqDjNfu5jE90qVvGB5lfpIsE3DRo8V5L4f0V4hMq47v6Jk1w1oV8snl-qbvjAfSXfKAfQNPkFwKowdwIpiZP9VDwxPaFDOg9Iczz1BeaB6kOZdvFnlr5YsnXPu_NZruAxZs4OnsE8XS1pnW3pJJVseLDMJt-Ao9zTTOsXI9P6FyKGgkquiyXQQkvBk4EcthUEu4mxkynYal53EIw3bHfsBb6sYUdKQ_fake"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// HS256 - not supported
+	auth = "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImFBWm9DR3Z1R2JGb2Z0V0h4UVp5UlNRZW4zeVg0VTBHUGxQNW9aT1FTd2MifQ.eyJleHAiOjE2MTA5NzYxODksImlhdCI6MTYxMDk3NTg4OSwiYXV0aF90aW1lIjoxNjEwOTc1ODg5LCJqdGkiOiI5YjE5NDE4Ny02YmI3LTQyNDQtOWQxYi01NTllYWIyZWY3ZjMiLCJpc3MiOiJodHRwczovL2xvY2FsaG9zdDo4NDQzL2F1dGgvcmVhbG1zL3Rlc3QiLCJhdWQiOiJhY2NvdW50Iiwic3ViIjoiNDYwODU5NDEtYjkyYi00NzFhLWIwNWEtOTU5OWNhMjlkYTFlIiwidHlwIjoiQmVhcmVyIiwiYXpwIjoiZ3JhZmFuYSIsInNlc3Npb25fc3RhdGUiOiIxMzc3ZDEwMi03NTJiLTQ0ODYtOTlkYS1jMjA4MjRiODJkMzEiLCJhY3IiOiIxIiwiYWxsb3dlZC1vcmlnaW5zIjpbImh0dHA6Ly9sb2NhbGhvc3Q6MzAwMCJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwicmVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFuYWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXctcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbGUiLCJ2bV9hY2Nlc3MiOnsiZXh0cmFfbGFiZWxzIjp7InByb2plY3QiOiJkZXYiLCJ0ZWFtIjoibW9iaWxlIn0sImV4dHJhX2ZpbHRlcnMiOlsie2Vudj1+XCJwcm9kfGRldlwifSIsInt0ZWFtIT1cInRlc3RcIn0iXSwidGVuYW50X2lkIjp7ImFjY291bnRfaWQiOjEsInByb2plY3RfaWQiOjV9fSwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoidGcgdGciLCJwcm9qZWN0IjoibW9iaWxlIiwicHJlZmVycmVkX3VzZXJuYW1lIjoidGciLCJ0ZWFtIjoiZGV2IiwiZ2l2ZW5fbmFtZSI6InRnIiwiZmFtaWx5X25hbWUiOiJ0ZyJ9.Nx9An-sqto8ClmNah8Mi6y16mjB6jk-I1kxQdtP0j0c"
+	key = "key"
+	f(auth, key)
+}
+
+func TestTokenVerify_Success(t *testing.T) {
+	f := func(auth, key string) {
+		t.Helper()
+
+		token, err := NewToken(auth, true)
+		if err != nil {
+			t.Fatalf("cannot parse token: %s", err)
+		}
+
+		k, err := ParseKey([]byte(key))
+		if err != nil {
+			t.Fatalf("cannot parse key: %s", err)
+		}
+		vp, err := NewVerifierPool([]any{k})
+		if err != nil {
+			t.Fatalf("NewVerifierPool() error: %s", err)
+		}
+		if err := vp.Verify(token); err != nil {
+			t.Fatalf("Verify() error: %s", err)
+		}
+	}
+
+	// RS256
+	auth := "Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.oa8LNeYBQz7HLvUEBvId2hDEbTZrHUBp1cnjqvJee45zAZ4Yf_UDMnpLIllJ5wsyyveDoIIJN1a6XbzZFbxCycBbmBycVwpow1k4y8JRvV8otjZTLSWeTChUFSGUnYWIuVrLEAX6DIA_N4QgTQj2rZ9Zjj9HgWhTvkCiuhFqg2Mvw9EjH8b2hEqwc50XubPvm34hkfy2ETK9sLvYyOwlMRwpAevikTVQ5TODgqhWp1NlagSHP7Dyw_Fx2zfrWWbS56k8fO_pcWvGxCaffYfclDLxNBfxypdl_CzZpXu3XoIlzSY7s2XZHlv6aqGom2ppyDGsdmnac5QCNzQzmhMvdA"
+	key := "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// RS384
+	auth = "Bearer eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.fZPteCKEsuGKsrKQQZvXU7pAykIYCPW23TTXDZtellqc5JwtXwFNy7T6uwYXklqUmcAJouoFqePjET2MNUHug6PdA_4WdP2v287LrOS26Eto0ddWixdR-w69CbKC0VsMRpCFqvZOYBWUuBgZp6IiDyIocaJ8gfH3Cigsj9simd_aKGIPzd5bu_TNzZeLfisw_HYxP-v0qEIeU84HwfszTHzJszTFg_32FdvqyUxZwFXPoLMMpHZlkScDrgpl4ja9krLlmj8kWPMA0kJ0O_5FSWtly7wvHe_L31lHqGRAqeykGXDEsuMccgE-1sCfqiDu6EqeorlXBOXIkUiuIPan9w"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// RS512
+	auth = "Bearer eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.DXQYSz2jx63JROB7Fs9ZD2Sp7b95kzY-vj0VOZczyOSm_j4By42bwkWydxkyfc2jgE4Qmj2Mxq-mNm0BEIChP2Klh4TrmziXQZr1ZzD1KhzoZRqANg4Ngck0_Wm-qynFE8gzPiKhPlgH26PsdCn98-L8oqvqjL40AI92ZgGTDQAQavE0tfFjOamz_LHYXwAd7sJsjaJTCuLbAfzH4F-zKx1WH9fQLEp2cB1kZz6Zaa85xzDk0ndLOtLuVF9HbrMTsvx2xT-a-6sPQ-nwuBcIky4Q2A4_KxOzmfPBOqePiwVJa0Y-icq9xvAwb54WvEk8E_QECwKgrje4c2GjWG2CKA"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// ES256
+	auth = "Bearer eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.llgN8-tHQfakAfsTnPBFqmaPRYC5axXEclk_Ajzp6e_61ASwxOwZvgKDvV4po0HKYLqI3sy89M4ZV9_0SEGuKQ"
+	key = "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEVs/o5+uQbTjL3chynL4wXgUg2R9\nq9UU8I5mEovUf86QZ7kOBIjJwqnzD1omageEHWwHdBO6B+dFabmdT9POxg==\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// ES384
+	auth = "Bearer eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.qbA54n3OJ7TZJ4XFq2AwdRFDLe2aZwfR04j0NvE1TbVGNipMCana5NY-yacgNRJKK3RjR2OsoWQGAHg9_R6VhTm3X7VNmwcR9u7L-BCAm1cFBU35X5a-PCuke9mbaY7c"
+	key = "-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEC1uWSXj2czCDwMTLWV5BFmwxdM6PX9p+\nPk9Yf9rIf374m5XP1U8q79dBhLSIuaojsvOT39UUcPJROSD1FqYLued0rXiooIii\n1D3jaW6pmGVJFhodzC31cy5sfOYotrzF\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// ES512
+	auth = "Bearer eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.AVVk7ksNokv6xP0Q1JH6Hbj2AnH3z_o-r56fFASSUFw1Rc1YIYHH_UuMwJi9YkEACKZc0lM8VPfp7h3DXaPGuGJSAKfA7Bezou7Kf1LO5zX0uc2mK-GTyxTiZQysU1JRrQ1bX_Ul7ujtI6FxQjhZw7tPAAssJPUnsB2WpavZgXnsPXrW"
+	key = "-----BEGIN PUBLIC KEY-----\nMIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBgc4HZz+/fBbC7lmEww0AO3NK9wVZ\nPDZ0VEnsaUFLEYpTzb90nITtJUcPUbvOsdZIZ1Q8fnbquAYgxXL5UgHMoywAib47\n6MkyyYgPk0BXZq3mq4zImTRNuaU9slj9TVJ3ScT3L1bXwVuPJDzpr5GOFpaj+WwM\nAl8G7CqwoJOsW7Kddns=\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// PS256
+	auth = "Bearer eyJhbGciOiJQUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.NXHoXwOduNa97qqsLsboHSuZFLT3WZ35A0opO9F_VZUCeLlViVxTwIgTmscaL8qjIqJvFqBn6Yb7PFaaZTfKHvd8q5aOUkN4or2aPajJv7kGADvH6emYmId6wqJU48oh0yhMQZBEB5trWyboUXM6yAsvdU3kFfyHgw4cNDmCY7oYv0LB2T3wiJ6Xi5q8ZjXt_vrilYTDzgwfI0t2Vu01fkXGuezXIyTzLia2saOFRbs6BedyrHXxGJyOvcXCPX2bqh5AQEegp9Q32_p7k6I1X67Dzd2J-2yUVsxQpRWS5ruOVZr4oE5yZBzRpLi4hr5Hwi9rHcSY7AANu-1h3eweqg"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// PS384
+	auth = "Bearer eyJhbGciOiJQUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.Wj61agr500y4fB0lrd7AJuW8DXoXF09bMZ1yCiqZl8VKbh9me2RHzEAt0orbPV_rSDdqsgvRn4V2asaodyJjf9A6E8cfNFBRJNMqi5-lpdMP7fs2YF5IVGYzGAkalGVB4QPFGf6xDpNAGk-AsAGGu1Tqpo3jawy2r9IAzoVUs9q8PlNpjKhNK06Ugvy9sxeiEQmYO5gwOrT768mU8_au4InnzXkQdQW_BdYnEZ48udtBDe1r_QQ-dLbHUTW8WnApo6xlONuQErCB7HlpTeDLTcEyAcl_8HmOGupQpMUJCJW1p8D7PCjNSDQFAjr-OOHG7eqH0VGFcglRu95nPdJtHg"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+
+	// PS512
+	auth = "Bearer eyJhbGciOiJQUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMiwidm1fYWNjZXNzIjp7fX0.JoPjCp97dGl4GGEvvaoVt1-SjQr9Qw29nGAl7NGXm9CCBt56dDTaoW_e42sF9XirSnzCztLT4841MbH-MOr7cWiEV-XHCLGNEpeSSZRcNAgnEmqTZSOJiMOHeqDjNfu5jE90qVvGB5lfpIsE3DRo8V5L4f0V4hMq47v6Jk1w1oV8snl-qbvjAfSXfKAfQNPkFwKowdwIpiZP9VDwxPaFDOg9Iczz1BeaB6kOZdvFnlr5YsnXPu_NZruAxZs4OnsE8XS1pnW3pJJVseLDMJt-Ao9zTTOsXI9P6FyKGgkquiyXQQkvBk4EcthUEu4mxkynYal53EIw3bHfsBb6sYUdKQ"
+	key = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo\n4lgOEePzNm0tRgeLezV6ffAt0gunVTLw7onLRnrq0/IzW7yWR7QkrmBL7jTKEn5u\n+qKhbwKfBstIs+bMY2Zkp18gnTxKLxoS2tFczGkPLPgizskuemMghRniWaoLcyeh\nkd3qqGElvW/VDL5AaWTg0nLVkjRo9z+40RQzuVaE8AkAFmxZzow3x+VJYKdjykkJ\n0iT9wCS0DRTXu269V264Vf/3jvredZiKRkgwlL9xNAwxXFg0x/XFw005UWVRIkdg\ncKWTjpBP2dPwVZ4WWC+9aGVd+Gyn1o0CLelf4rEjGoXbAAEgAqeGUxrcIlbjXfbc\nmwIDAQAB\n-----END PUBLIC KEY-----"
+	f(auth, key)
+}

lib/promauth/config.go

@@ -234,7 +234,9 @@ func urlValuesFromMap(m map[string]string) url.Values {
 
 func (oi *oauth2ConfigInternal) initTokenSource() error {
 	tr := httputil.NewTransport(false, "vm_oauth_client")
-	tr.Proxy = oi.proxyURLFunc
+	if oi.proxyURLFunc != nil {
+		tr.Proxy = oi.proxyURLFunc
+	}
 	c := &http.Client{
 		Transport: oi.ac.NewRoundTripper(tr),
 	}

lib/promscrape/client.go

@@ -81,7 +81,9 @@ func newClient(ctx context.Context, sw *ScrapeWork) (*client, error) {
 	}
 
 	tr := httputil.NewTransport(false, "vm_promscrape")
-	tr.Proxy = proxyURLFunc
+	if proxyURLFunc != nil {
+		tr.Proxy = proxyURLFunc
+	}
 	tr.TLSHandshakeTimeout = 10 * time.Second
 	tr.IdleConnTimeout = 2 * sw.ScrapeInterval
 	tr.DisableKeepAlives = *disableKeepAlive || sw.DisableKeepAlive

lib/promscrape/discovery/kubernetes/api_watcher.go

@@ -22,7 +22,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
@@ -269,11 +268,9 @@ func getHTTPClient(ac *promauth.Config, proxyURL *url.URL) *http.Client {
 	if !*useHTTP2Client {
 		// Proxy is not supported for http2 client.
 		// See https://github.com/golang/go/issues/26479
-		var proxy func(*http.Request) (*url.URL, error)
 		if proxyURL != nil {
-			proxy = http.ProxyURL(proxyURL)
+			tr.Proxy = http.ProxyURL(proxyURL)
 		}
-		tr.Proxy = proxy
 	}
 	c := &http.Client{
 		Transport: ac.NewRoundTripper(tr),
@@ -286,7 +283,6 @@ func getHTTPClient(ac *promauth.Config, proxyURL *url.URL) *http.Client {
 
 func newHTTPTransport(enableHTTP2 bool) *http.Transport {
 	tr := httputil.NewTransport(enableHTTP2, "vm_promscrape_discovery_kubernetes")
-	tr.DialContext = netutil.Dialer.DialContext
 	tr.TLSHandshakeTimeout = 10 * time.Second
 	tr.IdleConnTimeout = *apiServerTimeout
 	tr.MaxIdleConnsPerHost = 100

lib/promscrape/discoveryutil/client.go

@@ -112,7 +112,9 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 	}
 
 	tr := httputil.NewTransport(false, "vm_promscrape_discovery")
-	tr.Proxy = proxyURLFunc
+	if proxyURLFunc != nil {
+		tr.Proxy = proxyURLFunc
+	}
 	tr.TLSHandshakeTimeout = 10 * time.Second
 	tr.MaxIdleConnsPerHost = *maxConcurrency
 	tr.ResponseHeaderTimeout = DefaultClientReadTimeout
@@ -124,7 +126,9 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 	}
 
 	trBlocking := httputil.NewTransport(false, "vm_promscrape_discovery")
-	trBlocking.Proxy = proxyURLFunc
+	if proxyURLFunc != nil {
+		trBlocking.Proxy = proxyURLFunc
+	}
 	trBlocking.TLSHandshakeTimeout = 10 * time.Second
 	trBlocking.MaxIdleConnsPerHost = 1000
 	trBlocking.ResponseHeaderTimeout = BlockingClientReadTimeout

lib/protoparser/protoparserutil/compress_reader.go

@@ -74,7 +74,14 @@ func ReadUncompressedData(r io.Reader, contentType string, maxDataSize *flagutil
 func readUncompressedData(r io.Reader, maxDataSize *flagutil.Bytes, decompress func(dst, src []byte) ([]byte, error), callback func(data []byte) error) error {
 	return readFull(r, maxDataSize, func(data []byte) error {
 		dbb := decompressedBufPool.Get()
-		defer decompressedBufPool.Put(dbb)
+		defer func() {
+			if cap(dbb.B) > 1024*1024 && cap(dbb.B) > 4*len(dbb.B) {
+				// Do not store too big dbb to the pool if only a small part of the buffer is used last time.
+				// This should reduce memory waste.
+				return
+			}
+			decompressedBufPool.Put(dbb)
+		}()
 
 		var err error
 		dbb.B, err = decompress(dbb.B, data)
@@ -95,7 +102,7 @@ func readFull(r io.Reader, maxDataSize *flagutil.Bytes, callback func(data []byt
 
 	bb := fullReaderBufPool.Get()
 	defer func() {
-		if len(bb.B) > 1024*1024 && cap(bb.B) > 4*len(bb.B) {
+		if cap(bb.B) > 1024*1024 && cap(bb.B) > 4*len(bb.B) {
 			// Do not store too big bb to the pool if only a small part of the buffer is used last time.
 			// This should reduce memory waste.
 			return
@@ -234,14 +241,21 @@ func (sr *snappyReader) Reset(r io.Reader) error {
 	// Read the whole data in one go, since it is expected that Snappy data
 	// is compressed in block mode instead of stream mode.
 	// See https://pkg.go.dev/github.com/golang/snappy
+
+	lr := ioutil.GetLimitedReader(r, maxSnappyBlockSize+1)
+	defer ioutil.PutLimitedReader(lr)
+
 	cbb := compressedBufPool.Get()
-	_, err := cbb.ReadFrom(r)
+	defer compressedBufPool.Put(cbb)
+
+	_, err := cbb.ReadFrom(lr)
 	if err != nil {
-		compressedBufPool.Put(cbb)
 		return fmt.Errorf("cannot read snappy-encoded data block: %w", err)
 	}
+	if len(cbb.B) > maxSnappyBlockSize {
+		return fmt.Errorf("cannot read snappy-encoded data block because its' size exceeds %d bytes", maxSnappyBlockSize)
+	}
 	sr.b, err = snappy.Decode(sr.b, cbb.B, maxSnappyBlockSize)
-	compressedBufPool.Put(cbb)
 	sr.offset = 0
 	if err != nil {
 		return fmt.Errorf("cannot decode snappy-encoded data block: %w", err)

lib/storage/index_db.go

@@ -14,6 +14,7 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/VictoriaMetrics/fastcache"
 	"github.com/VictoriaMetrics/metricsql"
 	"github.com/cespare/xxhash/v2"
 
@@ -28,6 +29,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 )
 
 const (
@@ -124,7 +126,7 @@ type indexDB struct {
 
 	// Cache for (date, tagFilter) -> loopsCount, which is used for reducing
 	// the amount of work when matching a set of filters.
-	loopsPerDateTagFilterCache *lrucache.Cache
+	loopsPerDateTagFilterCache *workingsetcache.Cache
 
 	// A cache that stores metricIDs that have been added to the index.
 	// The cache is not populated on startup nor does it store a complete set of
@@ -162,10 +164,6 @@ func getTagFiltersCacheSize() uint64 {
 	return maxTagFiltersCacheSize
 }
 
-func getTagFiltersLoopsCacheSize() uint64 {
-	return uint64(float64(memory.Allowed()) / 128)
-}
-
 var maxMetricIDsForDirectLabelsLookup int = 100e3
 
 func mustOpenIndexDB(id uint64, tr TimeRange, name, path string, s *Storage, isReadOnly *atomic.Bool, noRegisterNewSeries bool) *indexDB {
@@ -183,7 +181,7 @@ func mustOpenIndexDB(id uint64, tr TimeRange, name, path string, s *Storage, isR
 		tb:                             tb,
 		s:                              s,
 		tagFiltersToMetricIDsCache:     tfssCache,
-		loopsPerDateTagFilterCache:     lrucache.NewCache(getTagFiltersLoopsCacheSize),
+		loopsPerDateTagFilterCache:     workingsetcache.New(memory.Allowed() / 128),
 		metricIDCache:                  newMetricIDCache(),
 		dateMetricIDCache:              newDateMetricIDCache(),
 	}
@@ -203,6 +201,12 @@ type IndexDBMetrics struct {
 	TagFiltersToMetricIDsCacheMisses       uint64
 	TagFiltersToMetricIDsCacheResets       uint64
 
+	LoopsPerDateTagFilterCacheSize         uint64
+	LoopsPerDateTagFilterCacheSizeBytes    uint64
+	LoopsPerDateTagFilterCacheSizeMaxBytes uint64
+	LoopsPerDateTagFilterCacheRequests     uint64
+	LoopsPerDateTagFilterCacheMisses       uint64
+
 	MetricIDCacheSize           uint64
 	MetricIDCacheSizeBytes      uint64
 	MetricIDCacheSyncsCount     uint64
@@ -262,6 +266,18 @@ func (db *indexDB) UpdateMetrics(m *IndexDBMetrics) {
 	m.TagFiltersToMetricIDsCacheMisses += db.tagFiltersToMetricIDsCache.Misses()
 	m.TagFiltersToMetricIDsCacheResets += db.tagFiltersToMetricIDsCache.Resets()
 
+	// Report only once and for either the first met indexDB instance or whose
+	// loopsPerDateTagFilterCache is utilized the most.
+	var cs fastcache.Stats
+	db.loopsPerDateTagFilterCache.UpdateStats(&cs)
+	if cs.BytesSize > m.LoopsPerDateTagFilterCacheSizeBytes {
+		m.LoopsPerDateTagFilterCacheSize = cs.EntriesCount
+		m.LoopsPerDateTagFilterCacheSizeBytes = cs.BytesSize
+		m.LoopsPerDateTagFilterCacheSizeMaxBytes = cs.MaxBytesSize
+	}
+	m.LoopsPerDateTagFilterCacheRequests += cs.GetCalls
+	m.LoopsPerDateTagFilterCacheMisses += cs.Misses
+
 	// Report only once and for either the first met indexDB instance or whose
 	// metricIDCache is utilized the most.
 	mcs := db.metricIDCache.Stats()
@@ -300,7 +316,7 @@ func (db *indexDB) MustClose() {
 
 	// Free space occupied by caches owned by db.
 	db.tagFiltersToMetricIDsCache.MustStop()
-	db.loopsPerDateTagFilterCache.MustStop()
+	db.loopsPerDateTagFilterCache.Stop()
 	db.metricIDCache.MustStop()
 	db.dateMetricIDCache.MustStop()
 
@@ -2969,32 +2985,25 @@ func (is *indexSearch) getLoopsCountAndTimestampForDateFilter(date uint64, tf *t
 	is.kb.B = appendDateTagFilterCacheKey(is.kb.B[:0], is.db.name, date, tf)
 	kb := kbPool.Get()
 	defer kbPool.Put(kb)
-	e := is.db.loopsPerDateTagFilterCache.GetEntry(bytesutil.ToUnsafeString(is.kb.B))
-	if e == nil {
+	kb.B = is.db.loopsPerDateTagFilterCache.Get(kb.B[:0], is.kb.B)
+	if len(kb.B) != 3*8 {
 		return 0, 0, 0
 	}
-	v := e.(*tagFiltersLoops)
-	return v.loopsCount, v.filterLoopsCount, v.timestamp
-}
-
-type tagFiltersLoops struct {
-	loopsCount       int64
-	filterLoopsCount int64
-	timestamp        uint64
-}
-
-func (v *tagFiltersLoops) SizeBytes() uint64 {
-	return uint64(unsafe.Sizeof(*v))
+	loopsCount := encoding.UnmarshalInt64(kb.B)
+	filterLoopsCount := encoding.UnmarshalInt64(kb.B[8:])
+	timestamp := encoding.UnmarshalUint64(kb.B[16:])
+	return loopsCount, filterLoopsCount, timestamp
 }
 
 func (is *indexSearch) storeLoopsCountForDateFilter(date uint64, tf *tagFilter, loopsCount, filterLoopsCount int64) {
-	v := tagFiltersLoops{
-		loopsCount:       loopsCount,
-		filterLoopsCount: filterLoopsCount,
-		timestamp:        fasttime.UnixTimestamp(),
-	}
+	currentTimestamp := fasttime.UnixTimestamp()
 	is.kb.B = appendDateTagFilterCacheKey(is.kb.B[:0], is.db.name, date, tf)
-	is.db.loopsPerDateTagFilterCache.PutEntry(string(is.kb.B), &v)
+	kb := kbPool.Get()
+	kb.B = encoding.MarshalInt64(kb.B[:0], loopsCount)
+	kb.B = encoding.MarshalInt64(kb.B, filterLoopsCount)
+	kb.B = encoding.MarshalUint64(kb.B, currentTimestamp)
+	is.db.loopsPerDateTagFilterCache.Set(is.kb.B, kb.B)
+	kbPool.Put(kb)
 }
 
 func appendDateTagFilterCacheKey(dst []byte, indexDBName string, date uint64, tf *tagFilter) []byte {

lib/storage/search_synctest_test.go

@@ -1,4 +1,4 @@
-//go:build goexperiment.synctest
+//go:build synctest
 
 package storage
 
@@ -13,7 +13,7 @@ import (
 func TestSearch_metricNamesIndifferentIndexDBs(t *testing.T) {
 	defer testRemoveAll(t)
 
-	synctest.Run(func() {
+	synctest.Test(t, func(t *testing.T) {
 		const numSeries = 10
 		tr := TimeRange{
 			MinTimestamp: time.Now().UnixMilli(),

lib/storage/storage_synctest_test.go

@@ -1,4 +1,4 @@
-//go:build goexperiment.synctest
+//go:build synctest
 
 package storage
 
@@ -289,7 +289,7 @@ func TestStorageRotateIndexDBPrefill(t *testing.T) {
 	f := func(t *testing.T, opts OpenOptions, prefillStart time.Duration) {
 		t.Helper()
 
-		synctest.Run(func() {
+		synctest.Test(t, func(t *testing.T) {
 			// Prefill of the next partition indexDB happens during the
 			// (nextMonth-prefillStart, nextMonth] time interval.
 			// Advance current time right before the the beginning of that interval.

lib/streamaggr/streamaggr_synctest_test.go

@@ -1,4 +1,4 @@
-//go:build goexperiment.synctest
+//go:build synctest
 
 package streamaggr
 
@@ -16,7 +16,7 @@ import (
 func TestAggregatorsSuccess(t *testing.T) {
 	f := func(inputMetrics []string, interval time.Duration, outputMetricsExpected, config, matchIdxsStrExpected string) {
 		t.Helper()
-		synctest.Run(func() {
+		synctest.Test(t, func(t *testing.T) {
 			var matchIdxs []uint32
 			var tssOutput []prompb.TimeSeries
 			var tssOutputLock sync.Mutex

lib/workingsetcache/cache_synctest_test.go

@@ -1,4 +1,4 @@
-//go:build goexperiment.synctest
+//go:build synctest
 
 package workingsetcache