docs/articles: add https://medium.com/airbnb-engineering/building-a-high-volume-metrics-pipeline-with-opentelemetry-and-vmagent-c714d6910b45

Signed-off-by: hagen1778 <roman@victoriametrics.com>

.github/copilot-instructions.md

@@ -1,23 +0,0 @@
-# Project Overview
-
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
-
-## Folder Structure
-
-- `/app`: Contains the compilable binaries.
-- `/lib`: Contains the golang reusable libraries
-- `/docs/victoriametrics`: Contains documentation for the project.
-- `/apptest/tests`: Contains integration tests.
-
-## Libraries and Frameworks
-
-- Backend: Golang, no framework. Use third-party libraries sparingly.
-- Frontend: React.
-
-## Code review guidelines
-
-Ensure the feature or bugfix includes a changelog entry in /docs/victoriametrics/changelog/CHANGELOG.md.
-Verify the entry is under the ## tip section and matches the structure and style of existing entries.
-Chore-only changes may be omitted from the changelog.
-
-

.github/dependabot.yml

@@ -4,6 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    cooldown:
+      default-days: 21
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
@@ -23,6 +25,8 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    cooldown:
+      default-days: 21
   - package-ecosystem: "npm"
     directory: "/app/vmui/packages/vmui"
     schedule:

.github/workflows/check-commit-signed.yml

@@ -27,11 +27,21 @@ jobs:
             exit 0
           fi
           
-          unsigned=$(git log --pretty="%H %G?" $RANGE | grep -vE " (G|E)$" || true)
+          # Check raw commit objects for a "gpgsig" header as a fast early signal for
+          # contributors. Both GPG and SSH signatures use this header. 
+          # This avoids relying on %G? which returns N for SSH commits.
+          # This check is not a security enforcement — unsigned commits cannot be merged
+          # anyway due to the GitHub repository merge policy.
+          unsigned=""
+          for sha in $(git rev-list $RANGE); do
+            if ! git cat-file commit "$sha" | grep -q "^gpgsig"; then
+              unsigned="$unsigned $sha"
+            fi
+          done
           if [ -n "$unsigned" ]; then
             echo "Found unsigned commits:"
             echo "$unsigned"
             exit 1
           fi
-          
-          echo "All commits in PR are signed (G or E)"
+
+          echo "All commits in PR are signed (GPG or SSH)"

.github/workflows/check-licenses.yml

@@ -34,7 +34,7 @@ jobs:
             ~/go/pkg/mod
             ~/go/bin
           key: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-
+          restore-keys: go-artifacts-${{ runner.os }}-check-licenses-${{ steps.go.outputs.go-version }}-
 
       - name: Check License
         run: make check-licenses

.github/workflows/codeql-analysis-go.yml

@@ -47,7 +47,7 @@ jobs:
             ~/go/bin
             ~/go/pkg/mod
           key: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
-          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-
+          restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v4

.github/workflows/docs.yaml

@@ -28,7 +28,7 @@ jobs:
           path: __vm-docs
 
       - name: Import GPG key
-        uses: crazy-max/ghaction-import-gpg@v6
+        uses: crazy-max/ghaction-import-gpg@v7
         id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}

.github/workflows/test.yml

@@ -43,6 +43,7 @@ jobs:
             Makefile
             app/**/Makefile
           go-version-file: 'go.mod'
+
       - run: go version
 
       - name: Cache golangci-lint
@@ -51,7 +52,7 @@ jobs:
           path: |
             ~/.cache/golangci-lint
             ~/go/bin
-          key: golangci-lint-${{ runner.os }}-${{ hashFiles('.golangci.yml') }}
+          key: golangci-lint-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('.golangci.yml') }}
 
       - name: Run check-all
         run: |
@@ -85,16 +86,16 @@ jobs:
       - run: go version
 
       - name: Run tests
-        run: GOGC=10 make ${{ matrix.scenario}}
+        run: make ${{ matrix.scenario}}
 
       - name: Publish coverage
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v6
         with:
           files: ./coverage.txt
 
-  integration:
-    name: integration
-    runs-on: ubuntu-latest
+  apptest:
+    name: apptest
+    runs-on: apptest
 
     steps:
       - name: Code checkout
@@ -111,5 +112,5 @@ jobs:
           go-version-file: 'go.mod'
       - run: go version
 
-      - name: Run integration tests
-        run: make integration-test
+      - name: Run app tests
+        run: make apptest

Makefile

@@ -17,7 +17,7 @@ EXTRA_GO_BUILD_TAGS ?=
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
 TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
-GOLANGCI_LINT_VERSION := 2.7.2
+GOLANGCI_LINT_VERSION := 2.9.0
 
 .PHONY: $(MAKECMDGOALS)
 
@@ -443,7 +443,7 @@ fmt:
 	gofmt -l -w -s ./apptest
 
 vet:
-	GOEXPERIMENT=synctest go vet ./lib/...
+	go vet -tags 'synctest' ./lib/...
 	go vet ./app/...
 	go vet ./apptest/...
 
@@ -452,28 +452,25 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	GOEXPERIMENT=synctest go test ./lib/... ./app/...
+	go test -tags 'synctest' ./lib/... ./app/...
 
 test-race:
-	GOEXPERIMENT=synctest go test -race ./lib/... ./app/...
+	go test -tags 'synctest' -race ./lib/... ./app/...
 
 test-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test -tags 'synctest' ./lib/... ./app/...
 
 test-full:
-	GOEXPERIMENT=synctest go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -tags 'synctest' -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
-
-integration-test:
-	$(MAKE) apptest
+	GOARCH=386 go test -tags 'synctest' -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 apptest:
-	$(MAKE) victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
+	$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
 	go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
 
-integration-test-legacy: victoria-metrics vmbackup vmrestore
+apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
 	OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
 	ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
 	VERSION=v1.132.0; \
@@ -490,17 +487,17 @@ integration-test-legacy: victoria-metrics vmbackup vmrestore
 	go test ./apptest/tests -run="^TestLegacySingle.*"
 
 benchmark:
-	GOEXPERIMENT=synctest go test -bench=. ./lib/...
-	go test -bench=. ./app/...
+	go test -run=NO_TESTS -bench=. ./lib/...
+	go test -run=NO_TESTS -bench=. ./app/...
 
 benchmark-pure:
-	GOEXPERIMENT=synctest CGO_ENABLED=0 go test -bench=. ./lib/...
-	CGO_ENABLED=0 go test -bench=. ./app/...
+	CGO_ENABLED=0 go test -run=NO_TESTS -bench=. ./lib/...
+	CGO_ENABLED=0 go test -run=NO_TESTS -bench=. ./app/...
 
 vendor-update:
 	go get -u ./lib/...
 	go get -u ./app/...
-	go mod tidy -compat=1.24
+	go mod tidy -compat=1.26
 	go mod vendor
 
 app-local:
@@ -524,7 +521,7 @@ install-qtc:
 
 
 golangci-lint: install-golangci-lint
-	GOEXPERIMENT=synctest golangci-lint run
+	golangci-lint run --build-tags 'synctest'
 
 install-golangci-lint:
 	which golangci-lint && (golangci-lint --version | grep -q $(GOLANGCI_LINT_VERSION)) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v$(GOLANGCI_LINT_VERSION)

README.md

@@ -1,12 +1,12 @@
 # VictoriaMetrics
 
 [![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
-![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
+[![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)](https://hub.docker.com/u/victoriametrics)
 [![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
 [![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
 [![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)](https://app.codecov.io/gh/VictoriaMetrics/VictoriaMetrics)
 [![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
-![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)
+[![Join Slack](https://img.shields.io/badge/Join%20Slack-4A154B?logo=slack)](https://slack.victoriametrics.com)
 [![X](https://img.shields.io/twitter/follow/VictoriaMetrics?style=flat&label=Follow&color=black&logo=x&labelColor=black&link=https%3A%2F%2Fx.com%2FVictoriaMetrics)](https://x.com/VictoriaMetrics/)
 [![Reddit](https://img.shields.io/reddit/subreddit-subscribers/VictoriaMetrics?style=flat&label=Join&labelColor=red&logoColor=white&logo=reddit&link=https%3A%2F%2Fwww.reddit.com%2Fr%2FVictoriaMetrics)](https://www.reddit.com/r/VictoriaMetrics/)
 
@@ -16,16 +16,21 @@
   <img src="docs/victoriametrics/logo.webp" width="300" alt="VictoriaMetrics logo">
 </picture>
 
-VictoriaMetrics is a fast, cost-saving, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
+VictoriaMetrics is a fast, cost-effective, and scalable solution for monitoring and managing time series data. It delivers high performance and reliability, making it an ideal choice for businesses of all sizes.
 
 Here are some resources and information about VictoriaMetrics:
 
-- Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
-- Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
-- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
-- Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/)
-- Changelog: [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics)
-- Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)
+- **Case studies**: [Grammarly, Roblox, Wix, Spotify,...](https://docs.victoriametrics.com/victoriametrics/casestudies/).
+- **Available**: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), Docker images on [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
+- **Deployment types**: [Single-node version](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [Cluster version](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) under [Apache License 2.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
+- **Getting started:** Read [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/) and follow the
+  [quick start guide](https://docs.victoriametrics.com/victoriametrics/quick-start/).
+- **Community**: [Slack](https://slack.victoriametrics.com/) (join via [Slack Inviter](https://slack.victoriametrics.com/)), [X (Twitter)](https://x.com/VictoriaMetrics), [YouTube](https://www.youtube.com/@VictoriaMetrics). See full list [here](https://docs.victoriametrics.com/victoriametrics/#community-and-contributions).
+- **Changelog**: Project evolves fast - check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/), and [How to upgrade](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
+- **Enterprise support:** [Contact us](mailto:info@victoriametrics.com) for commercial support with additional [enterprise features](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+- **Enterprise releases:** Enterprise and [long-term support releases (LTS)](https://docs.victoriametrics.com/victoriametrics/lts-releases/) are publicly available and can be evaluated for free
+  using a [free trial license](https://victoriametrics.com/products/enterprise/trial/).
+- **Security:** we achieved [security certifications](https://victoriametrics.com/security/) for Database Software Development and Software-Based Monitoring Services.
 
 Yes, we open-source both the single-node VictoriaMetrics and the cluster version.
 

SECURITY.md

@@ -12,6 +12,31 @@ The following versions of VictoriaMetrics receive regular security fixes:
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
+## Software Bill of Materials (SBOM)
+
+Every VictoriaMetrics container{{% available_from "#" %}} image published to
+[Docker Hub](https://hub.docker.com/u/victoriametrics)
+and [Quay.io](https://quay.io/organization/victoriametrics)
+includes an [SPDX](https://spdx.dev/) SBOM attestation
+generated automatically by BuildKit during
+`docker buildx build`.
+
+To inspect the SBOM for an image:
+
+```sh
+docker buildx imagetools inspect \
+  docker.io/victoriametrics/victoria-metrics:latest \
+  --format "{{ json .SBOM }}"
+```
+
+To scan an image using its SBOM attestation with
+[Trivy](https://github.com/aquasecurity/trivy):
+
+```sh
+trivy image --sbom-sources oci \
+  docker.io/victoriametrics/victoria-metrics:latest
+```
+
 ## Reporting a Vulnerability
 
 Please report any security issues to <security@victoriametrics.com>

app/victoria-metrics/test/parser.go

@@ -33,13 +33,13 @@ func PopulateTimeTpl(b []byte, tGlobal time.Time) []byte {
 		}
 		switch strings.TrimSpace(parts[0]) {
 		case `TIME_S`:
-			return []byte(fmt.Sprintf("%d", t.Unix()))
+			return fmt.Appendf(nil, "%d", t.Unix())
 		case `TIME_MSZ`:
-			return []byte(fmt.Sprintf("%d", t.Unix()*1e3))
+			return fmt.Appendf(nil, "%d", t.Unix()*1e3)
 		case `TIME_MS`:
-			return []byte(fmt.Sprintf("%d", timeToMillis(t)))
+			return fmt.Appendf(nil, "%d", timeToMillis(t))
 		case `TIME_NS`:
-			return []byte(fmt.Sprintf("%d", t.UnixNano()))
+			return fmt.Appendf(nil, "%d", t.UnixNano())
 		default:
 			log.Fatalf("unknown time pattern %s in %s", parts[0], repl)
 		}

app/vmagent/datadogsketches/request_handler.go

@@ -49,6 +49,11 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 				Name:  "__name__",
 				Value: m.Name,
 			})
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
+			labels = append(labels, prompb.Label{
+				Name:  "host",
+				Value: sketch.Host,
+			})
 			for _, label := range m.Labels {
 				labels = append(labels, prompb.Label{
 					Name:  label.Name,
@@ -57,9 +62,6 @@ func insertRows(at *auth.Token, sketches []*datadogsketches.Sketch, extraLabels
 			}
 			for _, tag := range sketch.Tags {
 				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
 				labels = append(labels, prompb.Label{
 					Name:  name,
 					Value: value,

app/vmagent/remotewrite/client.go

@@ -13,6 +13,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/golang/snappy"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -21,10 +24,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/golang/snappy"
 )
 
 var (
@@ -290,7 +290,7 @@ func getAWSAPIConfig(argIdx int) (*awsapi.Config, error) {
 	accessKey := awsAccessKey.GetOptionalArg(argIdx)
 	secretKey := awsSecretKey.GetOptionalArg(argIdx)
 	service := awsService.GetOptionalArg(argIdx)
-	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service)
+	cfg, err := awsapi.NewConfig(ec2Endpoint, stsEndpoint, region, roleARN, accessKey, secretKey, service, "")
 	if err != nil {
 		return nil, err
 	}
@@ -405,8 +405,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // Otherwise, it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxInterval)
-	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
+	bt := timeutil.NewBackoffTimer(c.retryMinInterval, c.retryMaxInterval)
 	retriesCount := 0
 
 again:
@@ -415,19 +414,10 @@ again:
 	c.requestDuration.UpdateDuration(startTime)
 	if err != nil {
 		c.errorsCount.Inc()
-		retryDuration *= 2
-		if retryDuration > maxRetryDuration {
-			retryDuration = maxRetryDuration
-		}
-		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %.3f seconds",
-			len(block), c.sanitizedURL, err, retryDuration.Seconds())
-		t := timerpool.Get(retryDuration)
-		select {
-		case <-c.stopCh:
-			timerpool.Put(t)
+		remoteWriteRetryLogger.Warnf("couldn't send a block with size %d bytes to %q: %s; re-sending the block in %s",
+			len(block), c.sanitizedURL, err, bt.CurrentDelay())
+		if !bt.Wait(c.stopCh) {
 			return false
-		case <-t.C:
-			timerpool.Put(t)
 		}
 		c.retriesCount.Inc()
 		goto again
@@ -493,7 +483,10 @@ again:
 	// Unexpected status code returned
 	retriesCount++
 	retryAfterHeader := parseRetryAfterHeader(resp.Header.Get("Retry-After"))
-	retryDuration = getRetryDuration(retryAfterHeader, retryDuration, maxRetryDuration)
+	// retryAfterDuration has the highest priority duration
+	if retryAfterHeader > 0 {
+		bt.SetDelay(retryAfterHeader)
+	}
 
 	// Handle response
 	body, err := io.ReadAll(resp.Body)
@@ -502,15 +495,10 @@ again:
 		logger.Errorf("cannot read response body from %q during retry #%d: %s", c.sanitizedURL, retriesCount, err)
 	} else {
 		logger.Errorf("unexpected status code received after sending a block with size %d bytes to %q during retry #%d: %d; response body=%q; "+
-			"re-sending the block in %.3f seconds", len(block), c.sanitizedURL, retriesCount, statusCode, body, retryDuration.Seconds())
+			"re-sending the block in %s", len(block), c.sanitizedURL, retriesCount, statusCode, body, bt.CurrentDelay())
 	}
-	t := timerpool.Get(retryDuration)
-	select {
-	case <-c.stopCh:
-		timerpool.Put(t)
+	if !bt.Wait(c.stopCh) {
 		return false
-	case <-t.C:
-		timerpool.Put(t)
 	}
 	c.retriesCount.Inc()
 	goto again
@@ -519,27 +507,6 @@ again:
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 
-// getRetryDuration returns retry duration.
-// retryAfterDuration has the highest priority.
-// If retryAfterDuration is not specified, retryDuration gets doubled.
-// retryDuration can't exceed maxRetryDuration.
-//
-// Also see: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6097
-func getRetryDuration(retryAfterDuration, retryDuration, maxRetryDuration time.Duration) time.Duration {
-	// retryAfterDuration has the highest priority duration
-	if retryAfterDuration > 0 {
-		return timeutil.AddJitterToDuration(retryAfterDuration)
-	}
-
-	// default backoff retry policy
-	retryDuration *= 2
-	if retryDuration > maxRetryDuration {
-		retryDuration = maxRetryDuration
-	}
-
-	return retryDuration
-}
-
 // repackBlockFromZstdToSnappy repacks the given zstd-compressed block to snappy-compressed block.
 //
 // The input block may be corrupted, for example, if vmagent was shut down ungracefully and
@@ -570,24 +537,20 @@ func logBlockRejected(block []byte, sanitizedURL string, resp *http.Response) {
 }
 
 // parseRetryAfterHeader parses `Retry-After` value retrieved from HTTP response header.
-// retryAfterString should be in either HTTP-date or a number of seconds.
-// It will return time.Duration(0) if `retryAfterString` does not follow RFC 7231.
-func parseRetryAfterHeader(retryAfterString string) (retryAfterDuration time.Duration) {
-	if retryAfterString == "" {
-		return retryAfterDuration
+//
+// s should be in either HTTP-date or a number of seconds.
+// It returns time.Duration(0) if s does not follow RFC 7231.
+func parseRetryAfterHeader(s string) time.Duration {
+	if s == "" {
+		return 0
 	}
 
-	defer func() {
-		v := retryAfterDuration.Seconds()
-		logger.Infof("'Retry-After: %s' parsed into %.2f second(s)", retryAfterString, v)
-	}()
-
 	// Retry-After could be in "Mon, 02 Jan 2006 15:04:05 GMT" format.
-	if parsedTime, err := time.Parse(http.TimeFormat, retryAfterString); err == nil {
+	if parsedTime, err := time.Parse(http.TimeFormat, s); err == nil {
 		return time.Duration(time.Until(parsedTime).Seconds()) * time.Second
 	}
 	// Retry-After could be in seconds.
-	if seconds, err := strconv.Atoi(retryAfterString); err == nil {
+	if seconds, err := strconv.Atoi(s); err == nil {
 		return time.Duration(seconds) * time.Second
 	}
 

app/vmagent/remotewrite/client_test.go

@@ -6,66 +6,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 )
 
-func TestCalculateRetryDuration(t *testing.T) {
-	// `testFunc` call `calculateRetryDuration` for `n` times
-	// and evaluate if the result of `calculateRetryDuration` is
-	// 1. >= expectMinDuration
-	// 2. <= expectMinDuration + 10% (see timeutil.AddJitterToDuration)
-	f := func(retryAfterDuration, retryDuration time.Duration, n int, expectMinDuration time.Duration) {
-		t.Helper()
-
-		for i := 0; i < n; i++ {
-			retryDuration = getRetryDuration(retryAfterDuration, retryDuration, time.Minute)
-		}
-
-		expectMaxDuration := helper(expectMinDuration)
-		expectMinDuration = expectMinDuration - (1000 * time.Millisecond) // Avoid edge case when calculating time.Until(now)
-
-		if retryDuration < expectMinDuration || retryDuration > expectMaxDuration {
-			t.Fatalf(
-				"incorrect retry duration, want (ms): [%d, %d], got (ms): %d",
-				expectMinDuration.Milliseconds(), expectMaxDuration.Milliseconds(),
-				retryDuration.Milliseconds(),
-			)
-		}
-	}
-
-	// Call calculateRetryDuration for 1 time.
-	{
-		// default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-		// default backoff policy exceed max limit"
-		f(0, 10*time.Minute, 1, time.Minute)
-
-		// retry after > default backoff policy
-		f(10*time.Second, 1*time.Second, 1, 10*time.Second)
-		// retry after < default backoff policy
-		f(1*time.Second, 10*time.Second, 1, 1*time.Second)
-		// retry after invalid and < default backoff policy
-		f(0, time.Second, 1, 2*time.Second)
-
-	}
-
-	// Call calculateRetryDuration for multiple times.
-	{
-		// default backoff policy 2 times
-		f(0, time.Second, 2, 4*time.Second)
-		// default backoff policy 3 times
-		f(0, time.Second, 3, 8*time.Second)
-		// default backoff policy N times exceed max limit
-		f(0, time.Second, 10, time.Minute)
-
-		// retry after 120s 1 times
-		f(120*time.Second, time.Second, 1, 120*time.Second)
-		// retry after 120s 2 times
-		f(120*time.Second, time.Second, 2, 120*time.Second)
-	}
-}
-
 func TestParseRetryAfterHeader(t *testing.T) {
 	f := func(retryAfterString string, expectResult time.Duration) {
 		t.Helper()
@@ -91,13 +36,6 @@ func TestParseRetryAfterHeader(t *testing.T) {
 	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
 }
 
-// helper calculate the max possible time duration calculated by timeutil.AddJitterToDuration.
-func helper(d time.Duration) time.Duration {
-	dv := min(d/10, 10*time.Second)
-
-	return d + dv
-}
-
 func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 	expectedPlainBlock := []byte(`foobar`)
 

app/vmagent/remotewrite/pendingseries_test.go

@@ -51,9 +51,9 @@ func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expecte
 
 func newTestWriteRequest(seriesCount, labelsCount int) *prompb.WriteRequest {
 	var wr prompb.WriteRequest
-	for i := 0; i < seriesCount; i++ {
+	for i := range seriesCount {
 		var labels []prompb.Label
-		for j := 0; j < labelsCount; j++ {
+		for j := range labelsCount {
 			labels = append(labels, prompb.Label{
 				Name:  fmt.Sprintf("label_%d_%d", i, j),
 				Value: fmt.Sprintf("value_%d_%d", i, j),

app/vmagent/remotewrite/relabel.go

@@ -38,7 +38,7 @@ var (
 	labelsGlobal []prompb.Label
 
 	remoteWriteRelabelConfigData    atomic.Pointer[[]byte]
-	remoteWriteURLRelabelConfigData atomic.Pointer[[]interface{}]
+	remoteWriteURLRelabelConfigData atomic.Pointer[[]any]
 
 	relabelConfigReloads      *metrics.Counter
 	relabelConfigReloadErrors *metrics.Counter
@@ -90,8 +90,8 @@ func WriteURLRelabelConfigData(w io.Writer) {
 		return
 	}
 	type urlRelabelCfg struct {
-		Url           string      `yaml:"url"`
-		RelabelConfig interface{} `yaml:"relabel_config"`
+		Url           string `yaml:"url"`
+		RelabelConfig any    `yaml:"relabel_config"`
 	}
 	var cs []urlRelabelCfg
 	for i, url := range *remoteWriteURLs {
@@ -144,7 +144,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 			len(*relabelConfigPaths), (len(*remoteWriteURLs)))
 	}
 
-	var urlRelabelCfgs []interface{}
+	var urlRelabelCfgs []any
 	rcs.perURL = make([]*promrelabel.ParsedConfigs, len(*remoteWriteURLs))
 	for i, path := range *relabelConfigPaths {
 		if len(path) == 0 {
@@ -157,7 +157,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) {
 		}
 		rcs.perURL[i] = prc
 
-		var parsedCfg interface{}
+		var parsedCfg any
 		_ = yaml.Unmarshal(rawCfg, &parsedCfg)
 		urlRelabelCfgs = append(urlRelabelCfgs, parsedCfg)
 	}

app/vmagent/remotewrite/remotewrite.go

@@ -3,6 +3,7 @@ package remotewrite
 import (
 	"flag"
 	"fmt"
+	"math"
 	"net/http"
 	"net/url"
 	"path/filepath"
@@ -11,6 +12,10 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
+
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -23,6 +28,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prommetadata"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
@@ -30,8 +36,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/cespare/xxhash/v2"
 )
 
 var (
@@ -80,10 +84,14 @@ var (
 		`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
 		`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+
 		`Enabled sorting for labels can slow down ingestion performance a bit`)
-	maxHourlySeries = flag.Int("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
-		"Excess series are logged and dropped. This can be useful for limiting series cardinality. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
-	maxDailySeries = flag.Int("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
-		"Excess series are logged and dropped. This can be useful for limiting series churn rate. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
+	maxHourlySeries = flag.Int64("remoteWrite.maxHourlySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last hour. "+
+		"Excess series are logged and dropped. This can be useful for limiting series cardinality. "+
+		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
+	maxDailySeries = flag.Int64("remoteWrite.maxDailySeries", 0, "The maximum number of unique series vmagent can send to remote storage systems during the last 24 hours. "+
+		"Excess series are logged and dropped. This can be useful for limiting series churn rate. "+
+		fmt.Sprintf("Setting this flag to '-1' sets limit to maximum possible value (%d) which is useful in order to enable series tracking without enforcing limits. ", math.MaxInt32)+
+		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter")
 	maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmagent can receive per second. Data ingestion is paused when the limit is exceeded. "+
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
@@ -92,6 +100,8 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
 	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
 		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
+	disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
+		"By default, metadata sending is controlled by the global -enableMetadata flag")
 )
 
 var (
@@ -157,8 +167,8 @@ func Init() {
 	if len(*remoteWriteURLs) == 0 {
 		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
-	if *maxHourlySeries > 0 {
-		hourlySeriesLimiter = bloomfilter.NewLimiter(*maxHourlySeries, time.Hour)
+	if limit := getMaxHourlySeries(); limit > 0 {
+		hourlySeriesLimiter = bloomfilter.NewLimiter(limit, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
 			return float64(hourlySeriesLimiter.MaxItems())
 		})
@@ -166,8 +176,8 @@ func Init() {
 			return float64(hourlySeriesLimiter.CurrentItems())
 		})
 	}
-	if *maxDailySeries > 0 {
-		dailySeriesLimiter = bloomfilter.NewLimiter(*maxDailySeries, 24*time.Hour)
+	if limit := getMaxDailySeries(); limit > 0 {
+		dailySeriesLimiter = bloomfilter.NewLimiter(limit, 24*time.Hour)
 		_ = metrics.NewGauge(`vmagent_daily_series_limit_max_series`, func() float64 {
 			return float64(dailySeriesLimiter.MaxItems())
 		})
@@ -540,6 +550,10 @@ func tryPushMetadataToRemoteStorages(rwctxs []*remoteWriteCtx, mms []prompb.Metr
 	var wg sync.WaitGroup
 	var anyPushFailed atomic.Bool
 	for _, rwctx := range rwctxs {
+		if !rwctx.enableMetadata {
+			// Skip remote storage with disabled metadata
+			continue
+		}
 		wg.Go(func() {
 			if !rwctx.tryPushMetadataInternal(mms) {
 				rwctx.pushFailures.Inc()
@@ -811,6 +825,11 @@ type remoteWriteCtx struct {
 	streamAggrKeepInput bool
 	streamAggrDropInput bool
 
+	// enableMetadata indicates whether metadata should be sent to this remote storage.
+	// It is determined by -remoteWrite.enableMetadata per-URL flag if set,
+	// otherwise by the global -enableMetadata flag.
+	enableMetadata bool
+
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
 
@@ -822,6 +841,18 @@ type remoteWriteCtx struct {
 	rowsDroppedOnPushFailure     *metrics.Counter
 }
 
+// isMetadataEnabledForURL returns true if metadata should be sent to the remote storage at argIdx.
+// It checks the per-URL -remoteWrite.disableMetadata flag first.
+// If not set, it falls back to the global -enableMetadata flag.
+func isMetadataEnabledForURL(argIdx int) bool {
+	if disableMetadataPerURL.GetOptionalArg(argIdx) {
+		// Metadata is explicitly disabled for this URL
+		return false
+	}
+	// Use global -enableMetadata value
+	return prommetadata.IsEnabled()
+}
+
 func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
@@ -892,10 +923,11 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
 	}
 
 	rwctx := &remoteWriteCtx{
-		idx: argIdx,
-		fq:  fq,
-		c:   c,
-		pss: pss,
+		idx:            argIdx,
+		fq:             fq,
+		c:              c,
+		pss:            pss,
+		enableMetadata: isMetadataEnabledForURL(argIdx),
 
 		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
@@ -1116,3 +1148,21 @@ func newMapFromStrings(a []string) map[string]struct{} {
 	}
 	return m
 }
+
+func getMaxHourlySeries() int {
+	limit := *maxHourlySeries
+	if limit == -1 || limit > math.MaxInt32 {
+		return math.MaxInt32
+	}
+
+	return int(limit)
+}
+
+func getMaxDailySeries() int {
+	limit := *maxDailySeries
+	if limit == -1 || limit > math.MaxInt32 {
+		return math.MaxInt32
+	}
+
+	return int(limit)
+}

app/vmagent/remotewrite/remotewrite_test.go

@@ -28,12 +28,12 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 		itemsCount := 1_000 * bucketsCount
 		m := make([]int, bucketsCount)
 		var labels []prompb.Label
-		for i := 0; i < itemsCount; i++ {
+		for i := range itemsCount {
 			labels = append(labels[:0], prompb.Label{
 				Name:  "__name__",
 				Value: fmt.Sprintf("some_name_%d", i),
 			})
-			for j := 0; j < 10; j++ {
+			for j := range 10 {
 				labels = append(labels, prompb.Label{
 					Name:  fmt.Sprintf("label_%d", j),
 					Value: fmt.Sprintf("value_%d_%d", i, j),
@@ -248,7 +248,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
 		seriesCount := 100000
 		// build 1000000 series
 		tssBlock := make([]prompb.TimeSeries, 0, seriesCount)
-		for i := 0; i < seriesCount; i++ {
+		for i := range seriesCount {
 			tssBlock = append(tssBlock, prompb.TimeSeries{
 				Labels: []prompb.Label{
 					{
@@ -269,7 +269,7 @@ func TestShardAmountRemoteWriteCtx(t *testing.T) {
 		// build active time series set
 		nodes := make([]string, 0, remoteWriteCount)
 		activeTimeSeriesByNodes := make([]map[string]struct{}, remoteWriteCount)
-		for i := 0; i < remoteWriteCount; i++ {
+		for i := range remoteWriteCount {
 			nodes = append(nodes, fmt.Sprintf("node%d", i))
 			activeTimeSeriesByNodes[i] = make(map[string]struct{})
 		}

app/vmalert-tool/unittest/input_test.go

@@ -41,7 +41,7 @@ func TestParseInputValue_Success(t *testing.T) {
 		if len(outputExpected) != len(output) {
 			t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
 		}
-		for i := 0; i < len(outputExpected); i++ {
+		for i := range outputExpected {
 			if outputExpected[i].Omitted != output[i].Omitted {
 				t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}

app/vmalert-tool/unittest/unittest.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"maps"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -12,6 +13,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"reflect"
+	"slices"
 	"sort"
 	"strings"
 	"syscall"
@@ -348,9 +350,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	for k := range alertEvalTimesMap {
 		alertEvalTimes = append(alertEvalTimes, k)
 	}
-	sort.Slice(alertEvalTimes, func(i, j int) bool {
-		return alertEvalTimes[i] < alertEvalTimes[j]
-	})
+	slices.Sort(alertEvalTimes)
 
 	// sort group eval order according to the given "group_eval_order".
 	sort.Slice(testGroups, func(i, j int) bool {
@@ -361,12 +361,8 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	var groups []*rule.Group
 	for _, group := range testGroups {
 		mergedExternalLabels := make(map[string]string)
-		for k, v := range tg.ExternalLabels {
-			mergedExternalLabels[k] = v
-		}
-		for k, v := range externalLabels {
-			mergedExternalLabels[k] = v
-		}
+		maps.Copy(mergedExternalLabels, tg.ExternalLabels)
+		maps.Copy(mergedExternalLabels, externalLabels)
 		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
 		ng.Init()
 		groups = append(groups, ng)

app/vmalert/config/config.go

@@ -81,12 +81,9 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.Interval.Duration() < 0 {
 		return fmt.Errorf("interval shouldn't be lower than 0")
 	}
-	if g.EvalOffset.Duration() < 0 {
-		return fmt.Errorf("eval_offset shouldn't be lower than 0")
-	}
-	// if `eval_offset` is set, interval won't use global evaluationInterval flag and must bigger than offset.
-	if g.EvalOffset.Duration() > g.Interval.Duration() {
-		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
+	// if `eval_offset` is set, the group interval must be specified explicitly(instead of inherited from global evaluationInterval flag) and must bigger than offset.
+	if g.EvalOffset.Duration().Abs() > g.Interval.Duration() {
+		return fmt.Errorf("the abs value of eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
 	}
 	if g.EvalOffset != nil && g.EvalDelay != nil {
 		return fmt.Errorf("eval_offset cannot be used with eval_delay")

app/vmalert/config/config_test.go

@@ -176,11 +176,17 @@ func TestGroupValidate_Failure(t *testing.T) {
 	}, false, "interval shouldn't be lower than 0")
 
 	f(&Group{
-		Name:       "wrong eval_offset",
+		Name:       "too big eval_offset",
 		Interval:   promutil.NewDuration(time.Minute),
 		EvalOffset: promutil.NewDuration(2 * time.Minute),
 	}, false, "eval_offset should be smaller than interval")
 
+	f(&Group{
+		Name:       "too big negative eval_offset",
+		Interval:   promutil.NewDuration(time.Minute),
+		EvalOffset: promutil.NewDuration(-2 * time.Minute),
+	}, false, "eval_offset should be smaller than interval")
+
 	limit := -1
 	f(&Group{
 		Name:  "wrong limit",

app/vmalert/config/types.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"slices"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaLogs/lib/logstorage"
@@ -80,12 +81,8 @@ func (t *Type) ValidateExpr(expr string) error {
 		if err != nil {
 			return fmt.Errorf("cannot obtain labels from LogsQL expr: %q, err: %w", expr, err)
 		}
-		for i := range labels {
-			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
-			// making the result meaningless and may lead to cardinality issues.
-			if labels[i] == "_time" {
-				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
-			}
+		if slices.Contains(labels, "_time") {
+			return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
 		}
 	default:
 		return fmt.Errorf("unknown datasource type=%q", t.Name)

app/vmalert/datasource/client.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"net/http"
 	"net/url"
 	"strings"
@@ -91,9 +92,7 @@ func (c *Client) Clone() *Client {
 		ns.extraHeaders = make([]keyValue, len(c.extraHeaders))
 		copy(ns.extraHeaders, c.extraHeaders)
 	}
-	for k, v := range c.extraParams {
-		ns.extraParams[k] = v
-	}
+	maps.Copy(ns.extraParams, c.extraParams)
 
 	return ns
 }

app/vmalert/datasource/client_prom.go

@@ -34,7 +34,7 @@ type promResponse struct {
 	// Stats supported by VictoriaMetrics since v1.90
 	Stats struct {
 		SeriesFetched *string `json:"seriesFetched,omitempty"`
-	} `json:"stats,omitempty"`
+	} `json:"stats"`
 	// IsPartial supported by VictoriaMetrics
 	IsPartial *bool `json:"isPartial,omitempty"`
 }

app/vmalert/datasource/datasource.go

@@ -134,7 +134,7 @@ func (ls Labels) String() string {
 func LabelCompare(a, b Labels) int {
 	l := min(len(b), len(a))
 
-	for i := 0; i < l; i++ {
+	for i := range l {
 		if a[i].Name != b[i].Name {
 			if a[i].Name < b[i].Name {
 				return -1

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -13,7 +13,7 @@ func BenchmarkPromInstantUnmarshal(b *testing.B) {
 
 	// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10                    1760            668959 ns/op          280147 B/op       5781 allocs/op
 	b.Run("Instant std+fastjson", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
+		for range b.N {
 			var pi promInstant
 			err = pi.Unmarshal(data)
 			if err != nil {

app/vmalert/main.go

@@ -56,7 +56,7 @@ absolute path to all .tpl files in root.
  -rule.templates="dir/**/*.tpl". Includes all the .tpl files in "dir" subfolders recursively.
 `)
 
-	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
+	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule', '-rule.templates' and '-notifier.config' files. "+
 		"By default, the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
 	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "Address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")

app/vmalert/manager.go

@@ -98,7 +98,7 @@ func (m *manager) close() {
 	m.wg.Wait()
 }
 
-func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) error {
+func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) {
 	id := g.GetID()
 	g.Init()
 	m.wg.Go(func() {
@@ -110,7 +110,6 @@ func (m *manager) startGroup(ctx context.Context, g *rule.Group, restore bool) e
 	})
 
 	m.groups[id] = g
-	return nil
 }
 
 func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore bool) error {
@@ -119,7 +118,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 	for _, cfg := range groupsCfg {
 		for _, r := range cfg.Rules {
 			if rrPresent && arPresent {
-				continue
+				break
 			}
 			if r.Record != "" {
 				rrPresent = true
@@ -162,10 +161,7 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 		}
 	}
 	for _, ng := range groupsRegistry {
-		if err := m.startGroup(ctx, ng, restore); err != nil {
-			m.groupsMu.Unlock()
-			return err
-		}
+		m.startGroup(ctx, ng, restore)
 	}
 	m.groupsMu.Unlock()
 

app/vmalert/manager_test.go

@@ -69,7 +69,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	for n := range workers {
 		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
-			for i := 0; i < iterations; i++ {
+			for range iterations {
 				rnd := r.Intn(len(paths))
 				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
@@ -259,7 +259,7 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
 	for i, r := range a.Rules {
 		got, want := r, b.Rules[i]
 		if a.CreateID() != b.CreateID() {
-			t.Fatalf("expected to have rule %q; got %q", want.ID(), got.ID())
+			t.Fatalf("expected to have rule %d; got %d", want.ID(), got.ID())
 		}
 		if err := rule.CompareRules(t, want, got); err != nil {
 			t.Fatalf("comparison error: %s", err)

app/vmalert/notifier/config_watcher_test.go

@@ -216,7 +216,7 @@ consul_sd_configs:
 	for n := range workers {
 		wg.Go(func() {
 			r := rand.New(rand.NewSource(int64(n)))
-			for i := 0; i < iterations; i++ {
+			for range iterations {
 				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()

app/vmalert/remotewrite/client.go

@@ -13,14 +13,18 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/golang/snappy"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -114,7 +118,9 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
 	}
 
 	for i := 0; i < cc; i++ {
-		c.run(ctx)
+		c.wg.Go(func() {
+			c.run(ctx, i)
+		})
 	}
 	return c, nil
 }
@@ -156,8 +162,7 @@ func (c *Client) Close() error {
 	return nil
 }
 
-func (c *Client) run(ctx context.Context) {
-	ticker := time.NewTicker(c.flushInterval)
+func (c *Client) run(ctx context.Context, id int) {
 	wr := &prompb.WriteRequest{}
 	shutdown := func() {
 		lastCtx, cancel := context.WithTimeout(context.Background(), defaultWriteTimeout)
@@ -174,40 +179,72 @@ func (c *Client) run(ctx context.Context) {
 		cancel()
 	}
 
-	c.wg.Go(func() {
-		defer ticker.Stop()
-		for {
+	// add jitter to spread remote write flushes over the flush interval to avoid congestion at the remote write destination
+	h := xxhash.Sum64(bytesutil.ToUnsafeBytes(fmt.Sprintf("%d", id)))
+	randJitter := uint64(float64(c.flushInterval) * (float64(h) / (1 << 64)))
+	timer := time.NewTimer(time.Duration(randJitter))
+addJitter:
+	for {
+		select {
+		case <-c.doneCh:
+			timer.Stop()
+			shutdown()
+			return
+		case <-ctx.Done():
+			timer.Stop()
+			shutdown()
+			return
+		case <-timer.C:
+			break addJitter
+		}
+	}
+
+	ticker := time.NewTicker(c.flushInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-c.doneCh:
+			shutdown()
+			return
+		case <-ctx.Done():
+			shutdown()
+			return
+		case <-ticker.C:
+			c.flush(ctx, wr)
+			// drain the potential stale tick to avoid small or empty flushes after a slow flush.
 			select {
-			case <-c.doneCh:
-				shutdown()
-				return
-			case <-ctx.Done():
-				shutdown()
-				return
 			case <-ticker.C:
+			default:
+			}
+		case ts, ok := <-c.input:
+			if !ok {
+				continue
+			}
+			wr.Timeseries = append(wr.Timeseries, ts)
+			if len(wr.Timeseries) >= c.maxBatchSize {
 				c.flush(ctx, wr)
-			case ts, ok := <-c.input:
-				if !ok {
-					continue
-				}
-				wr.Timeseries = append(wr.Timeseries, ts)
-				if len(wr.Timeseries) >= c.maxBatchSize {
-					c.flush(ctx, wr)
-				}
 			}
 		}
-	})
+	}
 }
 
 var (
 	rwErrors = metrics.NewCounter(`vmalert_remotewrite_errors_total`)
 	rwTotal  = metrics.NewCounter(`vmalert_remotewrite_total`)
 
-	sentRows            = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
-	sentBytes           = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
-	droppedRows         = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
-	sendDuration        = metrics.NewFloatCounter(`vmalert_remotewrite_send_duration_seconds_total`)
-	bufferFlushDuration = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
+	// sentRows and sentBytes are historical counters that can now be replaced by flushedRows and flushedBytes histograms. They may be deprecated in the future after the new histograms have been adopted for some time.
+	sentRows             = metrics.NewCounter(`vmalert_remotewrite_sent_rows_total`)
+	sentBytes            = metrics.NewCounter(`vmalert_remotewrite_sent_bytes_total`)
+	flushedRows          = metrics.NewHistogram(`vmalert_remotewrite_sent_rows`)
+	flushedBytes         = metrics.NewHistogram(`vmalert_remotewrite_sent_bytes`)
+	droppedRows          = metrics.NewCounter(`vmalert_remotewrite_dropped_rows_total`)
+	sendDuration         = metrics.NewFloatCounter(`vmalert_remotewrite_send_duration_seconds_total`)
+	bufferFlushDuration  = metrics.NewHistogram(`vmalert_remotewrite_flush_duration_seconds`)
+	remoteWriteQueueSize = metrics.NewHistogram(`vmalert_remotewrite_queue_size`)
+
+	_ = metrics.NewGauge(`vmalert_remotewrite_queue_capacity`, func() float64 {
+		return float64(*maxQueueSize)
+	})
 
 	_ = metrics.NewGauge(`vmalert_remotewrite_concurrency`, func() float64 {
 		return float64(*concurrency)
@@ -221,6 +258,7 @@ func GetDroppedRows() int { return int(droppedRows.Get()) }
 // it to remote-write endpoint. Flush performs limited amount of retries
 // if request fails.
 func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
+	remoteWriteQueueSize.Update(float64(len(c.input)))
 	if len(wr.Timeseries) < 1 {
 		return
 	}
@@ -230,16 +268,16 @@ func (c *Client) flush(ctx context.Context, wr *prompb.WriteRequest) {
 	data := wr.MarshalProtobuf(nil)
 	b := snappy.Encode(nil, data)
 
-	retryInterval, maxRetryInterval := *retryMinInterval, *retryMaxTime
-	if retryInterval > maxRetryInterval {
-		retryInterval = maxRetryInterval
-	}
+	maxRetryInterval := *retryMaxTime
+	bt := timeutil.NewBackoffTimer(*retryMinInterval, maxRetryInterval)
 	timeStart := time.Now()
 	defer func() {
 		sendDuration.Add(time.Since(timeStart).Seconds())
 	}()
+
+	attempts := 0
 L:
-	for attempts := 0; ; attempts++ {
+	for {
 		err := c.send(ctx, b)
 		if err != nil && (errors.Is(err, io.EOF) || netutil.IsTrivialNetworkError(err)) {
 			// Something in the middle between client and destination might be closing
@@ -249,6 +287,8 @@ L:
 		if err == nil {
 			sentRows.Add(len(wr.Timeseries))
 			sentBytes.Add(len(b))
+			flushedRows.Update(float64(len(wr.Timeseries)))
+			flushedBytes.Update(float64(len(b)))
 			return
 		}
 
@@ -274,13 +314,13 @@ L:
 			break
 		}
 
-		if retryInterval > timeLeftForRetries {
-			retryInterval = timeLeftForRetries
+		if bt.CurrentDelay() > timeLeftForRetries {
+			bt.SetDelay(timeLeftForRetries)
 		}
 		// sleeping to prevent remote db hammering
-		time.Sleep(retryInterval)
-		retryInterval *= 2
+		bt.Wait(ctx.Done())
 
+		attempts++
 	}
 
 	rwErrors.Inc()

app/vmalert/remotewrite/client_test.go

@@ -44,7 +44,7 @@ func TestClient_Push(t *testing.T) {
 
 	r := rand.New(rand.NewSource(1))
 	const rowsN = int(1e4)
-	for i := 0; i < rowsN; i++ {
+	for range rowsN {
 		s := prompb.TimeSeries{
 			Samples: []prompb.Sample{{
 				Value:     r.Float64(),
@@ -102,7 +102,7 @@ func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
 		}
 
 		// push time series to the client.
-		for i := 0; i < pushCnt; i++ {
+		for range pushCnt {
 			if err = rwClient.Push(prompb.TimeSeries{}); err != nil {
 				t.Fatalf("cannot time series to the client: %s", err)
 			}

app/vmalert/remotewrite/debug_client_test.go

@@ -22,7 +22,7 @@ func TestDebugClient_Push(t *testing.T) {
 
 	const rowsN = 100
 	var sent int
-	for i := 0; i < rowsN; i++ {
+	for i := range rowsN {
 		s := prompb.TimeSeries{
 			Samples: []prompb.Sample{{
 				Value:     float64(i),

app/vmalert/rule/alerting_synctest_test.go

@@ -0,0 +1,106 @@
+//go:build synctest
+
+package rule
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
+)
+
+// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
+// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
+// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
+func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
+	// wrap into synctest because of time manipulations
+	synctest.Test(t, func(t *testing.T) {
+		fq := &datasource.FakeQuerier{}
+
+		ar := &AlertingRule{
+			Name: "TestActiveAtPreservation",
+			Labels: map[string]string{
+				"test_query_in_label": `{{ "static_value" }}`,
+			},
+			Annotations: map[string]string{
+				"description": "Alert active since {{ $activeAt }}",
+			},
+			alerts: make(map[uint64]*notifier.Alert),
+			q:      fq,
+			state: &ruleState{
+				entries: make([]StateEntry, 10),
+			},
+		}
+
+		// Mock query result - return empty result to make suppress_for_mass_alert = false
+		// (no need to add anything to fq for empty result)
+
+		// Add a metric that should trigger the alert
+		fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
+
+		// First execution - creates new alert
+		ts1 := time.Now()
+		_, err := ar.exec(context.TODO(), ts1, 0)
+		if err != nil {
+			t.Fatalf("unexpected error on first exec: %s", err)
+		}
+
+		if len(ar.alerts) != 1 {
+			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
+		}
+
+		firstAlert := ar.GetAlerts()[0]
+		// Verify first execution: activeAt should be ts1 and annotation should reflect it
+		if !firstAlert.ActiveAt.Equal(ts1) {
+			t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
+		}
+
+		// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
+		expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
+		if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
+			t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
+		}
+
+		// Second execution - should preserve activeAt in annotation
+
+		// Ensure different timestamp with different seconds
+		// sleep is non-blocking thanks to synctest
+		time.Sleep(2 * time.Second)
+		ts2 := time.Now()
+		_, err = ar.exec(context.TODO(), ts2, 0)
+		if err != nil {
+			t.Fatalf("unexpected error on second exec: %s", err)
+		}
+
+		// Get the alert again (should be the same alert)
+		if len(ar.alerts) != 1 {
+			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
+		}
+		secondAlert := ar.GetAlerts()[0]
+
+		// Critical test: activeAt should still be ts1, not ts2
+		if !secondAlert.ActiveAt.Equal(ts1) {
+			t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
+		}
+
+		// Critical test: annotation should still contain ts1 time, not ts2
+		if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
+			t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
+		}
+
+		// Additional verification: annotation should NOT contain ts2 time
+		ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
+		if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
+			t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
+		}
+
+		// Verify query template in labels still works (this would fail if query templates were broken)
+		if firstAlert.Labels["test_query_in_label"] != "static_value" {
+			t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
+		}
+	})
+}

app/vmalert/rule/alerting_test.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"testing/synctest"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -1479,95 +1478,3 @@ func TestAlertingRule_QueryTemplateInLabels(t *testing.T) {
 		t.Fatalf("expected 'suppress_for_mass_alert' label to be 'true' or 'false', got '%s'", suppressLabel)
 	}
 }
-
-// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
-// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
-// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
-func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
-	// wrap into synctest because of time manipulations
-	synctest.Test(t, func(t *testing.T) {
-		fq := &datasource.FakeQuerier{}
-
-		ar := &AlertingRule{
-			Name: "TestActiveAtPreservation",
-			Labels: map[string]string{
-				"test_query_in_label": `{{ "static_value" }}`,
-			},
-			Annotations: map[string]string{
-				"description": "Alert active since {{ $activeAt }}",
-			},
-			alerts: make(map[uint64]*notifier.Alert),
-			q:      fq,
-			state: &ruleState{
-				entries: make([]StateEntry, 10),
-			},
-		}
-
-		// Mock query result - return empty result to make suppress_for_mass_alert = false
-		// (no need to add anything to fq for empty result)
-
-		// Add a metric that should trigger the alert
-		fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
-
-		// First execution - creates new alert
-		ts1 := time.Now()
-		_, err := ar.exec(context.TODO(), ts1, 0)
-		if err != nil {
-			t.Fatalf("unexpected error on first exec: %s", err)
-		}
-
-		if len(ar.alerts) != 1 {
-			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-		}
-
-		firstAlert := ar.GetAlerts()[0]
-		// Verify first execution: activeAt should be ts1 and annotation should reflect it
-		if !firstAlert.ActiveAt.Equal(ts1) {
-			t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
-		}
-
-		// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
-		expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
-		if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
-			t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
-		}
-
-		// Second execution - should preserve activeAt in annotation
-
-		// Ensure different timestamp with different seconds
-		// sleep is non-blocking thanks to synctest
-		time.Sleep(2 * time.Second)
-		ts2 := time.Now()
-		_, err = ar.exec(context.TODO(), ts2, 0)
-		if err != nil {
-			t.Fatalf("unexpected error on second exec: %s", err)
-		}
-
-		// Get the alert again (should be the same alert)
-		if len(ar.alerts) != 1 {
-			t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
-		}
-		secondAlert := ar.GetAlerts()[0]
-
-		// Critical test: activeAt should still be ts1, not ts2
-		if !secondAlert.ActiveAt.Equal(ts1) {
-			t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
-		}
-
-		// Critical test: annotation should still contain ts1 time, not ts2
-		if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
-			t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
-		}
-
-		// Additional verification: annotation should NOT contain ts2 time
-		ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
-		if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
-			t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
-		}
-
-		// Verify query template in labels still works (this would fail if query templates were broken)
-		if firstAlert.Labels["test_query_in_label"] != "static_value" {
-			t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
-		}
-	})
-}

app/vmalert/rule/group.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"hash/fnv"
+	"maps"
 	"net/url"
 	"sync"
 	"time"
@@ -30,8 +31,8 @@ var (
 		"0 means no limit.")
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
-	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
-	maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maxiMum duration for automatic alert expiration, "+
+	resendDelay        = flag.Duration("rule.resendDelay", 0, "Minimum amount of time to wait before resending an alert to notifier.")
+	maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maximum duration for automatic alert expiration, "+
 		"which by default is 4 times evaluationInterval of the parent group")
 	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the 'time' parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
 		"Normally, should be equal to '-search.latencyOffset' (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
@@ -97,9 +98,7 @@ type groupMetrics struct {
 // set2 has priority over set1.
 func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
 	r := map[string]string{}
-	for k, v := range set1 {
-		r[k] = v
-	}
+	maps.Copy(r, set1)
 	for k, v := range set2 {
 		if prevV, ok := r[k]; ok {
 			logger.Infof("label %q=%q for rule %q.%q overwritten with external label %q=%q",
@@ -382,7 +381,9 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 
 		if len(g.Rules) < 1 {
 			g.metrics.iterationDuration.UpdateDuration(start)
+			g.mu.Lock()
 			g.LastEvaluation = start
+			g.mu.Unlock()
 			return ts
 		}
 
@@ -396,7 +397,9 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 			}
 		}
 		g.metrics.iterationDuration.UpdateDuration(start)
+		g.mu.Lock()
 		g.LastEvaluation = start
+		g.mu.Unlock()
 		return ts
 	}
 
@@ -406,11 +409,11 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
 	g.mu.Unlock()
 	defer g.evalCancel()
 
-	realEvalTS := eval(evalCtx, evalTS)
-
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
 
+	realEvalTS := eval(evalCtx, evalTS)
+
 	// restore the rules state after the first evaluation
 	// so only active alerts can be restored.
 	if rr != nil {
@@ -485,8 +488,15 @@ func (g *Group) UpdateWith(newGroup *Group) {
 // delayBeforeStart calculates delay based on Group ID, so all groups will start at different moments of time.
 func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Duration {
 	if g.EvalOffset != nil {
+		offset := *g.EvalOffset
+		// adjust the offset for negative evalOffset, the rule is:
+		// `eval_offset: -x` is equivalent to `eval_offset: y` for `interval: x+y`.
+		// For example, `eval_offset: -6m` is equivalent to `eval_offset: 4m` for `interval: 10m`.
+		if offset < 0 {
+			offset += g.Interval
+		}
 		// if offset is specified, ignore the maxDelay and return a duration aligned with offset
-		currentOffsetPoint := ts.Truncate(g.Interval).Add(*g.EvalOffset)
+		currentOffsetPoint := ts.Truncate(g.Interval).Add(offset)
 		if currentOffsetPoint.Before(ts) {
 			// wait until the next offset point
 			return currentOffsetPoint.Add(g.Interval).Sub(ts)
@@ -495,11 +505,8 @@ func (g *Group) delayBeforeStart(ts time.Time, maxDelay time.Duration) time.Dura
 	}
 
 	// otherwise, return a random duration between [0..min(interval, maxDelay)] based on group ID
-	interval := g.Interval
-	if interval > maxDelay {
-		// artificially limit interval, so groups with big intervals could start sooner.
-		interval = maxDelay
-	}
+	// artificially limit interval, so groups with big intervals could start sooner.
+	interval := min(g.Interval, maxDelay)
 	var randSleep time.Duration
 	randSleep = time.Duration(float64(interval) * (float64(g.GetID()) / (1 << 64)))
 	sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())

app/vmalert/rule/group_test.go

@@ -405,7 +405,8 @@ func TestGroupStart(t *testing.T) {
 
 		var cur uint64
 		prev := g.metrics.iterationTotal.Get()
-		for i := 0; ; i++ {
+		i := 0
+		for {
 			if i > 40 {
 				t.Fatalf("group wasn't able to perform %d evaluations during %d eval intervals", n, i)
 			}
@@ -414,6 +415,7 @@ func TestGroupStart(t *testing.T) {
 				return
 			}
 			time.Sleep(interval)
+			i++
 		}
 	}
 
@@ -604,6 +606,15 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
 	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
 
+	// test group with negative offset -2min, which is equivalent to 3min offset for 5min interval
+	offset = -2 * time.Minute
+	g.EvalOffset = &offset
+
+	f("2023-01-01T00:00:15.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
+	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
+
 	maxDelay = time.Minute * 1
 	g.EvalOffset = nil
 

app/vmalert/rule/rule.go

@@ -121,7 +121,7 @@ func (s *ruleState) add(e StateEntry) {
 func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
 	var err error
 	var tss []prompb.TimeSeries
-	for i := 0; i < replayRuleRetryAttempts; i++ {
+	for i := range replayRuleRetryAttempts {
 		tss, err = r.execRange(context.Background(), start, end)
 		if err == nil {
 			break

app/vmalert/rule/rule_test.go

@@ -40,7 +40,7 @@ func TestRule_state(t *testing.T) {
 	}
 
 	var last time.Time
-	for i := 0; i < stateEntriesN*2; i++ {
+	for range stateEntriesN * 2 {
 		last = time.Now()
 		r.state.add(StateEntry{At: last})
 	}
@@ -68,7 +68,7 @@ func TestRule_stateConcurrent(_ *testing.T) {
 	var wg sync.WaitGroup
 	for range workers {
 		wg.Go(func() {
-			for i := 0; i < iterations; i++ {
+			for range iterations {
 				r.state.add(StateEntry{At: time.Now()})
 				r.state.getAll()
 				r.state.getLast()

app/vmalert/rule/test_helpers.go

@@ -19,13 +19,13 @@ func CompareRules(t *testing.T, a, b Rule) error {
 	case *AlertingRule:
 		br, ok := b.(*AlertingRule)
 		if !ok {
-			return fmt.Errorf("rule %q supposed to be of type AlertingRule", b.ID())
+			return fmt.Errorf("rule %d supposed to be of type AlertingRule", b.ID())
 		}
 		return compareAlertingRules(t, v, br)
 	case *RecordingRule:
 		br, ok := b.(*RecordingRule)
 		if !ok {
-			return fmt.Errorf("rule %q supposed to be of type RecordingRule", b.ID())
+			return fmt.Errorf("rule %d supposed to be of type RecordingRule", b.ID())
 		}
 		return compareRecordingRules(t, v, br)
 	default:

app/vmalert/rule/web.go

@@ -57,12 +57,8 @@ type ApiGroup struct {
 	EvalOffset float64 `json:"eval_offset,omitempty"`
 	// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
 	EvalDelay float64 `json:"eval_delay,omitempty"`
-	// Unhealthy unhealthy rules count
-	Unhealthy int
-	// Healthy passing rules count
-	Healthy int
-	// NoMatch not matching rules count
-	NoMatch int
+	// States represents counts per each rule state
+	States map[string]int `json:"states"`
 }
 
 // APILink returns a link to the group's JSON representation.
@@ -134,6 +130,11 @@ type ApiRule struct {
 	Updates []StateEntry `json:"-"`
 }
 
+// IsNoMatch returns true if rule is in nomatch state
+func (r *ApiRule) IsNoMatch() bool {
+	return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0
+}
+
 // ApiAlert represents a notifier.AlertingRule state
 // for WEB view
 // https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
@@ -235,6 +236,20 @@ func NewAlertAPI(ar *AlertingRule, a *notifier.Alert) *ApiAlert {
 	return aa
 }
 
+func (r *ApiRule) ExtendState() {
+	if len(r.Alerts) > 0 {
+		return
+	}
+	if r.State == "" {
+		r.State = "ok"
+	}
+	if r.Health != "ok" {
+		r.State = "unhealthy"
+	} else if r.IsNoMatch() {
+		r.State = "nomatch"
+	}
+}
+
 // ToAPI returns ApiGroup representation of g
 func (g *Group) ToAPI() *ApiGroup {
 	g.mu.RLock()
@@ -252,6 +267,7 @@ func (g *Group) ToAPI() *ApiGroup {
 		Headers:         headersToStrings(g.Headers),
 		NotifierHeaders: headersToStrings(g.NotifierHeaders),
 		Labels:          g.Labels,
+		States:          make(map[string]int),
 	}
 	if g.EvalOffset != nil {
 		ag.EvalOffset = g.EvalOffset.Seconds()
@@ -259,9 +275,10 @@ func (g *Group) ToAPI() *ApiGroup {
 	if g.EvalDelay != nil {
 		ag.EvalDelay = g.EvalDelay.Seconds()
 	}
-	ag.Rules = make([]ApiRule, 0)
+	ag.Rules = make([]ApiRule, 0, len(g.Rules))
 	for _, r := range g.Rules {
-		ag.Rules = append(ag.Rules, r.ToAPI())
+		ar := r.ToAPI()
+		ag.Rules = append(ag.Rules, ar)
 	}
 	return &ag
 }

app/vmalert/static/icons/icons.svg

@@ -11,7 +11,7 @@
     <path d="M224.163 175.27a1.9 1.9 0 0 0 2.8 0l6-5.9a2.1 2.1 0 0 0 .2-2.7 1.9 1.9 0 0 0-3-.2l-2.6 2.6v-5.2c0-1.54-1.667-2.502-3-1.732-.619.357-1 1.017-1 1.732v5.2l-2.6-2.6a1.9 1.9 0 0 0-3 .2 2.1 2.1 0 0 0 .2 2.7zm-16.459-23.297h36c1.54 0 2.502-1.667 1.732-3a2 2 0 0 0-1.732-1h-36c-1.54 0-2.502 1.667-1.732 3 .357.619 1.017 1 1.732 1m36 4h-36c-1.54 0-2.502 1.667-1.732 3 .357.619 1.017 1 1.732 1h36c1.54 0 2.502-1.667 1.732-3a2 2 0 0 0-1.732-1m-16.59-23.517a1.9 1.9 0 0 0-2.8 0l-6 5.9a2.1 2.1 0 0 0-.2 2.7 1.9 1.9 0 0 0 3 .2l2.6-2.6v5.2c0 1.54 1.667 2.502 3 1.732.619-.357 1-1.017 1-1.732v-5.2l2.6 2.6a1.9 1.9 0 0 0 3-.2 2.1 2.1 0 0 0-.2-2.7z"/>
   </symbol>
 
-  <symbol id="filter" viewBox="-10 -10 320 310">
+  <symbol id="state" viewBox="-10 -10 320 310">
     <path d="M288.953 0h-277c-5.522 0-10 4.478-10 10v49.531c0 5.522 4.478 10 10 10h12.372l91.378 107.397v113.978a10 10 0 0 0 15.547 8.32l49.5-33a10 10 0 0 0 4.453-8.32v-80.978l91.378-107.397h12.372c5.522 0 10-4.478 10-10V10c0-5.522-4.477-10-10-10M167.587 166.77a10 10 0 0 0-2.384 6.48v79.305l-29.5 19.666V173.25a10 10 0 0 0-2.384-6.48L50.585 69.531h199.736zM278.953 49.531h-257V20h257z"/>
   </symbol>
 

app/vmalert/static/js/custom.js

@@ -8,9 +8,9 @@ function actionAll(isCollapse) {
     });
 }
 
-function groupFilter(key) {
+function groupForState(key) {
     if (key) {
-        location.href = `?filter=${key}`;
+        location.href = `?state=${key}`;
     } else {
         window.location = window.location.pathname;
     }

app/vmalert/vmalertutil/err_group_test.go

@@ -42,7 +42,7 @@ func TestErrGroupConcurrent(_ *testing.T) {
 
 	const writersN = 4
 	payload := make(chan error, writersN)
-	for i := 0; i < writersN; i++ {
+	for range writersN {
 		go func() {
 			for err := range payload {
 				eg.Add(err)
@@ -51,7 +51,7 @@ func TestErrGroupConcurrent(_ *testing.T) {
 	}
 
 	const iterations = 500
-	for i := 0; i < iterations; i++ {
+	for i := range iterations {
 		payload <- fmt.Errorf("error %d", i)
 		if i%10 == 0 {
 			_ = eg.Err()

app/vmalert/web.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"cmp"
 	"embed"
 	"encoding/json"
 	"fmt"
+	"math"
 	"net/http"
 	"slices"
 	"strconv"
@@ -50,6 +52,7 @@ var (
 		"alert":  rule.TypeAlerting,
 		"record": rule.TypeRecording,
 	}
+	ruleStates = []string{"ok", "nomatch", "inactive", "firing", "pending", "unhealthy"}
 )
 
 type requestHandler struct {
@@ -63,6 +66,14 @@ var (
 	staticServer  = http.StripPrefix("/vmalert", staticHandler)
 )
 
+func marshalJson(v any, kind string) ([]byte, *httpserver.ErrorWithStatusCode) {
+	data, err := json.Marshal(v)
+	if err != nil {
+		return nil, errResponse(fmt.Errorf("failed to marshal %s: %s", kind, err), http.StatusInternalServerError)
+	}
+	return data, nil
+}
+
 func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	if strings.HasPrefix(r.URL.Path, "/vmalert/static") {
 		staticServer.ServeHTTP(w, r)
@@ -94,40 +105,32 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		WriteRuleDetails(w, r, rule)
+		WriteRule(w, r, rule)
 		return true
-	case "/vmalert/groups":
+	// current used by old vmalert UI and Grafana Alerts
+	case "/vmalert/groups", "/rules":
 		rf, err := newRulesFilter(r)
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		data := rh.groups(rf)
-		WriteListGroups(w, r, data, rf.filter)
+		// only support filtering by a single state
+		state := ""
+		if len(rf.states) > 0 {
+			state = rf.states[0]
+			rf.states = rf.states[:1]
+		}
+		lr := rh.groups(rf)
+		WriteListGroups(w, r, lr.Data.Groups, state)
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
 		return true
 
-	// special cases for Grafana requests,
-	// served without `vmalert` prefix:
-	case "/rules":
-		// Grafana makes an extra request to `/rules`
-		// handler in addition to `/api/v1/rules` calls in alerts UI
-		var data []*rule.ApiGroup
-		rf, err := newRulesFilter(r)
-		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
-			return true
-		}
-		data = rh.groups(rf)
-		WriteListGroups(w, r, data, rf.filter)
-		return true
-
 	case "/vmalert/api/v1/notifiers", "/api/v1/notifiers":
 		data, err := rh.listNotifiers()
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -135,15 +138,14 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
-		var data []byte
 		rf, err := newRulesFilter(r)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
-		data, err = rh.listGroups(rf)
+		data, err := rh.listGroups(rf)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -152,14 +154,14 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
-		rf, err := newRulesFilter(r)
+		gf, err := newGroupsFilter(r)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
-		data, err := rh.listAlerts(rf)
+		data, err := rh.listAlerts(gf)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -168,12 +170,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/vmalert/api/v1/alert", "/api/v1/alert":
 		alert, err := rh.getAlert(r)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
-		data, err := json.Marshal(alert)
+		data, err := marshalJson(alert, "alert")
 		if err != nil {
-			httpserver.Errorf(w, r, "failed to marshal alert: %s", err)
+			errJson(w, r, err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -182,16 +184,16 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/vmalert/api/v1/rule", "/api/v1/rule":
 		apiRule, err := rh.getRule(r)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
 		rwu := rule.ApiRuleWithUpdates{
 			ApiRule:      apiRule,
 			StateUpdates: apiRule.Updates,
 		}
-		data, err := json.Marshal(rwu)
+		data, err := marshalJson(rwu, "rule")
 		if err != nil {
-			httpserver.Errorf(w, r, "failed to marshal rule: %s", err)
+			errJson(w, r, err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -200,12 +202,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	case "/vmalert/api/v1/group", "/api/v1/group":
 		group, err := rh.getGroup(r)
 		if err != nil {
-			httpserver.Errorf(w, r, "%s", err)
+			errJson(w, r, err)
 			return true
 		}
-		data, err := json.Marshal(group)
+		data, err := marshalJson(group, "group")
 		if err != nil {
-			httpserver.Errorf(w, r, "failed to marshal group: %s", err)
+			errJson(w, r, err)
 			return true
 		}
 		w.Header().Set("Content-Type", "application/json")
@@ -225,10 +227,10 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	}
 }
 
-func (rh *requestHandler) getGroup(r *http.Request) (*rule.ApiGroup, error) {
+func (rh *requestHandler) getGroup(r *http.Request) (*rule.ApiGroup, *httpserver.ErrorWithStatusCode) {
 	groupID, err := strconv.ParseUint(r.FormValue(rule.ParamGroupID), 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err)
+		return nil, errResponse(fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err), http.StatusBadRequest)
 	}
 	obj, err := rh.m.groupAPI(groupID)
 	if err != nil {
@@ -237,14 +239,14 @@ func (rh *requestHandler) getGroup(r *http.Request) (*rule.ApiGroup, error) {
 	return obj, nil
 }
 
-func (rh *requestHandler) getRule(r *http.Request) (rule.ApiRule, error) {
+func (rh *requestHandler) getRule(r *http.Request) (rule.ApiRule, *httpserver.ErrorWithStatusCode) {
 	groupID, err := strconv.ParseUint(r.FormValue(rule.ParamGroupID), 10, 64)
 	if err != nil {
-		return rule.ApiRule{}, fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err)
+		return rule.ApiRule{}, errResponse(fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err), http.StatusBadRequest)
 	}
 	ruleID, err := strconv.ParseUint(r.FormValue(rule.ParamRuleID), 10, 64)
 	if err != nil {
-		return rule.ApiRule{}, fmt.Errorf("failed to read %q param: %w", rule.ParamRuleID, err)
+		return rule.ApiRule{}, errResponse(fmt.Errorf("failed to read %q param: %w", rule.ParamRuleID, err), http.StatusBadRequest)
 	}
 	obj, err := rh.m.ruleAPI(groupID, ruleID)
 	if err != nil {
@@ -253,14 +255,14 @@ func (rh *requestHandler) getRule(r *http.Request) (rule.ApiRule, error) {
 	return obj, nil
 }
 
-func (rh *requestHandler) getAlert(r *http.Request) (*rule.ApiAlert, error) {
+func (rh *requestHandler) getAlert(r *http.Request) (*rule.ApiAlert, *httpserver.ErrorWithStatusCode) {
 	groupID, err := strconv.ParseUint(r.FormValue(rule.ParamGroupID), 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err)
+		return nil, errResponse(fmt.Errorf("failed to read %q param: %w", rule.ParamGroupID, err), http.StatusBadRequest)
 	}
 	alertID, err := strconv.ParseUint(r.FormValue(rule.ParamAlertID), 10, 64)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read %q param: %w", rule.ParamAlertID, err)
+		return nil, errResponse(fmt.Errorf("failed to read %q param: %w", rule.ParamAlertID, err), http.StatusBadRequest)
 	}
 	a, err := rh.m.alertAPI(groupID, alertID)
 	if err != nil {
@@ -270,28 +272,76 @@ func (rh *requestHandler) getAlert(r *http.Request) (*rule.ApiAlert, error) {
 }
 
 type listGroupsResponse struct {
-	Status string `json:"status"`
-	Data   struct {
+	Status      string `json:"status"`
+	Page        int    `json:"page,omitempty"`
+	TotalPages  int    `json:"total_pages,omitempty"`
+	TotalGroups int    `json:"total_groups,omitempty"`
+	TotalRules  int    `json:"total_rules,omitempty"`
+	Data        struct {
 		Groups []*rule.ApiGroup `json:"groups"`
 	} `json:"data"`
 }
 
-// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
-type rulesFilter struct {
-	files         []string
-	groupNames    []string
-	ruleNames     []string
-	ruleType      string
-	excludeAlerts bool
-	filter        string
-	dsType        config.Type
+type groupsFilter struct {
+	groupNames []string
+	files      []string
+	dsType     config.Type
 }
 
-func newRulesFilter(r *http.Request) (*rulesFilter, error) {
-	rf := &rulesFilter{}
-	query := r.URL.Query()
+func newGroupsFilter(r *http.Request) (*groupsFilter, *httpserver.ErrorWithStatusCode) {
+	_ = r.ParseForm()
+	vs := r.Form
+	gf := &groupsFilter{
+		groupNames: vs["rule_group[]"],
+		files:      vs["file[]"],
+	}
+	dsType := vs.Get("datasource_type")
+	if len(dsType) > 0 {
+		if config.SupportedType(dsType) {
+			gf.dsType = config.NewRawType(dsType)
+		} else {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "datasource_type": not supported value %q`, dsType), http.StatusBadRequest)
+		}
+	}
+	return gf, nil
+}
 
-	ruleTypeParam := query.Get("type")
+func (gf *groupsFilter) matches(group *rule.Group) bool {
+	if len(gf.groupNames) > 0 && !slices.Contains(gf.groupNames, group.Name) {
+		return false
+	}
+	if len(gf.files) > 0 && !slices.Contains(gf.files, group.File) {
+		return false
+	}
+	if len(gf.dsType.Name) > 0 && gf.dsType.String() != group.Type.String() {
+		return false
+	}
+	return true
+}
+
+// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
+type rulesFilter struct {
+	gf             *groupsFilter
+	ruleNames      []string
+	ruleType       string
+	excludeAlerts  bool
+	states         []string
+	maxGroups      int
+	pageNum        int
+	search         string
+	extendedStates bool
+}
+
+func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusCode) {
+	gf, err := newGroupsFilter(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var rf rulesFilter
+	rf.gf = gf
+	vs := r.Form
+	ruleTypeParam := vs.Get("type")
 	if len(ruleTypeParam) > 0 {
 		if ruleType, ok := ruleTypeMap[ruleTypeParam]; ok {
 			rf.ruleType = ruleType
@@ -300,102 +350,146 @@ func newRulesFilter(r *http.Request) (*rulesFilter, error) {
 		}
 	}
 
-	dsType := query.Get("datasource_type")
-	if len(dsType) > 0 {
-		if config.SupportedType(dsType) {
-			rf.dsType = config.NewRawType(dsType)
-		} else {
-			return nil, errResponse(fmt.Errorf(`invalid parameter "datasource_type": not supported value %q`, dsType), http.StatusBadRequest)
-		}
+	states := vs["state"]
+	if len(states) == 0 {
+		states = vs["filter"]
 	}
-
-	filter := strings.ToLower(query.Get("filter"))
-	if len(filter) > 0 {
-		if filter == "nomatch" || filter == "unhealthy" {
-			rf.filter = filter
-		} else {
-			return nil, errResponse(fmt.Errorf(`invalid parameter "filter": not supported value %q`, filter), http.StatusBadRequest)
+	for _, s := range states {
+		values := strings.Split(s, ",")
+		for _, v := range values {
+			if len(v) == 0 {
+				continue
+			}
+			if !slices.Contains(ruleStates, v) {
+				return nil, errResponse(fmt.Errorf(`invalid parameter "state": contains not supported value %q`, v), http.StatusBadRequest)
+			}
+			rf.states = append(rf.states, v)
 		}
 	}
 
 	rf.excludeAlerts = httputil.GetBool(r, "exclude_alerts")
-	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
-	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
-	rf.files = append([]string{}, r.Form["file[]"]...)
-	return rf, nil
+	rf.extendedStates = httputil.GetBool(r, "extended_states")
+	rf.ruleNames = append([]string{}, vs["rule_name[]"]...)
+	rf.search = strings.ToLower(vs.Get("search"))
+
+	pageNum := vs.Get("page_num")
+	maxGroups := vs.Get("group_limit")
+	if pageNum != "" {
+		if maxGroups == "" {
+			return nil, errResponse(fmt.Errorf(`"group_limit" needs to be present in order to paginate over the groups`), http.StatusBadRequest)
+		}
+		v, err := strconv.Atoi(pageNum)
+		if err != nil || v <= 0 {
+			return nil, errResponse(fmt.Errorf(`"page_num" is expected to be a positive number, found %q`, pageNum), http.StatusBadRequest)
+		}
+		rf.pageNum = v
+	}
+	if maxGroups != "" {
+		v, err := strconv.Atoi(maxGroups)
+		if err != nil || v <= 0 {
+			return nil, errResponse(fmt.Errorf(`"group_limit" is expected to be a positive number, found %q`, maxGroups), http.StatusBadRequest)
+		}
+		rf.maxGroups = v
+	}
+	return &rf, nil
 }
 
-func (rf *rulesFilter) matchesGroup(group *rule.Group) bool {
-	if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
+func (rf *rulesFilter) matchesRule(r *rule.ApiRule) bool {
+	if rf.ruleType != "" && rf.ruleType != r.Type {
 		return false
 	}
-	if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
+	if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
 		return false
 	}
-	if len(rf.dsType.Name) > 0 && rf.dsType.String() != group.Type.String() {
-		return false
+	if len(rf.states) == 0 {
+		return true
 	}
-	return true
+	return slices.Contains(rf.states, r.State)
 }
 
-func (rh *requestHandler) groups(rf *rulesFilter) []*rule.ApiGroup {
+func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	groups := make([]*rule.ApiGroup, 0)
+	skipGroups := (rf.pageNum - 1) * rf.maxGroups
+	lr := &listGroupsResponse{
+		Status: "success",
+	}
+	lr.Data.Groups = make([]*rule.ApiGroup, 0)
+	if skipGroups >= len(rh.m.groups) {
+		return lr
+	}
+	// sort list of groups for deterministic output
+	groups := make([]*rule.Group, 0, len(rh.m.groups))
 	for _, group := range rh.m.groups {
-		if !rf.matchesGroup(group) {
+		groups = append(groups, group)
+	}
+
+	slices.SortFunc(groups, func(a, b *rule.Group) int {
+		nameCmp := cmp.Compare(a.Name, b.Name)
+		if nameCmp != 0 {
+			return nameCmp
+		}
+		return cmp.Compare(a.File, b.File)
+	})
+	for _, group := range groups {
+		if !rf.gf.matches(group) {
 			continue
 		}
+		groupFound := len(rf.search) == 0 || strings.Contains(strings.ToLower(group.Name), rf.search) || strings.Contains(strings.ToLower(group.File), rf.search)
 		g := group.ToAPI()
 		// the returned list should always be non-nil
 		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
 		filteredRules := make([]rule.ApiRule, 0)
 		for _, rule := range g.Rules {
-			if rf.ruleType != "" && rf.ruleType != rule.Type {
+			if !groupFound && !strings.Contains(strings.ToLower(rule.Name), rf.search) {
 				continue
 			}
-			if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, rule.Name) {
-				continue
+			if rf.extendedStates {
+				rule.ExtendState()
 			}
-			if (rule.LastError == "" && rf.filter == "unhealthy") || (!isNoMatch(rule) && rf.filter == "nomatch") {
+			if !rf.matchesRule(&rule) {
 				continue
 			}
 			if rf.excludeAlerts {
 				rule.Alerts = nil
 			}
-			if rule.LastError != "" {
-				g.Unhealthy++
-			} else {
-				g.Healthy++
-			}
-			if isNoMatch(rule) {
-				g.NoMatch++
-			}
+			g.States[rule.State]++
 			filteredRules = append(filteredRules, rule)
 		}
-		g.Rules = filteredRules
-		groups = append(groups, g)
-	}
-	// sort list of groups for deterministic output
-	slices.SortFunc(groups, func(a, b *rule.ApiGroup) int {
-		if a.Name != b.Name {
-			return strings.Compare(a.Name, b.Name)
+		if len(g.Rules) == 0 || len(filteredRules) > 0 {
+			if rf.maxGroups > 0 {
+				lr.TotalGroups++
+				lr.TotalRules += len(filteredRules)
+			}
+			if skipGroups > 0 {
+				skipGroups--
+				continue
+			}
+			if rf.maxGroups == 0 || len(lr.Data.Groups) < rf.maxGroups {
+				g.Rules = filteredRules
+				lr.Data.Groups = append(lr.Data.Groups, g)
+			}
 		}
-		return strings.Compare(a.File, b.File)
-	})
-	return groups
+	}
+	if rf.maxGroups > 0 {
+		lr.Page = rf.pageNum
+		lr.TotalPages = max(int(math.Ceil(float64(lr.TotalGroups)/float64(rf.maxGroups))), 1)
+	}
+	return lr
 }
 
-func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, error) {
-	lr := listGroupsResponse{Status: "success"}
-	lr.Data.Groups = rh.groups(rf)
+func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
+	lr := rh.groups(rf)
+	if rf.pageNum > 1 && len(lr.Data.Groups) == 0 {
+		return nil, errResponse(fmt.Errorf(`page_num exceeds total amount of pages`), http.StatusBadRequest)
+	}
+	if lr.Page > lr.TotalPages {
+		return nil, errResponse(fmt.Errorf(`page_num=%d exceeds total amount of pages in result=%d`, lr.Page, lr.TotalPages), http.StatusBadRequest)
+	}
 	b, err := json.Marshal(lr)
 	if err != nil {
-		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`error encoding list of active alerts: %w`, err),
-			StatusCode: http.StatusInternalServerError,
-		}
+		return nil, errResponse(fmt.Errorf(`error encoding list of groups: %w`, err), http.StatusInternalServerError)
 	}
 	return b, nil
 }
@@ -434,14 +528,14 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
 	return gAlerts
 }
 
-func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
+func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
 	lr.Data.Alerts = make([]*rule.ApiAlert, 0)
 	for _, group := range rh.m.groups {
-		if !rf.matchesGroup(group) {
+		if !gf.matches(group) {
 			continue
 		}
 		g := group.ToAPI()
@@ -460,10 +554,7 @@ func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 
 	b, err := json.Marshal(lr)
 	if err != nil {
-		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`error encoding list of active alerts: %w`, err),
-			StatusCode: http.StatusInternalServerError,
-		}
+		return nil, errResponse(fmt.Errorf(`error encoding list of active alerts: %w`, err), http.StatusInternalServerError)
 	}
 	return b, nil
 }
@@ -475,7 +566,7 @@ type listNotifiersResponse struct {
 	} `json:"data"`
 }
 
-func (rh *requestHandler) listNotifiers() ([]byte, error) {
+func (rh *requestHandler) listNotifiers() ([]byte, *httpserver.ErrorWithStatusCode) {
 	targets := notifier.GetTargets()
 
 	lr := listNotifiersResponse{Status: "success"}
@@ -497,10 +588,7 @@ func (rh *requestHandler) listNotifiers() ([]byte, error) {
 
 	b, err := json.Marshal(lr)
 	if err != nil {
-		return nil, &httpserver.ErrorWithStatusCode{
-			Err:        fmt.Errorf(`error encoding list of notifiers: %w`, err),
-			StatusCode: http.StatusInternalServerError,
-		}
+		return nil, errResponse(fmt.Errorf(`error encoding list of notifiers: %w`, err), http.StatusInternalServerError)
 	}
 	return b, nil
 }
@@ -511,3 +599,8 @@ func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
 		StatusCode: sc,
 	}
 }
+
+func errJson(w http.ResponseWriter, r *http.Request, err *httpserver.ErrorWithStatusCode) {
+	w.Header().Set("Content-Type", "application/json")
+	httpserver.Errorf(w, r, `{"error":%q,"errorType":%d}`, err, err.StatusCode)
+}

app/vmalert/web.qtpl

@@ -12,7 +12,7 @@
     "github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 ) %}
 
-{% func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) %}
+{% func Controls(prefix, currentIcon, currentText string, icons, states map[string]string, search bool) %}
     <div class="btn-toolbar mb-3" role="toolbar">
         <div class="d-flex gap-2 justify-content-between w-100">
             <div class="d-flex gap-2 align-items-center">
@@ -28,10 +28,10 @@
                         <use href="{%s prefix %}static/icons/icons.svg#expand"/>
                     </svg>
                 </a>
-                {% if len(filters) > 0 %}
+                {% if len(states) > 0 %}
                     <span class="d-none d-md-inline-block">Filter by status:</span>
                     <svg class="d-md-none" width="20" height="20">
-                        <use href="{%s prefix %}static/icons/icons.svg#filter">
+                        <use href="{%s prefix %}static/icons/icons.svg#state">
                     </svg>
                     <div class="dropdown">
                         <button
@@ -46,10 +46,10 @@
                             </svg>
                         </button>
                         <ul class="dropdown-menu">
-                            {% for key, title := range filters %}
+                            {% for key, title := range states %}
                                 {% if title != currentText %}
                                     <li>
-                                        <a class="dropdown-item" onclick="groupFilter('{%s key %}')">
+                                        <a class="dropdown-item" onclick="groupForState('{%s key %}')">
                                             <span class="d-none d-md-inline-block">{%s title %}</span>
                                             <svg class="d-md-none" width="22" height="22">
                                                 <use href="{%s prefix %}static/icons/icons.svg#{%s icons[key] %}"/>
@@ -97,10 +97,10 @@
     {%= tpl.Footer(r) %}
 {% endfunc %}
 
-{% func ListGroups(r *http.Request, groups []*rule.ApiGroup, filter string) %}
+{% func ListGroups(r *http.Request, groups []*rule.ApiGroup, state string) %}
     {%code
         prefix := vmalertutil.Prefix(r.URL.Path)
-        filters := map[string]string{
+        states := map[string]string{
             "":          "All",
             "unhealthy": "Unhealthy",
             "nomatch":   "No Match",
@@ -110,14 +110,14 @@
             "unhealthy": "unhealthy",
             "nomatch":   "nomatch",
         }
-        currentText := filters[filter]
-        currentIcon := icons[filter]
+        currentText := states[state]
+        currentIcon := icons[state]
     %}
     {%= tpl.Header(r, navItems, "Groups", getLastConfigError()) %}
-        {%= Controls(prefix, currentIcon, currentText, icons, filters, true) %}
+        {%= Controls(prefix, currentIcon, currentText, icons, states, true) %}
         {%  if len(groups) > 0 %}
             {% for _, g := range groups %}
-                <div id="group-{%s g.ID %}" class="w-100 border-0 flex-column vm-group{% if g.Unhealthy > 0 %} alert-danger{% endif %}">
+                <div id="group-{%s g.ID %}" class="w-100 border-0 flex-column vm-group{% if g.States["unhealthy"] > 0 %} alert-danger{% endif %}">
                     <span class="d-flex justify-content-between">
                         <a
                             class="vm-group-search"
@@ -130,9 +130,9 @@
                             data-bs-target="#item-{%s g.ID %}"
                         >
                             <span class="d-flex gap-2">
-                                {% if g.Unhealthy > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.Unhealthy %}</span> {% endif %}
-                                {% if g.NoMatch > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.NoMatch %}</span> {% endif %}
-                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.Healthy %}</span>
+                                {% if g.States["unhealthy"] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d g.States["unhealthy"] %}</span> {% endif %}
+                                {% if g.States["nomatch"] > 0 %}<span class="badge bg-warning" title="Number of rules with status NoMatch">{%d g.States["nomatch"] %}</span> {% endif %}
+                                <span class="badge bg-success" title="Number of rules with status Ok">{%d g.States["ok"] %}</span>
                             </span>
                         </span>
                     </span>
@@ -189,7 +189,7 @@
                                                         <b>record:</b> {%s r.Name %}
                                                     {% endif %}
                                                     |
-                                                    {%= seriesFetchedWarn(prefix, r) %}
+                                                    {%= seriesFetchedWarn(prefix, &r) %}
                                                     <span><a target="_blank" href="{%s prefix+r.WebLink() %}">Details</a></span>
                                                 </div>
                                                 <div class="col-12">
@@ -476,7 +476,7 @@
 {% endfunc %}
 
 
-{% func RuleDetails(r *http.Request, rule rule.ApiRule) %}
+{% func Rule(r *http.Request, rule rule.ApiRule) %}
     {%code prefix := vmalertutil.Prefix(r.URL.Path) %}
     {%= tpl.Header(r, navItems, "", getLastConfigError()) %}
     {%code
@@ -661,8 +661,8 @@
 <span class="badge bg-warning text-dark" title="This firing state is kept because of `keep_firing_for`">stabilizing</span>
 {% endfunc %}
 
-{% func seriesFetchedWarn(prefix string, r rule.ApiRule) %}
-{% if isNoMatch(r) %}
+{% func seriesFetchedWarn(prefix string, r *rule.ApiRule) %}
+{% if r.IsNoMatch() %}
 <svg
     data-bs-toggle="tooltip"
     title="No match! This rule's last evaluation hasn't selected any time series from the datasource.
@@ -673,9 +673,3 @@
 </svg>
 {% endif %}
 {% endfunc %}
-
-{%code
-  func isNoMatch (r rule.ApiRule) bool {
-    return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0
-  }
-%}

app/vmalert/web.qtpl.go

@@ -31,7 +31,7 @@ var (
 )
 
 //line app/vmalert/web.qtpl:15
-func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) {
+func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText string, icons, states map[string]string, search bool) {
 //line app/vmalert/web.qtpl:15
 	qw422016.N().S(`
     <div class="btn-toolbar mb-3" role="toolbar">
@@ -59,7 +59,7 @@ func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText
                 </a>
                 `)
 //line app/vmalert/web.qtpl:31
-	if len(filters) > 0 {
+	if len(states) > 0 {
 //line app/vmalert/web.qtpl:31
 		qw422016.N().S(`
                     <span class="d-none d-md-inline-block">Filter by status:</span>
@@ -68,7 +68,7 @@ func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText
 //line app/vmalert/web.qtpl:34
 		qw422016.E().S(prefix)
 //line app/vmalert/web.qtpl:34
-		qw422016.N().S(`static/icons/icons.svg#filter">
+		qw422016.N().S(`static/icons/icons.svg#state">
                     </svg>
                     <div class="dropdown">
                         <button
@@ -97,7 +97,7 @@ func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText
                         <ul class="dropdown-menu">
                             `)
 //line app/vmalert/web.qtpl:49
-		for key, title := range filters {
+		for key, title := range states {
 //line app/vmalert/web.qtpl:49
 			qw422016.N().S(`
                                 `)
@@ -106,7 +106,7 @@ func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText
 //line app/vmalert/web.qtpl:50
 				qw422016.N().S(`
                                     <li>
-                                        <a class="dropdown-item" onclick="groupFilter('`)
+                                        <a class="dropdown-item" onclick="groupForState('`)
 //line app/vmalert/web.qtpl:52
 				qw422016.E().S(key)
 //line app/vmalert/web.qtpl:52
@@ -176,22 +176,22 @@ func StreamControls(qw422016 *qt422016.Writer, prefix, currentIcon, currentText
 }
 
 //line app/vmalert/web.qtpl:77
-func WriteControls(qq422016 qtio422016.Writer, prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) {
+func WriteControls(qq422016 qtio422016.Writer, prefix, currentIcon, currentText string, icons, states map[string]string, search bool) {
 //line app/vmalert/web.qtpl:77
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vmalert/web.qtpl:77
-	StreamControls(qw422016, prefix, currentIcon, currentText, icons, filters, search)
+	StreamControls(qw422016, prefix, currentIcon, currentText, icons, states, search)
 //line app/vmalert/web.qtpl:77
 	qt422016.ReleaseWriter(qw422016)
 //line app/vmalert/web.qtpl:77
 }
 
 //line app/vmalert/web.qtpl:77
-func Controls(prefix, currentIcon, currentText string, icons, filters map[string]string, search bool) string {
+func Controls(prefix, currentIcon, currentText string, icons, states map[string]string, search bool) string {
 //line app/vmalert/web.qtpl:77
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vmalert/web.qtpl:77
-	WriteControls(qb422016, prefix, currentIcon, currentText, icons, filters, search)
+	WriteControls(qb422016, prefix, currentIcon, currentText, icons, states, search)
 //line app/vmalert/web.qtpl:77
 	qs422016 := string(qb422016.B)
 //line app/vmalert/web.qtpl:77
@@ -324,13 +324,13 @@ func Welcome(r *http.Request) string {
 }
 
 //line app/vmalert/web.qtpl:100
-func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule.ApiGroup, filter string) {
+func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule.ApiGroup, state string) {
 //line app/vmalert/web.qtpl:100
 	qw422016.N().S(`
     `)
 //line app/vmalert/web.qtpl:102
 	prefix := vmalertutil.Prefix(r.URL.Path)
-	filters := map[string]string{
+	states := map[string]string{
 		"":          "All",
 		"unhealthy": "Unhealthy",
 		"nomatch":   "No Match",
@@ -340,8 +340,8 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
 		"unhealthy": "unhealthy",
 		"nomatch":   "nomatch",
 	}
-	currentText := filters[filter]
-	currentIcon := icons[filter]
+	currentText := states[state]
+	currentIcon := icons[state]
 
 //line app/vmalert/web.qtpl:115
 	qw422016.N().S(`
@@ -352,7 +352,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
 	qw422016.N().S(`
         `)
 //line app/vmalert/web.qtpl:117
-	StreamControls(qw422016, prefix, currentIcon, currentText, icons, filters, true)
+	StreamControls(qw422016, prefix, currentIcon, currentText, icons, states, true)
 //line app/vmalert/web.qtpl:117
 	qw422016.N().S(`
         `)
@@ -371,7 +371,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
 //line app/vmalert/web.qtpl:120
 			qw422016.N().S(`" class="w-100 border-0 flex-column vm-group`)
 //line app/vmalert/web.qtpl:120
-			if g.Unhealthy > 0 {
+			if g.States["unhealthy"] > 0 {
 //line app/vmalert/web.qtpl:120
 				qw422016.N().S(` alert-danger`)
 //line app/vmalert/web.qtpl:120
@@ -418,11 +418,11 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
                             <span class="d-flex gap-2">
                                 `)
 //line app/vmalert/web.qtpl:133
-			if g.Unhealthy > 0 {
+			if g.States["unhealthy"] > 0 {
 //line app/vmalert/web.qtpl:133
 				qw422016.N().S(`<span class="badge bg-danger" title="Number of rules with status Error">`)
 //line app/vmalert/web.qtpl:133
-				qw422016.N().D(g.Unhealthy)
+				qw422016.N().D(g.States["unhealthy"])
 //line app/vmalert/web.qtpl:133
 				qw422016.N().S(`</span> `)
 //line app/vmalert/web.qtpl:133
@@ -431,11 +431,11 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
 			qw422016.N().S(`
                                 `)
 //line app/vmalert/web.qtpl:134
-			if g.NoMatch > 0 {
+			if g.States["nomatch"] > 0 {
 //line app/vmalert/web.qtpl:134
 				qw422016.N().S(`<span class="badge bg-warning" title="Number of rules with status NoMatch">`)
 //line app/vmalert/web.qtpl:134
-				qw422016.N().D(g.NoMatch)
+				qw422016.N().D(g.States["nomatch"])
 //line app/vmalert/web.qtpl:134
 				qw422016.N().S(`</span> `)
 //line app/vmalert/web.qtpl:134
@@ -444,7 +444,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
 			qw422016.N().S(`
                                 <span class="badge bg-success" title="Number of rules with status Ok">`)
 //line app/vmalert/web.qtpl:135
-			qw422016.N().D(g.Healthy)
+			qw422016.N().D(g.States["ok"])
 //line app/vmalert/web.qtpl:135
 			qw422016.N().S(`</span>
                             </span>
@@ -617,7 +617,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
                                                     |
                                                     `)
 //line app/vmalert/web.qtpl:192
-				streamseriesFetchedWarn(qw422016, prefix, r)
+				streamseriesFetchedWarn(qw422016, prefix, &r)
 //line app/vmalert/web.qtpl:192
 				qw422016.N().S(`
                                                     <span><a target="_blank" href="`)
@@ -750,22 +750,22 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*rule
 }
 
 //line app/vmalert/web.qtpl:234
-func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*rule.ApiGroup, filter string) {
+func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*rule.ApiGroup, state string) {
 //line app/vmalert/web.qtpl:234
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vmalert/web.qtpl:234
-	StreamListGroups(qw422016, r, groups, filter)
+	StreamListGroups(qw422016, r, groups, state)
 //line app/vmalert/web.qtpl:234
 	qt422016.ReleaseWriter(qw422016)
 //line app/vmalert/web.qtpl:234
 }
 
 //line app/vmalert/web.qtpl:234
-func ListGroups(r *http.Request, groups []*rule.ApiGroup, filter string) string {
+func ListGroups(r *http.Request, groups []*rule.ApiGroup, state string) string {
 //line app/vmalert/web.qtpl:234
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vmalert/web.qtpl:234
-	WriteListGroups(qb422016, r, groups, filter)
+	WriteListGroups(qb422016, r, groups, state)
 //line app/vmalert/web.qtpl:234
 	qs422016 := string(qb422016.B)
 //line app/vmalert/web.qtpl:234
@@ -1462,7 +1462,7 @@ func Alert(r *http.Request, alert *rule.ApiAlert) string {
 }
 
 //line app/vmalert/web.qtpl:479
-func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule rule.ApiRule) {
+func StreamRule(qw422016 *qt422016.Writer, r *http.Request, rule rule.ApiRule) {
 //line app/vmalert/web.qtpl:479
 	qw422016.N().S(`
     `)
@@ -1859,22 +1859,22 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule rule.Api
 }
 
 //line app/vmalert/web.qtpl:642
-func WriteRuleDetails(qq422016 qtio422016.Writer, r *http.Request, rule rule.ApiRule) {
+func WriteRule(qq422016 qtio422016.Writer, r *http.Request, rule rule.ApiRule) {
 //line app/vmalert/web.qtpl:642
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vmalert/web.qtpl:642
-	StreamRuleDetails(qw422016, r, rule)
+	StreamRule(qw422016, r, rule)
 //line app/vmalert/web.qtpl:642
 	qt422016.ReleaseWriter(qw422016)
 //line app/vmalert/web.qtpl:642
 }
 
 //line app/vmalert/web.qtpl:642
-func RuleDetails(r *http.Request, rule rule.ApiRule) string {
+func Rule(r *http.Request, rule rule.ApiRule) string {
 //line app/vmalert/web.qtpl:642
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vmalert/web.qtpl:642
-	WriteRuleDetails(qb422016, r, rule)
+	WriteRule(qb422016, r, rule)
 //line app/vmalert/web.qtpl:642
 	qs422016 := string(qb422016.B)
 //line app/vmalert/web.qtpl:642
@@ -2015,12 +2015,12 @@ func badgeStabilizing() string {
 }
 
 //line app/vmalert/web.qtpl:664
-func streamseriesFetchedWarn(qw422016 *qt422016.Writer, prefix string, r rule.ApiRule) {
+func streamseriesFetchedWarn(qw422016 *qt422016.Writer, prefix string, r *rule.ApiRule) {
 //line app/vmalert/web.qtpl:664
 	qw422016.N().S(`
 `)
 //line app/vmalert/web.qtpl:665
-	if isNoMatch(r) {
+	if r.IsNoMatch() {
 //line app/vmalert/web.qtpl:665
 		qw422016.N().S(`
 <svg
@@ -2045,7 +2045,7 @@ func streamseriesFetchedWarn(qw422016 *qt422016.Writer, prefix string, r rule.Ap
 }
 
 //line app/vmalert/web.qtpl:675
-func writeseriesFetchedWarn(qq422016 qtio422016.Writer, prefix string, r rule.ApiRule) {
+func writeseriesFetchedWarn(qq422016 qtio422016.Writer, prefix string, r *rule.ApiRule) {
 //line app/vmalert/web.qtpl:675
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vmalert/web.qtpl:675
@@ -2056,7 +2056,7 @@ func writeseriesFetchedWarn(qq422016 qtio422016.Writer, prefix string, r rule.Ap
 }
 
 //line app/vmalert/web.qtpl:675
-func seriesFetchedWarn(prefix string, r rule.ApiRule) string {
+func seriesFetchedWarn(prefix string, r *rule.ApiRule) string {
 //line app/vmalert/web.qtpl:675
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vmalert/web.qtpl:675
@@ -2069,8 +2069,3 @@ func seriesFetchedWarn(prefix string, r rule.ApiRule) string {
 	return qs422016
 //line app/vmalert/web.qtpl:675
 }
-
-//line app/vmalert/web.qtpl:678
-func isNoMatch(r rule.ApiRule) bool {
-	return r.LastSamples == 0 && r.LastSeriesFetched != nil && *r.LastSeriesFetched == 0
-}

app/vmalert/web_test.go

@@ -210,7 +210,7 @@ func TestHandler(t *testing.T) {
 		}
 	})
 
-	t.Run("/api/v1/rules&filters", func(t *testing.T) {
+	t.Run("/api/v1/rules&states", func(t *testing.T) {
 		check := func(url string, statusCode, expGroups, expRules int) {
 			t.Helper()
 			lr := listGroupsResponse{}
@@ -252,9 +252,15 @@ func TestHandler(t *testing.T) {
 		check("/api/v1/rules?rule_group[]=group&file[]=foo", 200, 0, 0)
 		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 200, 3, 6)
 
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 200, 3, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 200, 0, 0)
 		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 200, 3, 3)
 		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 200, 3, 6)
+
+		check("/api/v1/rules?group_limit=1", 200, 1, 2)
+		check("/api/v1/rules?group_limit=1&type=alert", 200, 1, 1)
+		check("/api/v1/rules?group_limit=1&type=record", 200, 1, 1)
+		check("/api/v1/rules?group_limit=2", 200, 2, 4)
+		check(fmt.Sprintf("/api/v1/rules?group_limit=1&page_num=%d", 1), 200, 1, 2)
 	})
 	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
 		// check if response returns active alerts by default

app/vmauth/auth_config.go

@@ -13,6 +13,7 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -28,6 +29,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs/fscore"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -65,10 +67,11 @@ type AuthConfig struct {
 type UserInfo struct {
 	Name string `yaml:"name,omitempty"`
 
-	BearerToken string `yaml:"bearer_token,omitempty"`
-	AuthToken   string `yaml:"auth_token,omitempty"`
-	Username    string `yaml:"username,omitempty"`
-	Password    string `yaml:"password,omitempty"`
+	BearerToken string     `yaml:"bearer_token,omitempty"`
+	JWT         *JWTConfig `yaml:"jwt,omitempty"`
+	AuthToken   string     `yaml:"auth_token,omitempty"`
+	Username    string     `yaml:"username,omitempty"`
+	Password    string     `yaml:"password,omitempty"`
 
 	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
 	DiscoverBackendIPs     *bool       `yaml:"discover_backend_ips,omitempty"`
@@ -89,6 +92,8 @@ type UserInfo struct {
 
 	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
 
+	AccessLog *AccessLog `yaml:"access_log,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
@@ -101,11 +106,40 @@ type UserInfo struct {
 	requestsDuration *metrics.Summary
 }
 
+// AccessLog represents configuration for access log settings.
+type AccessLog struct {
+	Filters *AccessLogFilters `yaml:"filters"`
+}
+
+// AccessLogFilters represents list of filters for access logs printing
+type AccessLogFilters struct {
+	// SkipStatusCodes is a list of HTTP status codes for which access logs will be skipped
+	SkipStatusCodes []int `yaml:"skip_status_codes"`
+}
+
+func (ui *UserInfo) logRequest(r *http.Request, userName string, statusCode int, duration time.Duration) {
+	if ui.AccessLog == nil {
+		return
+	}
+	filters := ui.AccessLog.Filters
+	if filters != nil && len(filters.SkipStatusCodes) > 0 {
+		if slices.Contains(filters.SkipStatusCodes, statusCode) {
+			return
+		}
+	}
+
+	remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+	requestURI := httpserver.GetRequestURI(r)
+	logger.Infof("access_log request_host=%q request_uri=%q status_code=%d remote_addr=%s user_agent=%q referer=%q duration_ms=%d username=%q",
+		r.Host, requestURI, statusCode, remoteAddr, r.UserAgent(), r.Referer(), duration.Milliseconds(), userName)
+}
+
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
-	RequestHeaders   []*Header `yaml:"headers,omitempty"`
-	ResponseHeaders  []*Header `yaml:"response_headers,omitempty"`
-	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
+	RequestHeaders     []*Header `yaml:"headers,omitempty"`
+	ResponseHeaders    []*Header `yaml:"response_headers,omitempty"`
+	KeepOriginalHost   *bool     `yaml:"keep_original_host,omitempty"`
+	hasAnyPlaceHolders bool
 }
 
 func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
@@ -113,7 +147,7 @@ func (ui *UserInfo) beginConcurrencyLimit(ctx context.Context) error {
 	case ui.concurrencyLimitCh <- struct{}{}:
 		return nil
 	default:
-		// The number of concurrently executed requests for the given user equals the limt.
+		// The number of concurrently executed requests for the given user equals the limit.
 		// Wait until some of the currently executed requests are finished, so the current request could be executed.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
 		select {
@@ -348,6 +382,7 @@ func (bus *backendURLs) add(u *url.URL) {
 		url:                u,
 		healthCheckContext: bus.healthChecksContext,
 		healthCheckWG:      &bus.healthChecksWG,
+		hasPlaceHolders:    hasAnyPlaceholders(u),
 	})
 }
 
@@ -365,6 +400,8 @@ type backendURL struct {
 	concurrentRequests atomic.Int32
 
 	url *url.URL
+
+	hasPlaceHolders bool
 }
 
 func (bu *backendURL) isBroken() bool {
@@ -588,7 +625,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 
 	// Slow path - select other backend urls.
 	n := atomicCounter.Add(1) - 1
-	for i := uint32(0); i < uint32(len(bus)); i++ {
+	for i := range uint32(len(bus)) {
 		idx := (n + i) % uint32(len(bus))
 		bu := bus[idx]
 		if bu.isBroken() {
@@ -598,7 +635,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 		// The Load() in front of CompareAndSwap() avoids CAS overhead for items with values bigger than 0.
 		if bu.concurrentRequests.Load() == 0 && bu.concurrentRequests.CompareAndSwap(0, 1) {
 			atomicCounter.CompareAndSwap(n+1, idx+1)
-			// There is no need in the call bu.get(), because we already incremented bu.concrrentRequests above.
+			// There is no need in the call bu.get(), because we already incremented bu.concurrentRequests above.
 			return bu
 		}
 	}
@@ -799,6 +836,9 @@ var (
 	// authUsers contains the currently loaded auth users
 	authUsers atomic.Pointer[map[string]*UserInfo]
 
+	// jwt authentication cache
+	jwtAuthCache atomic.Pointer[jwtCache]
+
 	authConfigWG sync.WaitGroup
 	stopCh       chan struct{}
 )
@@ -838,6 +878,16 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 		return false, fmt.Errorf("failed to parse auth config: %w", err)
 	}
 
+	jui, oidcDP, err := parseJWTUsers(ac)
+	if err != nil {
+		return false, fmt.Errorf("failed to parse JWT users from auth config: %w", err)
+	}
+	oidcDP.startDiscovery()
+	jwtc := &jwtCache{
+		users:  jui,
+		oidcDP: oidcDP,
+	}
+
 	m, err := parseAuthConfigUsers(ac)
 	if err != nil {
 		return false, fmt.Errorf("failed to parse users from auth config: %w", err)
@@ -854,9 +904,15 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 	}
 	metrics.RegisterSet(ac.ms)
 
+	jwtcPrev := jwtAuthCache.Load()
+	if jwtcPrev != nil {
+		jwtcPrev.oidcDP.stopDiscovery()
+	}
+
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
+	jwtAuthCache.Store(jwtc)
 
 	return true, nil
 }
@@ -881,12 +937,18 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if ui.BearerToken != "" {
 			return nil, fmt.Errorf("field bearer_token can't be specified for unauthorized_user section")
 		}
+		if ui.JWT != nil {
+			return nil, fmt.Errorf("field jwt can't be specified for unauthorized_user section")
+		}
 		if ui.AuthToken != "" {
 			return nil, fmt.Errorf("field auth_token can't be specified for unauthorized_user section")
 		}
 		if ui.Name != "" {
 			return nil, fmt.Errorf("field name can't be specified for unauthorized_user section")
 		}
+		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
+			return nil, err
+		}
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
@@ -927,16 +989,27 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	}
 	for i := range uis {
 		ui := &uis[i]
+		// users with jwt tokens are parsed by parseJWTUsers function.
+		// the function also checks that users with jwt tokens do not have auth tokens, bearer tokens, usernames and passwords.
+		if ui.JWT != nil {
+			continue
+		}
+
 		ats, err := getAuthTokens(ui.AuthToken, ui.BearerToken, ui.Username, ui.Password)
 		if err != nil {
 			return nil, err
 		}
+
 		for _, at := range ats {
 			if uiOld := byAuthToken[at]; uiOld != nil {
 				return nil, fmt.Errorf("duplicate auth token=%q found for username=%q, name=%q; the previous one is set for username=%q, name=%q",
 					at, ui.Username, ui.Name, uiOld.Username, uiOld.Name)
 			}
 		}
+
+		if err := parseJWTPlaceholdersForUserInfo(ui, false); err != nil {
+			return nil, err
+		}
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
@@ -1036,6 +1109,7 @@ func (ui *UserInfo) initURLs() error {
 			return err
 		}
 	}
+
 	for _, e := range ui.URLMaps {
 		if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 && len(e.SrcQueryArgs) == 0 && len(e.SrcHeaders) == 0 {
 			return fmt.Errorf("missing `src_paths`, `src_hosts`, `src_query_args` and `src_headers` in `url_map`")
@@ -1095,6 +1169,9 @@ func (ui *UserInfo) name() string {
 		h := xxhash.Sum64([]byte(ui.AuthToken))
 		return fmt.Sprintf("auth_token:hash:%016X", h)
 	}
+	if ui.JWT != nil {
+		return `jwt`
+	}
 	return ""
 }
 

app/vmauth/auth_config_test.go

@@ -4,8 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"net"
+	"net/http"
 	"net/url"
+	"strings"
 	"testing"
+	"time"
 
 	"gopkg.in/yaml.v2"
 
@@ -276,6 +279,50 @@ users:
   url_prefix: http://foo.bar
   metric_labels:
     not-prometheus-compatible: value
+`)
+	// placeholder in url_prefix
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix: 'http://ahost/{{a_placeholder}}/foobar'
+`)
+	// placeholder in a header
+	f(`
+users:
+- username: foo
+  password: bar
+  headers:
+  - 'X-Foo: {{a_placeholder}}'
+  url_prefix: 'http://ahost'
+`)
+	// placeholder in url_prefix
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix: 'http://ahost/{{a_placeholder}}/foobar'
+`)
+	// placeholder in a header in url_map
+	f(`
+users:
+- username: foo
+  password: bar
+  url_map:
+    - src_paths: ["/select/.*"]
+      headers:
+        - 'X-Foo: {{a_placeholder}}'
+      url_prefix: 'http://ahost'
+`)
+
+	// placeholder in a header in url_map
+	f(`
+users:
+- username: foo
+  password: bar
+  url_map:
+    - src_paths: ["/select/.*"]
+      url_prefix: 'http://ahost/{{a_placeholder}}/foobar'
 `)
 }
 
@@ -378,7 +425,7 @@ users:
 			RetryStatusCodes:       []int{500, 501},
 			LoadBalancingPolicy:    "first_available",
 			MergeQueryArgs:         []string{"foo", "bar"},
-			DropSrcPathPrefixParts: intp(1),
+			DropSrcPathPrefixParts: new(1),
 			DiscoverBackendIPs:     &discoverBackendIPsTrue,
 		},
 	}, nil)
@@ -621,6 +668,47 @@ unauthorized_user:
 			},
 		},
 	})
+
+	// skip user info with jwt, it is parsed by parseJWTUsers
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix: http://aaa:343/bbb
+- jwt: {skip_verify: true}
+  url_prefix: http://aaa:343/bbb
+`, map[string]*UserInfo{
+		getHTTPAuthBasicToken("foo", "bar"): {
+			Username:  "foo",
+			Password:  "bar",
+			URLPrefix: mustParseURL("http://aaa:343/bbb"),
+		},
+	}, nil)
+
+	// Multiple users with access logs enabled
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo
+  access_log: {}
+- username: bar
+  url_prefix: https://bar/x/
+  access_log:
+    filters:
+      skip_status_codes: [404]
+`, map[string]*UserInfo{
+		getHTTPAuthBasicToken("foo", ""): {
+			Username:  "foo",
+			URLPrefix: mustParseURL("http://foo"),
+			AccessLog: &AccessLog{},
+		},
+		getHTTPAuthBasicToken("bar", ""): {
+			Username:  "bar",
+			URLPrefix: mustParseURL("https://bar/x/"),
+			AccessLog: &AccessLog{Filters: &AccessLogFilters{SkipStatusCodes: []int{404}}},
+		},
+	}, nil)
+
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -831,7 +919,7 @@ func TestBrokenBackend(t *testing.T) {
 	bus[1].setBroken()
 
 	// broken backend should never return while there are healthy backends
-	for i := 0; i < 1e3; i++ {
+	for range int(1e3) {
 		b := up.getBackendURL()
 		if b.isBroken() {
 			t.Fatalf("unexpected broken backend %q", b.url)
@@ -908,6 +996,41 @@ func TestDiscoverBackendIPsWithIPV6(t *testing.T) {
 
 }
 
+func TestLogRequest(t *testing.T) {
+	ui := &UserInfo{AccessLog: &AccessLog{}}
+
+	testOutput := &bytes.Buffer{}
+	logger.SetOutputForTests(testOutput)
+	defer logger.ResetOutputForTest()
+
+	req, err := http.NewRequest("GET", "http://localhost:8080/select/0/prometheus", nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	f := func(user string, status int, duration time.Duration, expectedLog string) {
+		t.Helper()
+
+		testOutput.Reset()
+		ui.logRequest(req, user, status, duration)
+
+		got := testOutput.String()
+		if expectedLog == "" && got != "" {
+			t.Fatalf("expected empty log, got %q", got)
+		}
+		if !strings.Contains(got, expectedLog) {
+			t.Fatalf("output \n%q \nshould contain \n%q", testOutput.String(), expectedLog)
+		}
+	}
+
+	f("foo", 200, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=200 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
+	f("foo", 404, time.Second, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=1000 username="foo"`)
+
+	ui.AccessLog.Filters = &AccessLogFilters{SkipStatusCodes: []int{200}}
+	f("foo", 200, 10*time.Millisecond, ``)
+	f("foo", 404, 10*time.Millisecond, `access_log request_host="localhost:8080" request_uri="" status_code=404 remote_addr="" user_agent="" referer="" duration_ms=10 username="foo"`)
+}
+
 func getRegexs(paths []string) []*Regex {
 	var sps []*Regex
 	for _, path := range paths {
@@ -963,10 +1086,6 @@ func mustParseURLs(us []string) *URLPrefix {
 	return up
 }
 
-func intp(n int) *int {
-	return &n
-}
-
 func mustNewRegex(s string) *Regex {
 	var re Regex
 	if err := yaml.Unmarshal([]byte(s), &re); err != nil {

app/vmauth/example_config.yml

@@ -116,6 +116,20 @@ users:
   - "http://default1:8888/unsupported_url_handler"
   - "http://default2:8888/unsupported_url_handler"
 
+# A JWT token based routing:
+# - Requests with JWT token that has the following structure:
+# {"team": "ops", "security": {"read_access": "1"}, "vm_access": {"metrics_account_id": 1000,"metrics_project_id":5}}
+# is routed to vmselect nodes and request url placeholder replaced with metrics tenant identificators
+- name: jwt-opts-team
+  jwt:
+    match_claims:
+     team: ops
+     security.read_access: "1"
+    skip_verify: true
+  url_prefix:
+  - "http://vmselect1:8481/select/{{.MetricsTenant}}/prometheus"
+  - "http://vmselect2:8481/select/{{.MetricsTenant}}/prometheus"
+
 # Requests without Authorization header are proxied according to `unauthorized_user` section.
 # Requests are proxied in round-robin fashion between `url_prefix` backends.
 # The deny_partial_response query arg is added to all the proxied requests.
@@ -125,3 +139,8 @@ unauthorized_user:
   - http://vmselect-az1/?deny_partial_response=1
   - http://vmselect-az2/?deny_partial_response=1
   retry_status_codes: [503, 500]
+  # log access for requests routed to this user
+  access_log:
+    filters:
+      # except requests with Status Codes below
+      skip_status_codes: [200, 202]

app/vmauth/jwt.go

@@ -0,0 +1,486 @@
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+)
+
+const (
+	metricsTenantPlaceholder       = `{{.MetricsTenant}}`
+	metricsExtraLabelsPlaceholder  = `{{.MetricsExtraLabels}}`
+	metricsExtraFiltersPlaceholder = `{{.MetricsExtraFilters}}`
+
+	logsAccountIDPlaceholder          = `{{.LogsAccountID}}`
+	logsProjectIDPlaceholder          = `{{.LogsProjectID}}`
+	logsExtraFiltersPlaceholder       = `{{.LogsExtraFilters}}`
+	logsExtraStreamFiltersPlaceholder = `{{.LogsExtraStreamFilters}}`
+
+	placeholderPrefix = `{{`
+)
+
+var allPlaceholders = []string{
+	metricsTenantPlaceholder,
+	metricsExtraLabelsPlaceholder,
+	metricsExtraFiltersPlaceholder,
+	logsAccountIDPlaceholder,
+	logsProjectIDPlaceholder,
+	logsExtraFiltersPlaceholder,
+	logsExtraStreamFiltersPlaceholder,
+}
+
+var urlPathPlaceHolders = []string{
+	metricsTenantPlaceholder,
+	logsAccountIDPlaceholder,
+	logsProjectIDPlaceholder,
+}
+
+type jwtCache struct {
+	// users contain UserInfo`s from AuthConfig with JWTConfig set
+	users []*UserInfo
+
+	oidcDP *oidcDiscovererPool
+}
+
+type JWTConfig struct {
+	PublicKeys        []string          `yaml:"public_keys,omitempty"`
+	PublicKeyFiles    []string          `yaml:"public_key_files,omitempty"`
+	SkipVerify        bool              `yaml:"skip_verify,omitempty"`
+	OIDC              *oidcConfig       `yaml:"oidc,omitempty"`
+	MatchClaims       map[string]string `yaml:"match_claims,omitempty"`
+	parsedMatchClaims []*jwt.Claim
+
+	// verifierPool is used to verify JWT tokens.
+	// It is initialized from PublicKeys and/or PublicKeyFiles.
+	// In this case, it is initialized once at config reload and never updated until next reload
+	// In case of OIDC, it is initialized on config reload and periodically updated by discovery process.
+	verifierPool atomic.Pointer[jwt.VerifierPool]
+}
+
+func parseJWTUsers(ac *AuthConfig) ([]*UserInfo, *oidcDiscovererPool, error) {
+	jui := make([]*UserInfo, 0, len(ac.Users))
+	oidcDP := &oidcDiscovererPool{}
+
+	uniqClaims := make(map[string]*UserInfo)
+	var sortedClaims []string
+	for idx, ui := range ac.Users {
+		jwtToken := ui.JWT
+		if jwtToken == nil {
+			continue
+		}
+
+		if ui.AuthToken != "" || ui.BearerToken != "" || ui.Username != "" || ui.Password != "" {
+			return nil, nil, fmt.Errorf("auth_token, bearer_token, username and password cannot be specified if jwt is set")
+		}
+		if len(jwtToken.PublicKeys) == 0 && len(jwtToken.PublicKeyFiles) == 0 && !jwtToken.SkipVerify && jwtToken.OIDC == nil {
+			return nil, nil, fmt.Errorf("jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true")
+		}
+		var claimsString string
+		sortedClaims = sortedClaims[:0]
+		parsedClaims := make([]*jwt.Claim, 0, len(jwtToken.MatchClaims))
+		for ck, cv := range jwtToken.MatchClaims {
+			sortedClaims = append(sortedClaims, fmt.Sprintf("%s=%s", ck, cv))
+			pc, err := jwt.NewClaim(ck, cv)
+			if err != nil {
+				return nil, nil, fmt.Errorf("incorrect match claim, key=%q, value regex=%q: %w", ck, cv, err)
+			}
+			parsedClaims = append(parsedClaims, pc)
+		}
+		ui.JWT.parsedMatchClaims = parsedClaims
+		sort.Strings(sortedClaims)
+		claimsString = strings.Join(sortedClaims, ",")
+
+		if oldUI, ok := uniqClaims[claimsString]; ok {
+			return nil, nil, fmt.Errorf("duplicate match claims=%q found for name=%q at idx=%d; the previous one is set for name=%q", claimsString, ui.Name, idx, oldUI.Name)
+		}
+		uniqClaims[claimsString] = &ui
+		if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 {
+			keys := make([]any, 0, len(jwtToken.PublicKeys)+len(jwtToken.PublicKeyFiles))
+
+			for i := range jwtToken.PublicKeys {
+				k, err := jwt.ParseKey([]byte(jwtToken.PublicKeys[i]))
+				if err != nil {
+					return nil, nil, err
+				}
+				keys = append(keys, k)
+			}
+
+			for _, filePath := range jwtToken.PublicKeyFiles {
+				keyData, err := os.ReadFile(filePath)
+				if err != nil {
+					return nil, nil, fmt.Errorf("cannot read public key from file %q: %w", filePath, err)
+				}
+				k, err := jwt.ParseKey(keyData)
+				if err != nil {
+					return nil, nil, fmt.Errorf("cannot parse public key from file %q: %w", filePath, err)
+				}
+				keys = append(keys, k)
+			}
+
+			vp, err := jwt.NewVerifierPool(keys)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			jwtToken.verifierPool.Store(vp)
+		}
+		if jwtToken.OIDC != nil {
+			if len(jwtToken.PublicKeys) > 0 || len(jwtToken.PublicKeyFiles) > 0 || jwtToken.SkipVerify {
+				return nil, nil, fmt.Errorf("jwt with oidc cannot contain public keys or have skip_verify=true")
+			}
+
+			if jwtToken.OIDC.Issuer == "" {
+				return nil, nil, fmt.Errorf("oidc issuer cannot be empty")
+			}
+			isserURL, err := url.Parse(jwtToken.OIDC.Issuer)
+			if err != nil {
+				return nil, nil, fmt.Errorf("oidc issuer %q must be a valid URL", jwtToken.OIDC.Issuer)
+			}
+			if isserURL.Scheme != "https" && isserURL.Scheme != "http" {
+				return nil, nil, fmt.Errorf("oidc issuer %q must have http or https scheme", jwtToken.OIDC.Issuer)
+			}
+
+			oidcDP.createOrAdd(ui.JWT.OIDC.Issuer, &ui.JWT.verifierPool)
+		}
+
+		if err := parseJWTPlaceholdersForUserInfo(&ui, true); err != nil {
+			return nil, nil, err
+		}
+
+		if err := ui.initURLs(); err != nil {
+			return nil, nil, err
+		}
+
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, nil, fmt.Errorf("cannot parse metric_labels: %w", err)
+		}
+		ui.requests = ac.ms.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.requestErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_errors_total` + metricLabels)
+		ui.backendRequests = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_requests_total` + metricLabels)
+		ui.backendErrors = ac.ms.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = ac.ms.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
+		mcr := ui.getMaxConcurrentRequests()
+		ui.concurrencyLimitCh = make(chan struct{}, mcr)
+		ui.concurrencyLimitReached = ac.ms.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
+			return float64(cap(ui.concurrencyLimitCh))
+		})
+		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
+			return float64(len(ui.concurrencyLimitCh))
+		})
+
+		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
+		if err != nil {
+			return nil, nil, fmt.Errorf("cannot initialize HTTP RoundTripper: %w", err)
+		}
+		ui.rt = rt
+
+		jui = append(jui, &ui)
+	}
+
+	// sort by amount of matching claims
+	// it allows to more specific claim win in case of clash
+	sort.SliceStable(jui, func(i, j int) bool {
+		return len(jui[i].JWT.MatchClaims) > len(jui[j].JWT.MatchClaims)
+	})
+
+	return jui, oidcDP, nil
+}
+
+var tokenPool sync.Pool
+
+func getToken() *jwt.Token {
+	tkn := tokenPool.Get()
+	if tkn == nil {
+		return &jwt.Token{}
+	}
+	return tkn.(*jwt.Token)
+}
+
+func putToken(tkn *jwt.Token) {
+	tkn.Reset()
+	tokenPool.Put(tkn)
+}
+
+func getJWTUserInfo(ats []string) (*UserInfo, *jwt.Token) {
+	js := *jwtAuthCache.Load()
+	if len(js.users) == 0 {
+		return nil, nil
+	}
+
+	tkn := getToken()
+
+	for _, at := range ats {
+		if strings.Count(at, ".") != 2 {
+			continue
+		}
+
+		at, _ = strings.CutPrefix(at, `http_auth:`)
+		tkn.Reset()
+		if err := tkn.Parse(at, true); err != nil {
+			if *logInvalidAuthTokens {
+				logger.Infof("cannot parse jwt token: %s", err)
+			}
+			continue
+		}
+		if tkn.IsExpired(time.Now()) {
+			if *logInvalidAuthTokens {
+				// TODO: add more context:
+				// token claims with issuer
+				logger.Infof("jwt token is expired")
+			}
+			continue
+		}
+
+		if ui := getUserInfoByJWTToken(tkn, js.users); ui != nil {
+			return ui, tkn
+		}
+	}
+
+	putToken(tkn)
+	return nil, nil
+}
+
+func getUserInfoByJWTToken(tkn *jwt.Token, users []*UserInfo) *UserInfo {
+	for _, ui := range users {
+		if !tkn.MatchClaims(ui.JWT.parsedMatchClaims) {
+			continue
+		}
+
+		if ui.JWT.SkipVerify {
+			return ui
+		}
+
+		if ui.JWT.OIDC != nil {
+			// OIDC requires iss claim.
+			// It must match the discovery issuer URL set in OIDC config.
+			// https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata
+			if tkn.Issuer() == "" {
+				if *logInvalidAuthTokens {
+					logger.Infof("jwt token must have issuer filed")
+				}
+				return nil
+			}
+			if tkn.Issuer() != ui.JWT.OIDC.Issuer {
+				if *logInvalidAuthTokens {
+					logger.Infof("jwt token issuer: %q does not match oidc issuer: %q", tkn.Issuer(), ui.JWT.OIDC.Issuer)
+				}
+				return nil
+			}
+		}
+
+		vp := ui.JWT.verifierPool.Load()
+		if vp == nil {
+			if *logInvalidAuthTokens {
+				logger.Infof("jwt verifier not initialed")
+			}
+			return nil
+		}
+
+		if err := vp.Verify(tkn); err != nil {
+			if *logInvalidAuthTokens {
+				logger.Infof("cannot verify jwt token: %s", err)
+			}
+			return nil
+		}
+
+		return ui
+	}
+
+	if *logInvalidAuthTokens {
+		logger.Infof("no user match jwt token")
+	}
+
+	return nil
+}
+
+func replaceJWTPlaceholders(bu *backendURL, hc HeadersConf, vma *jwt.VMAccessClaim) (*url.URL, HeadersConf) {
+	if !bu.hasPlaceHolders && !hc.hasAnyPlaceHolders {
+		return bu.url, hc
+	}
+	targetURL := bu.url
+	data := jwtClaimsData(vma)
+	if bu.hasPlaceHolders {
+		// template url params and request path
+		// make a copy of url
+		uCopy := *bu.url
+		for _, uph := range urlPathPlaceHolders {
+			replacement := data[uph]
+			uCopy.Path = strings.ReplaceAll(uCopy.Path, uph, replacement[0])
+		}
+		query := uCopy.Query()
+		var foundAnyQueryPlaceholder bool
+		var templatedValues []string
+		for param, values := range query {
+			templatedValues = templatedValues[:0]
+			// filter in-place values with placeholders
+			// and accumulate replacements
+			// it will change the order of param values
+			// but it's not guaranteed
+			// and will be changed in any way with multiple arg templates
+			var cnt int
+			for _, value := range values {
+				if dv, ok := data[value]; ok {
+					foundAnyQueryPlaceholder = true
+					templatedValues = append(templatedValues, dv...)
+					continue
+				}
+				values[cnt] = value
+				cnt++
+			}
+			values = values[:cnt]
+			values = append(values, templatedValues...)
+			query[param] = values
+		}
+		if foundAnyQueryPlaceholder {
+			uCopy.RawQuery = query.Encode()
+		}
+		targetURL = &uCopy
+	}
+	if hc.hasAnyPlaceHolders {
+		// make a copy of headers and update only values with placeholder
+		rhs := make([]*Header, 0, len(hc.RequestHeaders))
+		for _, rh := range hc.RequestHeaders {
+			if dv, ok := data[rh.Value]; ok {
+				rh := &Header{
+					Name:  rh.Name,
+					Value: strings.Join(dv, ","),
+				}
+				rhs = append(rhs, rh)
+				continue
+			}
+			rhs = append(rhs, rh)
+		}
+		hc.RequestHeaders = rhs
+	}
+
+	return targetURL, hc
+}
+
+func jwtClaimsData(vma *jwt.VMAccessClaim) map[string][]string {
+	data := map[string][]string{
+		// TODO: optimize at parsing stage
+		metricsTenantPlaceholder:       {fmt.Sprintf("%d:%d", vma.MetricsAccountID, vma.MetricsProjectID)},
+		metricsExtraLabelsPlaceholder:  vma.MetricsExtraLabels,
+		metricsExtraFiltersPlaceholder: vma.MetricsExtraFilters,
+
+		// TODO: optimize at parsing stage
+		logsAccountIDPlaceholder:          {fmt.Sprintf("%d", vma.LogsAccountID)},
+		logsProjectIDPlaceholder:          {fmt.Sprintf("%d", vma.LogsProjectID)},
+		logsExtraFiltersPlaceholder:       vma.LogsExtraFilters,
+		logsExtraStreamFiltersPlaceholder: vma.LogsExtraStreamFilters,
+	}
+	return data
+}
+
+func parseJWTPlaceholdersForUserInfo(ui *UserInfo, isAllowed bool) error {
+	if ui.URLPrefix != nil {
+		if err := validateJWTPlaceholdersForURL(ui.URLPrefix, isAllowed); err != nil {
+			return err
+		}
+	}
+	if err := parsePlaceholdersForHC(&ui.HeadersConf, isAllowed); err != nil {
+		return err
+	}
+	if ui.DefaultURL != nil {
+		if err := validateJWTPlaceholdersForURL(ui.DefaultURL, isAllowed); err != nil {
+			return fmt.Errorf("invalid `default_url` placeholders: %w", err)
+		}
+	}
+	for i := range ui.URLMaps {
+		e := &ui.URLMaps[i]
+		if e.URLPrefix != nil {
+			if err := validateJWTPlaceholdersForURL(e.URLPrefix, isAllowed); err != nil {
+				return fmt.Errorf("invalid `url_map` `url_prefix` placeholders: %w", err)
+			}
+		}
+		if err := parsePlaceholdersForHC(&e.HeadersConf, isAllowed); err != nil {
+			return fmt.Errorf("invalid `url_map` headers placeholders: %w", err)
+		}
+
+	}
+	return nil
+}
+
+func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
+	for _, bu := range up.busOriginal {
+		ok := strings.Contains(bu.Path, placeholderPrefix)
+		if ok && !isAllowed {
+			return fmt.Errorf("placeholder: %q is only allowed at JWT token context", bu.Path)
+		}
+		if ok {
+			p := bu.Path
+			for _, ph := range allPlaceholders {
+				p = strings.ReplaceAll(p, ph, ``)
+			}
+			if strings.Contains(p, placeholderPrefix) {
+				return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
+
+			}
+		}
+		for param, values := range bu.Query() {
+			for _, value := range values {
+				ok := strings.Contains(value, placeholderPrefix)
+				if ok && !isAllowed {
+					return fmt.Errorf("query param: %q with placeholder: %q is only allowed at JWT token context", param, value)
+				}
+				if ok {
+					// possible placeholder
+					if !slices.Contains(allPlaceholders, value) {
+						return fmt.Errorf("query param: %q has unsupported placeholder string: %q, supported values are: %s", param, value, strings.Join(allPlaceholders, ", "))
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func parsePlaceholdersForHC(hc *HeadersConf, isAllowed bool) error {
+	for _, rhs := range hc.RequestHeaders {
+		ok := strings.Contains(rhs.Value, placeholderPrefix)
+		if ok && !isAllowed {
+			return fmt.Errorf("request header: %q placeholder: %q is only supported at JWT context", rhs.Name, rhs.Value)
+		}
+		if ok {
+			if !slices.Contains(allPlaceholders, rhs.Value) {
+				return fmt.Errorf("request header: %q has unsupported placeholder: %q, supported values are: %s", rhs.Name, rhs.Value, strings.Join(allPlaceholders, ", "))
+			}
+			hc.hasAnyPlaceHolders = true
+		}
+	}
+	for _, rhs := range hc.ResponseHeaders {
+		if strings.Contains(rhs.Value, placeholderPrefix) {
+			return fmt.Errorf("response header placeholders are not supported; found placeholder prefix at header: %q with value: %q", rhs.Name, rhs.Value)
+		}
+	}
+	return nil
+}
+
+func hasAnyPlaceholders(u *url.URL) bool {
+	if strings.Contains(u.Path, placeholderPrefix) {
+		return true
+	}
+	if len(u.Query()) == 0 {
+		return false
+	}
+	for _, values := range u.Query() {
+		for _, value := range values {
+			if strings.HasPrefix(value, placeholderPrefix) {
+				return true
+			}
+		}
+
+	}
+	return false
+}

app/vmauth/jwt_test.go

@@ -0,0 +1,503 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestJWTParseAuthConfigFailure(t *testing.T) {
+	validRSAPublicKey := `-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiX7oPWKOWRQsGFEWvwZO
+mL2PYsdYUsu9nr0qtPCjxQHUJgLfT3rdKlvKpPFYv7ZmKnqTncg36Wz9uiYmWJ7e
+IB5Z+fko8kVIMzarCqVvpAJDzYF/pUii68xvuYoK3L9TIOAeyCXv+prwnr2IH+Mw
+9AONzWbRrYoO74XyTE9vMU5qmI/L1VPk+PR8lqPOSptLvzsfoaIk2ED4yK2nRB+6
+st+k4nccPqbErqHc8aiXnXfugfnr6b+NPFYUzKsDqkymGOokVijrI8B3jNw6c6Do
+zphk+D3wgLsXYHfMcZbXIMqffqm/aB8Qg88OpFOkQ3rd2p6R9+hacnZkfkn3Phiw
+yQIDAQAB
+-----END PUBLIC KEY-----
+`
+	// ECDSA with the P-521 curve
+	validECDSAPublicKey := `-----BEGIN PUBLIC KEY-----
+MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAU9RmtkCRuYTKCyvLlDn5DtBZOHSe
+QTa5j9q/oQVpCKqcXVFrH5dgh0GL+P/ZhkeuowPzCZqntGf0+7wPt9OxSJcADVJm
+dv92m540MXss8zdHf5qtE0gsu2Ved0R7Z8a8QwGZ/1mYZ+kFGGbdQTlSvRqDySTq
+XOtclIk1uhc03oL9nOQ=
+-----END PUBLIC KEY-----
+`
+
+	f := func(s string, expErr string) {
+		t.Helper()
+		ac, err := parseAuthConfig([]byte(s))
+		if err != nil {
+			if expErr != err.Error() {
+				t.Fatalf("unexpected error; got\n%q\nwant\n%q", err.Error(), expErr)
+			}
+			return
+		}
+		users, oidcDP, err := parseJWTUsers(ac)
+		if err == nil {
+			t.Fatalf("expecting non-nil error; got %v", users)
+		}
+		if expErr != err.Error() {
+			t.Fatalf("unexpected error; got\n%q\nwant \n%q", err.Error(), expErr)
+		}
+		if oidcDP != nil {
+			t.Fatalf("expecting nil oidcDP; got %v", oidcDP)
+		}
+	}
+
+	// unauthorized_user cannot be used with jwt
+	f(`
+unauthorized_user:
+  jwt: {skip_verify: true}
+  url_prefix: http://foo.bar
+`, `field jwt can't be specified for unauthorized_user section`)
+
+	// username and jwt in a single config
+	f(`
+users:
+- username: foo
+  jwt: {skip_verify: true}
+  url_prefix: http://foo.bar
+`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
+	// bearer_token and jwt in a single config
+	f(`
+users:
+- bearer_token: foo
+  jwt: {skip_verify: true}
+  url_prefix: http://foo.bar
+`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
+	// bearer_token and jwt in a single config
+	f(`
+users:
+- auth_token: "Foo token"
+  jwt: {skip_verify: true}
+  url_prefix: http://foo.bar
+`, `auth_token, bearer_token, username and password cannot be specified if jwt is set`)
+
+	// jwt public_keys or skip_verify must be set, part 1
+	f(`
+users:
+- jwt: {}
+  url_prefix: http://foo.bar
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
+
+	// jwt public_keys or skip_verify must be set, part 2
+	f(`
+users:
+- jwt: {public_keys: null}
+  url_prefix: http://foo.bar
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
+
+	// jwt public_keys or skip_verify must be set, part 3
+	f(`
+users:
+- jwt: {public_keys: []}
+  url_prefix: http://foo.bar
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
+
+	// jwt public_keys, public_key_files or skip_verify must be set
+	f(`
+users:
+- jwt: {public_key_files: []}
+  url_prefix: http://foo.bar
+`, `jwt must contain at least a single public key, public_key_files, oidc or have skip_verify=true`)
+
+	// invalid public key, part 1
+	f(`
+users:
+- jwt: {public_keys: [""]}
+  url_prefix: http://foo.bar
+`, `failed to parse key "": failed to decode PEM block containing public key`)
+
+	// invalid public key, part 2
+	f(`
+users:
+- jwt: {public_keys: ["invalid"]}
+  url_prefix: http://foo.bar
+`, `failed to parse key "invalid": failed to decode PEM block containing public key`)
+
+	// invalid public key, part 2
+	f(fmt.Sprintf(`
+users:
+- jwt: 
+    public_keys:
+    - %q
+    - %q
+    - "invalid"
+  url_prefix: http://foo.bar
+`, validRSAPublicKey, validECDSAPublicKey), `failed to parse key "invalid": failed to decode PEM block containing public key`)
+
+	// several jwt users
+	// invalid public key, part 2
+	f(fmt.Sprintf(`
+users:
+- jwt: 
+    public_keys:
+    - %q
+  url_prefix: http://foo.bar
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: http://foo.bar
+`, validRSAPublicKey, validECDSAPublicKey), `duplicate match claims="" found for name="" at idx=1; the previous one is set for name=""`)
+
+	// public key file doesn't exist
+	f(`
+users:
+- jwt: 
+    public_key_files: 
+    - /path/to/nonexistent/file.pem
+  url_prefix: http://foo.bar
+`, "cannot read public key from file \"/path/to/nonexistent/file.pem\": open /path/to/nonexistent/file.pem: no such file or directory")
+
+	// public key file invalid
+	// auth with key from file
+	publicKeyFile := filepath.Join(t.TempDir(), "a_public_key.pem")
+	if err := os.WriteFile(publicKeyFile, []byte(`invalidPEM`), 0o644); err != nil {
+		t.Fatalf("failed to write public key file: %s", err)
+	}
+	f(`
+users:
+- jwt: 
+    public_key_files: 
+    - `+publicKeyFile+`
+  url_prefix: http://foo.bar
+`, "cannot parse public key from file \""+publicKeyFile+"\": failed to parse key \"invalidPEM\": failed to decode PEM block containing public key")
+
+	// unsupported placeholder in a header
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+  url_prefix: http://foo.bar/{{.UnsupportedPlaceholder}}/foo`,
+		"invalid placeholder found in URL request path: \"/{{.UnsupportedPlaceholder}}/foo\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+	)
+	// unsupported placeholder in a header
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+  headers:
+    - "AccountID: {{.UnsupportedPlaceholder}}"
+  url_prefix: http://foo.bar
+`,
+		"request header: \"AccountID\" has unsupported placeholder: \"{{.UnsupportedPlaceholder}}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+	)
+
+	// spaces in templating not allowed
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+  headers:
+    - "AccountID: {{ .LogsAccountID }}"
+  url_prefix: http://foo.bar
+`,
+		"request header: \"AccountID\" has unsupported placeholder: \"{{ .LogsAccountID }}\", supported values are: {{.MetricsTenant}}, {{.MetricsExtraLabels}}, {{.MetricsExtraFilters}}, {{.LogsAccountID}}, {{.LogsProjectID}}, {{.LogsExtraFilters}}, {{.LogsExtraStreamFilters}}",
+	)
+
+	// oidc is not an object
+	f(`
+users:
+- jwt: 
+    oidc: "not an object"
+  url_prefix: http://foo.bar
+`,
+		"cannot unmarshal AuthConfig data: yaml: unmarshal errors:\n  line 4: cannot unmarshal !!str `not an ...` into main.oidcConfig",
+	)
+
+	// oidc issuer empty
+	f(`
+users:
+- jwt: 
+    oidc: {}
+  url_prefix: http://foo.bar
+`,
+		"oidc issuer cannot be empty",
+	)
+
+	// oidc issuer invalid urls
+	f(`
+users:
+- jwt: 
+    oidc:
+      issuer: "::invalid-url"
+  url_prefix: http://foo.bar
+`,
+		"oidc issuer \"::invalid-url\" must be a valid URL",
+	)
+
+	// oidc issuer invalid urls
+	f(`
+users:
+- jwt: 
+    oidc:
+      issuer: "invalid-url"
+  url_prefix: http://foo.bar
+`,
+		"oidc issuer \"invalid-url\" must have http or https scheme",
+	)
+
+	// oidc and public_keys are not allowed
+	f(fmt.Sprintf(`
+users:
+- jwt: 
+    public_keys:
+    - %q
+    oidc: 
+      issuer: https://example.com
+  url_prefix: http://foo.bar
+`, validRSAPublicKey),
+		"jwt with oidc cannot contain public keys or have skip_verify=true",
+	)
+
+	// oidc and skip_verify are not allowed
+	f(`
+users:
+- jwt: 
+    skip_verify: true
+    oidc: 
+      issuer: https://example.com
+  url_prefix: http://foo.bar
+`,
+		"jwt with oidc cannot contain public keys or have skip_verify=true",
+	)
+	// duplicate claims
+	f(`
+users:
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: ops
+  name: user-1
+  url_prefix: http://foo.bar
+- jwt:
+    skip_verify: true
+    match_claims:
+     team: ops
+  name: user-2
+  url_prefix: http://foo.bar`,
+		"duplicate match claims=\"team=ops\" found for name=\"user-2\" at idx=1; the previous one is set for name=\"user-1\"",
+	)
+}
+
+func TestJWTParseAuthConfigSuccess(t *testing.T) {
+	validRSAPublicKey := `-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAiX7oPWKOWRQsGFEWvwZO
+mL2PYsdYUsu9nr0qtPCjxQHUJgLfT3rdKlvKpPFYv7ZmKnqTncg36Wz9uiYmWJ7e
+IB5Z+fko8kVIMzarCqVvpAJDzYF/pUii68xvuYoK3L9TIOAeyCXv+prwnr2IH+Mw
+9AONzWbRrYoO74XyTE9vMU5qmI/L1VPk+PR8lqPOSptLvzsfoaIk2ED4yK2nRB+6
+st+k4nccPqbErqHc8aiXnXfugfnr6b+NPFYUzKsDqkymGOokVijrI8B3jNw6c6Do
+zphk+D3wgLsXYHfMcZbXIMqffqm/aB8Qg88OpFOkQ3rd2p6R9+hacnZkfkn3Phiw
+yQIDAQAB
+-----END PUBLIC KEY-----
+`
+	// ECDSA with the P-521 curve
+	validECDSAPublicKey := `-----BEGIN PUBLIC KEY-----
+MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQAU9RmtkCRuYTKCyvLlDn5DtBZOHSe
+QTa5j9q/oQVpCKqcXVFrH5dgh0GL+P/ZhkeuowPzCZqntGf0+7wPt9OxSJcADVJm
+dv92m540MXss8zdHf5qtE0gsu2Ved0R7Z8a8QwGZ/1mYZ+kFGGbdQTlSvRqDySTq
+XOtclIk1uhc03oL9nOQ=
+-----END PUBLIC KEY-----
+`
+
+	f := func(s string) {
+		t.Helper()
+		ac, err := parseAuthConfig([]byte(s))
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		jui, oidcDP, err := parseJWTUsers(ac)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		oidcDP.startDiscovery()
+		defer oidcDP.stopDiscovery()
+
+		for _, ui := range jui {
+			if ui.JWT == nil {
+				t.Fatalf("unexpected nil JWTConfig")
+			}
+
+			if ui.JWT.SkipVerify {
+				if ui.JWT.verifierPool.Load() != nil {
+					t.Fatalf("unexpected non-nil verifier pool for skip_verify=true")
+				}
+				continue
+			}
+
+			if ui.JWT.verifierPool.Load() == nil {
+				t.Fatalf("unexpected nil verifier pool for non-empty public keys")
+			}
+		}
+	}
+
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: http://foo.bar
+`, validRSAPublicKey))
+
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: http://foo.bar
+`, validECDSAPublicKey))
+
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+    - %q
+  url_prefix: http://foo.bar
+`, validRSAPublicKey, validECDSAPublicKey))
+
+	f(`
+users:
+- jwt:
+    skip_verify: true
+  url_prefix: http://foo.bar
+`)
+
+	// combined with other auth methods
+	f(`
+users:
+- username: foo
+  password: bar
+  url_prefix: http://foo.bar
+
+- jwt:
+    skip_verify: true
+  url_prefix: http://foo.bar
+
+- bearer_token: foo
+  url_prefix: http://foo.bar
+`)
+
+	rsaKeyFile := filepath.Join(t.TempDir(), "rsa_public_key.pem")
+	if err := os.WriteFile(rsaKeyFile, []byte(validRSAPublicKey), 0o644); err != nil {
+		t.Fatalf("failed to write RSA key file: %s", err)
+	}
+	ecdsaKeyFile := filepath.Join(t.TempDir(), "ecdsa_public_key.pem")
+	if err := os.WriteFile(ecdsaKeyFile, []byte(validECDSAPublicKey), 0o644); err != nil {
+		t.Fatalf("failed to write ECDSA key file: %s", err)
+	}
+
+	// Test single public key file
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_key_files:
+    - %q
+  url_prefix: http://foo.bar
+`, rsaKeyFile))
+
+	// Test multiple public key files
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_key_files:
+    - %q
+    - %q
+  url_prefix: http://foo.bar
+`, rsaKeyFile, ecdsaKeyFile))
+
+	// Test combined inline keys and files
+	f(fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+    public_key_files:
+    - %q
+  url_prefix: http://foo.bar
+`, validECDSAPublicKey, rsaKeyFile))
+
+	// oidc stub server
+	var ipSrv *httptest.Server
+	ipSrv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/.well-known/openid-configuration" {
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]string{
+				"issuer":   ipSrv.URL,
+				"jwks_uri": fmt.Sprintf("%s/jwks", ipSrv.URL),
+			})
+			return
+		}
+		if r.URL.Path == "/jwks" {
+			// resp generated by https://jwkset.com/generate
+			w.Header().Set("Content-Type", "application/json")
+			w.Write([]byte(`
+{
+  "keys": [
+    {
+      "kty": "RSA",
+      "kid": "f13eee91-f566-4829-80fa-fca847c21f0e",
+      "d": "Ua1llEFz3LZ05CrK5a2JxKMUEWJGXhBPPF20hHQjzxd1w0IEJK_mhPZQG8dNtBROBNIi1FC9l6QRw-RTnVIVat5Xy4yDFNKXXL3ZLXejOHY8SXrNEIDqQ-cSwIpK9cK7Umib0PcPeEeeAED5mqDH75D8_YssWFF18kLbNB5Z9pZmn6Fshiht7l2Sh4GN-KcReOW6eiQQwckDte3OGmZCRbtEriLWJt5TUGUvfZVIlcclqNMycNB6jGa9E1pO5Up7Ki3ZbI_-6XmRgZPtqnR9oLJ1zn3fj3hYpCXo-zcqLuOu3qxcslsq5igsfBzgGtfIJHY9LfWmHUsaDEa5cAX1gQ",
+      "n": "xbLXXBTNREk70UCMiqZ53_mTzYh89W-UaPU61GZ-RZ5lYcLgyWOb5mdyRbvJpcgfZpsOeGAUWbk3GkQ4vqn8kUMnnWhUum2Qk9kGubOJGLW6yaURd00j3E-ilQ5xO2R_Hzz8bAojxV8GKdGTQ-iTf8z8nsSHH8kR2SERbNJCFFtwtFU7vyFWyoH4Lmvu2UpICTHFCR9RqwQVjyoKB1JjJ6Dh1L4zPTlsvQEnqoeFQHPYr0QcQSMYXdfPvlt_FiLOAOE89fX_9T2r9WbFAoda3uTRE5_aal0jxUU2cFyeVSIgauNtF07fp422XFb4XPkWQWrdNx0KX53laSIYQ9HOpw",
+      "e": "AQAB",
+      "p": "2JT57AD-Q2lamgjgyn0wL7DgYZ3OoCTTrDm5_NHg6h13uDvyIlXSukuUeWm4tzPSDedpstbS7dgXkLw5eQXBHwPYtByTcEZS8Z37CBnhMOOhfo_U1aNIPPanJACvWBgz47-TxHsxW1YhztZqghRoicBZPSSBAj49MgANJ4jF0zc",
+      "q": "6a4MkeSXJI-ZzQ-bgP8hwJqpLFr0AiNGQcjZMH4Nn4CPGdnGiqqe6flhfLimgbNhbb67B0-8fLIji8zGhGKDL_JSIpAAdmfs2vzeEsY2hScrqVbd1VbfRcRh0J6lsn7obxkbvQthp9sX2DQbeDcEeaFEvd9gDKQSATYEqWo7eBE",
+      "dp": "haL2yu6Z9RJuuxi7S3YPY33qFZF_y0St71j3L854zzw7gMxMTW9TRWwZQwk-1pv9AmNFzvnK0MNDVyUs-UXZsb932TrApshdqYRnPsppLvdl0GgDVYcYrbUr0IUzrFHSwraVAOlavRbaaXvX4EejcUvkRFvf1nh83fs2Iqy8E-U",
+      "dq": "Cnf5qC-Ndd3ZDg688LJ9WJuVKJ-Kfu4Fn7zXvgxnn9Wqk4XmFyA9rk21yFidXQIkQz5gMpun3g48-W5bFmMzbVp1w4af_q35NnZNnJm0p5Jxqkxx87TIm9-IYkg5NB3rW87MJ1PzNAnkr5LmCCSu1qQa6Eaxjt9qzxMUcmKH94E",
+      "qi": "saAeU11iaKHmye3cwCAYkegcyWbXV3xIXEVJtS9Af_yM19UhspwY2VhuwRaajcwYZwtvR9_ITmX9M-ea7uLdd7aDYO1fujC8NGbopeC4Hkr7yb5vTly3pfKf4h-3LwGGUucJUetdz1lmMIYiyuG4_gSf1yIEtPDLKzXiedgEMdI"
+    }
+  ]
+}
+`))
+			return
+		}
+
+		http.NotFound(w, r)
+	}))
+	defer ipSrv.Close()
+
+	f(`
+users:
+- jwt:
+    oidc:
+      issuer: ` + ipSrv.URL + `
+  url_prefix: http://foo.bar
+`)
+	// multiple match claims
+	f(fmt.Sprintf(`
+users:
+- jwt:
+   match_claims:
+    role: ro
+    team: dev
+   public_keys:
+    - %q
+  url_prefix: http://foo.bar
+- jwt:
+   match_claims:
+      role: admin
+      team: dev
+   public_key_files:
+    - %q
+    - %q
+  url_prefix: http://foo.bar
+- jwt:
+   match_claims:
+     role: viewer
+     team: dev
+     department: ceo
+   skip_verify: true
+  url_prefix: http://foo.bar
+
+
+`, validRSAPublicKey, rsaKeyFile, ecdsaKeyFile))
+
+}

app/vmauth/main.go

@@ -16,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
 	"github.com/VictoriaMetrics/metrics"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
@@ -47,7 +48,7 @@ var (
 	responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 
 	requestBufferSize = flagutil.NewBytes("requestBufferSize", 32*1024, "The size of the buffer for reading the request body before proxying the request to backends. "+
-		"This allows reducing the comsumption of backend resources when processing requests from clients connected via slow networks. "+
+		"This allows reducing the consumption of backend resources when processing requests from clients connected via slow networks. "+
 		"Set to 0 to disable request buffering. See https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering")
 	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size to buffer in memory for potential retries at other backends. "+
 		"Request bodies larger than this size cannot be retried if the backend fails. Zero or negative value disables request body buffering and retries. "+
@@ -173,7 +174,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		// Process requests for unauthorized users
 		ui := authConfig.Load().UnauthorizedUser
 		if ui != nil {
-			processUserRequest(w, r, ui)
+			processUserRequest(w, r, ui, nil)
 			return true
 		}
 
@@ -181,29 +182,36 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 
-	ui := getUserInfoByAuthTokens(ats)
-	if ui == nil {
-		uu := authConfig.Load().UnauthorizedUser
-		if uu != nil {
-			processUserRequest(w, r, uu)
-			return true
-		}
-
-		invalidAuthTokenRequests.Inc()
-		if *logInvalidAuthTokens {
-			err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
-			err = &httpserver.ErrorWithStatusCode{
-				Err:        err,
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
-		} else {
-			http.Error(w, "Unauthorized", http.StatusUnauthorized)
+	if ui := getUserInfoByAuthTokens(ats); ui != nil {
+		processUserRequest(w, r, ui, nil)
+		return true
+	}
+	if ui, tkn := getJWTUserInfo(ats); ui != nil {
+		if tkn == nil {
+			logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
 		}
+		defer putToken(tkn)
+		processUserRequest(w, r, ui, tkn)
 		return true
 	}
 
-	processUserRequest(w, r, ui)
+	uu := authConfig.Load().UnauthorizedUser
+	if uu != nil {
+		processUserRequest(w, r, uu, nil)
+		return true
+	}
+
+	invalidAuthTokenRequests.Inc()
+	if *logInvalidAuthTokens {
+		err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
+		err = &httpserver.ErrorWithStatusCode{
+			Err:        err,
+			StatusCode: http.StatusUnauthorized,
+		}
+		httpserver.Errorf(w, r, "%s", err)
+	} else {
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+	}
 	return true
 }
 
@@ -218,7 +226,37 @@ func getUserInfoByAuthTokens(ats []string) *UserInfo {
 	return nil
 }
 
-func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
+// responseWriterWithStatus is a wrapper around http.ResponseWriter that captures the status code written to the response.
+type responseWriterWithStatus struct {
+	http.ResponseWriter
+	status int
+}
+
+// WriteHeader records the status so it can be easily retrieved later
+func (rws *responseWriterWithStatus) WriteHeader(status int) {
+	rws.status = status
+	rws.ResponseWriter.WriteHeader(status)
+}
+
+// Flush implements net/http.Flusher interface
+//
+// This is needed for the copyStreamToClient()
+func (rws *responseWriterWithStatus) Flush() {
+	flusher, ok := rws.ResponseWriter.(http.Flusher)
+	if !ok {
+		logger.Panicf("BUG: it is expected http.ResponseWriter (%T) supports http.Flusher interface", rws.ResponseWriter)
+	}
+	flusher.Flush()
+}
+
+// Unwrap returns the original ResponseWriter wrapped by rws.
+//
+// This is needed for the net/http.ResponseController - see https://pkg.go.dev/net/http#NewResponseController
+func (rws *responseWriterWithStatus) Unwrap() http.ResponseWriter {
+	return rws.ResponseWriter
+}
+
+func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
 	startTime := time.Now()
 	defer ui.requestsDuration.UpdateDuration(startTime)
 
@@ -227,6 +265,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	ctx, cancel := context.WithTimeout(r.Context(), *maxQueueDuration)
 	defer cancel()
 
+	userName := ui.name()
+	if userName == "" {
+		userName = "unauthorized"
+	}
+
+	if ui.AccessLog != nil {
+		w = &responseWriterWithStatus{ResponseWriter: w}
+		defer func() {
+			rws := w.(*responseWriterWithStatus)
+			duration := time.Since(startTime)
+			ui.logRequest(r, userName, rws.status, duration)
+		}()
+	}
+
 	// Acquire global concurrency limit.
 	if err := beginConcurrencyLimit(ctx); err != nil {
 		handleConcurrencyLimitError(w, r, err)
@@ -245,10 +297,6 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	}
 
 	// Read the initial chunk for the request body.
-	userName := ui.name()
-	if userName == "" {
-		userName = "unauthorized"
-	}
 	bb, err := bufferRequestBody(ctx, r.Body, userName)
 	if err != nil {
 		httpserver.Errorf(w, r, "%s", err)
@@ -269,7 +317,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	defer ui.endConcurrencyLimit()
 
 	// Process the request.
-	processRequest(w, r, ui)
+	processRequest(w, r, ui, tkn)
 }
 
 func beginConcurrencyLimit(ctx context.Context) error {
@@ -342,7 +390,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
 	return bb, nil
 }
 
-func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
+func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
 	u := normalizeURL(r.URL)
 	up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
 	isDefault := false
@@ -368,22 +416,27 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	}
 
 	maxAttempts := up.getBackendsCount()
-	for i := 0; i < maxAttempts; i++ {
+	for range maxAttempts {
 		bu := up.getBackendURL()
 		if bu == nil {
 			break
 		}
 		targetURL := bu.url
+		if tkn != nil {
+			// for security reasons allow templating only for configured url values and headers
+			targetURL, hc = replaceJWTPlaceholders(bu, hc, tkn.VMAccess())
+		}
 		if isDefault {
 			// Don't change path and add request_path query param for default route.
+			targetURLCopy := *targetURL
 			query := targetURL.Query()
 			query.Set("request_path", u.String())
-			targetURL.RawQuery = query.Encode()
+			targetURLCopy.RawQuery = query.Encode()
+			targetURL = &targetURLCopy
 		} else {
 			// Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts, up.mergeQueryArgs)
 		}
-
 		wasLocalRetry := false
 	again:
 		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui, bu)
@@ -401,7 +454,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		ui.backendErrors.Inc()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
+		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
 		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
@@ -710,7 +763,7 @@ var concurrentRequestsLimitReached = metrics.NewCounter("vmauth_concurrent_reque
 
 func usage() {
 	const s = `
-vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.
+vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics components or any other HTTP backends.
 
 See the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/ .
 `

app/vmauth/main_timing_test.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func BenchmarkJWTRequestHandler(b *testing.B) {
+	// Generate RSA key pair for testing
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		b.Fatalf("cannot generate RSA key: %s", err)
+	}
+
+	// Generate public key PEM
+	publicKeyBytes, err := x509.MarshalPKIXPublicKey(&privateKey.PublicKey)
+	if err != nil {
+		b.Fatalf("cannot marshal public key: %s", err)
+	}
+	publicKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:  "PUBLIC KEY",
+		Bytes: publicKeyBytes,
+	})
+
+	genToken := func(t *testing.B, body map[string]any, valid bool) string {
+		t.Helper()
+
+		headerJSON, err := json.Marshal(map[string]any{
+			"alg": "RS256",
+			"typ": "JWT",
+		})
+		if err != nil {
+			t.Fatalf("cannot marshal header: %s", err)
+		}
+		headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
+
+		bodyJSON, err := json.Marshal(body)
+		if err != nil {
+			t.Fatalf("cannot marshal body: %s", err)
+		}
+		bodyB64 := base64.RawURLEncoding.EncodeToString(bodyJSON)
+
+		payload := headerB64 + "." + bodyB64
+
+		var signatureB64 string
+		if valid {
+			// Create real RSA signature
+			hash := crypto.SHA256
+			h := hash.New()
+			h.Write([]byte(payload))
+			digest := h.Sum(nil)
+
+			signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, hash, digest)
+			if err != nil {
+				t.Fatalf("cannot sign token: %s", err)
+			}
+			signatureB64 = base64.RawURLEncoding.EncodeToString(signature)
+		} else {
+			signatureB64 = base64.RawURLEncoding.EncodeToString([]byte("invalid_signature"))
+		}
+
+		return payload + "." + signatureB64
+	}
+
+	f := func(name string, cfgStr string, r *http.Request, statusCodeExpected int) {
+		b.Helper()
+
+		b.ReportAllocs()
+		b.ResetTimer()
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+			if _, err := w.Write([]byte("path: " + r.URL.Path + "\n")); err != nil {
+				panic(fmt.Errorf("cannot write response: %w", err))
+			}
+		}))
+		defer ts.Close()
+
+		cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			b.Fatalf("cannot load config data: %s", err)
+		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				b.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		b.Run(name, func(b *testing.B) {
+			b.ResetTimer()
+			b.ReportAllocs()
+			b.RunParallel(func(pb *testing.PB) {
+				w := &fakeResponseWriter{}
+				for pb.Next() {
+					w.reset()
+					if !requestHandlerWithInternalRoutes(w, r) {
+						b.Fatalf("unexpected false is returned from requestHandler")
+					}
+					if w.statusCode != statusCodeExpected {
+						b.Fatalf("unexpected response code (-%d;+%d)", statusCodeExpected, w.statusCode)
+					}
+
+				}
+			})
+		})
+	}
+
+	simpleCfgStr := fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+  url_prefix: {BACKEND}/foo`, string(publicKeyPEM))
+	noVMAccessClaimToken := genToken(b, nil, true)
+	expiredToken := genToken(b, map[string]any{
+		"exp":       10,
+		"vm_access": map[string]any{},
+	}, true)
+
+	fullToken := genToken(b, map[string]any{
+		"exp":   time.Now().Add(10 * time.Minute).Unix(),
+		"scope": "email id",
+		"vm_access": map[string]any{
+			"extra_labels": map[string]string{
+				"label":  "value1",
+				"label2": "value3",
+			},
+			"extra_filters":      []string{"stream_filter1", "stream_filter2"},
+			"metrics_account_id": 123,
+			"metrics_project_id": 234,
+			"metrics_extra_labels": []string{
+				"label1=value1",
+				"label2=value2",
+			},
+			"metrics_extra_filters": []string{
+				`{label3="value3"}`,
+				`{label4="value4"}`,
+			},
+			"logs_account_id": 345,
+			"logs_project_id": 456,
+			"logs_extra_filters": []string{
+				`{"namespace":"my-app","env":"prod"}`,
+			},
+			"logs_extra_stream_filters": []string{
+				`{"team":"dev"}`,
+			},
+		},
+	}, true)
+
+	// tenant headers are overwritten if set as placeholders
+	// extra_filters extra_stream_filters from vm_access claim merged with statically defined
+	request := httptest.NewRequest(`GET`, "http://some-host.com/query", nil)
+	request.Header.Set(`Authorization`, `Bearer `+fullToken)
+	f("full_template",
+		fmt.Sprintf(`
+users:
+- jwt:
+    public_keys:
+    - %q
+  headers:
+    - "AccountID: {{.LogsAccountID}}"
+    - "ProjectID: {{.LogsProjectID}}"
+  url_prefix: {BACKEND}/select/logsql/?extra_filters=aStaticFilter&extra_stream_filters=aStaticStreamFilter&extra_filters={{.LogsExtraFilters}}&extra_stream_filters={{.LogsExtraStreamFilters}}`, string(publicKeyPEM)),
+		request,
+		http.StatusOK,
+	)
+
+	// token without vm_access claim
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+noVMAccessClaimToken)
+	f("token_without_claim", simpleCfgStr, request, http.StatusUnauthorized)
+
+	// expired token
+	request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
+	request.Header.Set(`Authorization`, `Bearer `+expiredToken)
+	f("expired_token", simpleCfgStr, request, http.StatusUnauthorized)
+}

app/vmauth/oidc.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/jwt"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
+)
+
+type oidcConfig struct {
+	Issuer string `yaml:"issuer"`
+}
+
+type oidcDiscovererPool struct {
+	ds map[string]*oidcDiscoverer
+
+	context context.Context
+	cancel  func()
+	wg      *sync.WaitGroup
+}
+
+func (dp *oidcDiscovererPool) createOrAdd(issuer string, vp *atomic.Pointer[jwt.VerifierPool]) {
+	if dp.ds == nil {
+		dp.ds = make(map[string]*oidcDiscoverer)
+		dp.context, dp.cancel = context.WithCancel(context.Background())
+		dp.wg = &sync.WaitGroup{}
+	}
+
+	ds, found := dp.ds[issuer]
+	if !found {
+		ds = &oidcDiscoverer{
+			issuer: issuer,
+		}
+		dp.ds[issuer] = ds
+	}
+
+	ds.vps = append(ds.vps, vp)
+}
+
+func (dp *oidcDiscovererPool) startDiscovery() {
+	if len(dp.ds) == 0 {
+		return
+	}
+
+	for _, d := range dp.ds {
+		dp.wg.Go(func() {
+			if err := d.refreshVerifierPools(dp.context); err != nil {
+				logger.Errorf("failed to initialize OIDC verifier pool at start for issuer %q: %s", d.issuer, err)
+			}
+		})
+	}
+	dp.wg.Wait()
+
+	for _, d := range dp.ds {
+		dp.wg.Go(func() {
+			d.run(dp.context)
+		})
+	}
+}
+
+func (dp *oidcDiscovererPool) stopDiscovery() {
+	if len(dp.ds) == 0 {
+		return
+	}
+
+	dp.cancel()
+	dp.wg.Wait()
+}
+
+type oidcDiscoverer struct {
+	issuer string
+	vps    []*atomic.Pointer[jwt.VerifierPool]
+}
+
+func (d *oidcDiscoverer) run(ctx context.Context) {
+	t := time.NewTimer(timeutil.AddJitterToDuration(time.Second * 10))
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			if err := d.refreshVerifierPools(ctx); errors.Is(err, context.Canceled) {
+				return
+			} else if err != nil {
+				t.Reset(timeutil.AddJitterToDuration(time.Second * 10))
+				logger.Errorf("failed to refresh OIDC verifier pool for issuer %q: %v", d.issuer, err)
+				continue
+			}
+			// OIDC may return Cache-Control header with max-age directive.
+			// It could be used as time range for next refresh.
+			// https://openid.net/specs/openid-connect-core-1_0.html#RotateEncKeys
+			t.Reset(timeutil.AddJitterToDuration(time.Minute * 5))
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (d *oidcDiscoverer) refreshVerifierPools(ctx context.Context) error {
+	cfg, err := getOpenIDConfiguration(ctx, d.issuer)
+	if err != nil {
+		return err
+	}
+	// The issuer in the OIDC configuration must match the expected issuer.
+	// https://openid.net/specs/openid-connect-core-1_0.html#RotateEncKeys
+	if cfg.Issuer != d.issuer {
+		return fmt.Errorf("openid configuration issuer %q does not match expected issuer %q", cfg.Issuer, d.issuer)
+	}
+
+	verifierPool, err := fetchAndParseJWKs(ctx, cfg.JWKsURI)
+	if err != nil {
+		return err
+	}
+
+	for _, vp := range d.vps {
+		vp.Store(verifierPool)
+	}
+	return nil
+}
+
+// See https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata for details.
+type openidConfig struct {
+	Issuer  string `json:"issuer"`
+	JWKsURI string `json:"jwks_uri"`
+}
+
+var oidcHTTPClient = &http.Client{
+	Timeout: time.Second * 5,
+}
+
+func fetchAndParseJWKs(ctx context.Context, jwksURI string) (*jwt.VerifierPool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, jwksURI, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request for fetching jwks keys from %q: %w", jwksURI, err)
+	}
+
+	resp, err := oidcHTTPClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch jwks keys from %q: %w", jwksURI, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code %d when fetching jwks keys from %q", resp.StatusCode, jwksURI)
+	}
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body from %q: %w", jwksURI, err)
+	}
+
+	vp, err := jwt.ParseJWKs(b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse jwks keys from %q: %v", jwksURI, err)
+	}
+
+	return vp, nil
+}
+
+func getOpenIDConfiguration(ctx context.Context, issuer string) (openidConfig, error) {
+	issuer, _ = strings.CutSuffix(issuer, "/")
+	configURL := fmt.Sprintf("%s/.well-known/openid-configuration", issuer)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, configURL, nil)
+	if err != nil {
+		return openidConfig{}, fmt.Errorf("failed to create request for fetching openid config from %q: %w", configURL, err)
+	}
+
+	resp, err := oidcHTTPClient.Do(req)
+	if err != nil {
+		return openidConfig{}, fmt.Errorf("failed to fetch openid config from %q: %w", configURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return openidConfig{}, fmt.Errorf("unexpected status code %d when fetching openid config from %q", resp.StatusCode, configURL)
+	}
+
+	var cfg openidConfig
+	if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
+		return openidConfig{}, fmt.Errorf("failed to decode openid config from %q: %s", configURL, err)
+	}
+
+	return cfg, nil
+}

app/vmauth/target_url_test.go

@@ -174,7 +174,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		},
 		RetryStatusCodes:       []int{503, 501},
 		LoadBalancingPolicy:    "first_available",
-		DropSrcPathPrefixParts: intp(2),
+		DropSrcPathPrefixParts: new(2),
 	}, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar/federate"),
@@ -219,13 +219,13 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				},
 				RetryStatusCodes:       []int{503, 500, 501},
 				LoadBalancingPolicy:    "first_available",
-				DropSrcPathPrefixParts: intp(1),
+				DropSrcPathPrefixParts: new(1),
 			},
 			{
 				SrcPaths:               getRegexs([]string{"/api/v1/write"}),
 				URLPrefix:              mustParseURL("http://vminsert/0/prometheus"),
 				RetryStatusCodes:       []int{},
-				DropSrcPathPrefixParts: intp(0),
+				DropSrcPathPrefixParts: new(0),
 			},
 			{
 				SrcPaths:  getRegexs([]string{"/metrics"}),
@@ -242,7 +242,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 		RetryStatusCodes:       []int{502},
-		DropSrcPathPrefixParts: intp(2),
+		DropSrcPathPrefixParts: new(2),
 	}
 	f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up",
 		"xx: aa\nyy: asdf", "qwe: rty", []int{503, 500, 501}, "first_available", 1)
@@ -259,7 +259,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				SrcPaths:               getRegexs([]string{"/api/v1/write"}),
 				URLPrefix:              mustParseURL("http://vminsert/0/prometheus"),
 				RetryStatusCodes:       []int{},
-				DropSrcPathPrefixParts: intp(0),
+				DropSrcPathPrefixParts: new(0),
 			},
 			{
 				SrcPaths:  getRegexs([]string{"/metrics/a/b"}),
@@ -275,7 +275,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 		RetryStatusCodes:       []int{502},
-		DropSrcPathPrefixParts: intp(2),
+		DropSrcPathPrefixParts: new(2),
 	}
 	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
 	f(ui, "https://foo-host/metrics/a/b", "http://metrics-server/b", "", "", []int{502}, "least_loaded", 2)

app/vmbackup/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/snapshot/snapshotutil"
 )

app/vmctl/backoff/backoff.go

@@ -47,7 +47,7 @@ func New(retries int, factor float64, minDuration time.Duration) (*Backoff, erro
 // Retry process retries until all attempts are completed
 func (b *Backoff) Retry(ctx context.Context, cb retryableFunc) (uint64, error) {
 	var attempt uint64
-	for i := 0; i < b.retries; i++ {
+	for i := range b.retries {
 		err := cb()
 		if err == nil {
 			return attempt, nil

app/vmctl/flags.go

@@ -416,6 +416,16 @@ const (
 	promTemporaryDirPath = "prom-tmp-dir-path"
 )
 
+const (
+	thanosSnapshot         = "thanos-snapshot"
+	thanosConcurrency      = "thanos-concurrency"
+	thanosFilterTimeStart  = "thanos-filter-time-start"
+	thanosFilterTimeEnd    = "thanos-filter-time-end"
+	thanosFilterLabel      = "thanos-filter-label"
+	thanosFilterLabelValue = "thanos-filter-label-value"
+	thanosAggrTypes        = "thanos-aggr-types"
+)
+
 var (
 	promFlags = []cli.Flag{
 		&cli.StringFlag{
@@ -451,6 +461,43 @@ var (
 			Value: os.TempDir(),
 		},
 	}
+
+	thanosFlags = []cli.Flag{
+		&cli.StringFlag{
+			Name:     thanosSnapshot,
+			Usage:    "Path to Thanos snapshot directory containing raw and/or downsampled blocks.",
+			Required: true,
+		},
+		&cli.IntFlag{
+			Name:  thanosConcurrency,
+			Usage: "Number of concurrently running snapshot readers",
+			Value: 1,
+		},
+		&cli.StringFlag{
+			Name:  thanosFilterTimeStart,
+			Usage: "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
+		},
+		&cli.StringFlag{
+			Name:  thanosFilterTimeEnd,
+			Usage: "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
+		},
+		&cli.StringFlag{
+			Name:  thanosFilterLabel,
+			Usage: "Thanos label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
+		},
+		&cli.StringFlag{
+			Name:  thanosFilterLabelValue,
+			Usage: fmt.Sprintf("Thanos regular expression to filter label from %q flag.", thanosFilterLabel),
+			Value: ".*",
+		},
+		&cli.StringSliceFlag{
+			Name: thanosAggrTypes,
+			Usage: "Aggregate types to import from Thanos downsampled blocks. Supported values: count, sum, min, max, counter. " +
+				"Each aggregate will be imported as a separate metric with the aggregate type as suffix (e.g., metric_name:5m:count). " +
+				"If not specified, all aggregate types will be imported from downsampled blocks.",
+			Value: nil,
+		},
+	}
 )
 
 const (

app/vmctl/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/influx"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/thanos"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
@@ -285,6 +286,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to create prometheus client: %s", err)
 					}
+
 					pp := prometheusProcessor{
 						cl:        cl,
 						im:        importer,
@@ -294,6 +296,59 @@ func main() {
 					return pp.run(ctx)
 				},
 			},
+			{
+				Name:   "thanos",
+				Usage:  "Migrate time series from Thanos blocks (supports raw and downsampled data)",
+				Flags:  mergeFlags(globalFlags, thanosFlags, vmFlags),
+				Before: beforeFn,
+				Action: func(c *cli.Context) error {
+					fmt.Println("Thanos import mode")
+
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
+
+					importer, err = vm.NewImporter(ctx, vmCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create VM importer: %s", err)
+					}
+
+					thanosCfg := thanos.Config{
+						Snapshot: c.String(thanosSnapshot),
+						Filter: thanos.Filter{
+							TimeMin:    c.String(thanosFilterTimeStart),
+							TimeMax:    c.String(thanosFilterTimeEnd),
+							Label:      c.String(thanosFilterLabel),
+							LabelValue: c.String(thanosFilterLabelValue),
+						},
+					}
+					cl, err := thanos.NewClient(thanosCfg)
+					if err != nil {
+						return fmt.Errorf("failed to create thanos client: %s", err)
+					}
+
+					var aggrTypes []thanos.AggrType
+					if aggrTypesStr := c.StringSlice(thanosAggrTypes); len(aggrTypesStr) > 0 {
+						for _, typeStr := range aggrTypesStr {
+							aggrType, err := thanos.ParseAggrType(typeStr)
+							if err != nil {
+								return fmt.Errorf("failed to parse aggregate type %q: %s", typeStr, err)
+							}
+							aggrTypes = append(aggrTypes, aggrType)
+						}
+					}
+
+					tp := thanosProcessor{
+						cl:        cl,
+						im:        importer,
+						cc:        c.Int(thanosConcurrency),
+						isVerbose: c.Bool(globalVerbose),
+						aggrTypes: aggrTypes,
+					}
+					return tp.run(ctx)
+				},
+			},
 			{
 				Name:   "vm-native",
 				Usage:  "Migrate time series between VictoriaMetrics installations",

app/vmctl/opentsdb/opentsdb.go

@@ -109,7 +109,7 @@ func (c Client) FindMetrics(q string) ([]string, error) {
 		return nil, fmt.Errorf("failed to send GET request to %q: %s", q, err)
 	}
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
+		return nil, fmt.Errorf("bad return from OpenTSDB: %d: %v", resp.StatusCode, resp)
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := io.ReadAll(resp.Body)
@@ -133,7 +133,7 @@ func (c Client) FindSeries(metric string) ([]Meta, error) {
 		return nil, fmt.Errorf("failed to set GET request to %q: %s", q, err)
 	}
 	if resp.StatusCode != 200 {
-		return nil, fmt.Errorf("bad return from OpenTSDB: %q: %v", resp.StatusCode, resp)
+		return nil, fmt.Errorf("bad return from OpenTSDB: %d: %v", resp.StatusCode, resp)
 	}
 	defer func() { _ = resp.Body.Close() }()
 	body, err := io.ReadAll(resp.Body)

app/vmctl/thanos/aggr_chunk.go

@@ -0,0 +1,233 @@
+package thanos
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+)
+
+// ChunkEncAggr is the top level encoding byte for the AggrChunk.
+// It is defined by Thanos as 0xff to prevent collisions with Prometheus encodings.
+const ChunkEncAggr = chunkenc.Encoding(0xff)
+
+// AggrType represents an aggregation type in Thanos downsampled blocks.
+type AggrType uint8
+
+// AggrTypeNone indicates raw blocks with no aggregation.
+// It is used as a sentinel to distinguish raw block processing from downsampled.
+const AggrTypeNone AggrType = 255
+
+// Valid aggregation types matching Thanos definitions.
+const (
+	AggrCount AggrType = iota
+	AggrSum
+	AggrMin
+	AggrMax
+	AggrCounter
+)
+
+// AllAggrTypes contains all supported aggregation types.
+var AllAggrTypes = []AggrType{AggrCount, AggrSum, AggrMin, AggrMax, AggrCounter}
+
+func (t AggrType) String() string {
+	switch t {
+	case AggrCount:
+		return "count"
+	case AggrSum:
+		return "sum"
+	case AggrMin:
+		return "min"
+	case AggrMax:
+		return "max"
+	case AggrCounter:
+		return "counter"
+	}
+	return "<unknown>"
+}
+
+// ParseAggrType parses aggregate type from string.
+func ParseAggrType(s string) (AggrType, error) {
+	switch s {
+	case "count":
+		return AggrCount, nil
+	case "sum":
+		return AggrSum, nil
+	case "min":
+		return AggrMin, nil
+	case "max":
+		return AggrMax, nil
+	case "counter":
+		return AggrCounter, nil
+	}
+	return 0, fmt.Errorf("unknown aggregate type: %q", s)
+}
+
+// ErrAggrNotExist is returned if a requested aggregation is not present in an AggrChunk.
+var ErrAggrNotExist = errors.New("aggregate does not exist")
+
+// AggrChunk is a chunk that is composed of a set of aggregates for the same underlying data.
+// Not all aggregates must be present.
+// This is a read-only implementation for decoding Thanos downsampled blocks.
+type AggrChunk []byte
+
+// IsAggrChunk checks if the encoding byte indicates this is an AggrChunk.
+func IsAggrChunk(enc chunkenc.Encoding) bool {
+	return enc == ChunkEncAggr
+}
+
+// Get returns the sub-chunk for the given aggregate type if it exists.
+func (c AggrChunk) Get(t AggrType) (chunkenc.Chunk, error) {
+	b := c[:]
+	var x []byte
+
+	for i := AggrType(0); i <= t; i++ {
+		l, n := binary.Uvarint(b)
+		if n < 1 {
+			return nil, errors.New("invalid size: failed to read uvarint")
+		}
+		if l > uint64(len(b[n:])) || l+1 > uint64(len(b[n:])) {
+			if l > 0 {
+				return nil, errors.New("invalid size: not enough bytes")
+			}
+		}
+		b = b[n:]
+		// If length is set to zero explicitly, that means the aggregate is unset.
+		if l == 0 {
+			if i == t {
+				return nil, ErrAggrNotExist
+			}
+			continue
+		}
+		chunkLen := int(l) + 1
+		x = b[:chunkLen]
+		b = b[chunkLen:]
+	}
+	if len(x) == 0 {
+		return nil, ErrAggrNotExist
+	}
+	return chunkenc.FromData(chunkenc.Encoding(x[0]), x[1:])
+}
+
+// Encoding returns the encoding type for AggrChunk.
+func (c AggrChunk) Encoding() chunkenc.Encoding {
+	return ChunkEncAggr
+}
+
+// errIterator wraps a nop iterator but reports an error via Err().
+// It embeds chunkenc.Iterator to inherit all methods (including Seek)
+// which avoids go vet stdmethods warning about Seek signature.
+type errIterator struct {
+	chunkenc.Iterator
+	err error
+}
+
+// Err returns the underlying error.
+func (it *errIterator) Err() error {
+	return it.err
+}
+
+// newAggrChunkIterator creates a new iterator for the specified aggregate type.
+// If the aggregate is not present in the chunk (ErrAggrNotExist), a nop iterator
+// is returned without error — the caller will simply see zero samples.
+// Real decoding/corruption errors are reported via the iterator's Err() method.
+func newAggrChunkIterator(data []byte, aggrType AggrType) chunkenc.Iterator {
+	chunk := AggrChunk(data)
+	subChunk, err := chunk.Get(aggrType)
+	if err != nil {
+		if errors.Is(err, ErrAggrNotExist) {
+			return chunkenc.NewNopIterator()
+		}
+		return &errIterator{
+			Iterator: chunkenc.NewNopIterator(),
+			err:      err,
+		}
+	}
+	return subChunk.Iterator(nil)
+}
+
+// AggrChunkWrapper wraps AggrChunk to implement chunkenc.Chunk interface.
+// It delegates iteration to a specific aggregate type.
+type AggrChunkWrapper struct {
+	data     []byte
+	aggrType AggrType
+}
+
+// NewAggrChunkWrapper creates a new AggrChunk wrapper for the specified aggregate type.
+func NewAggrChunkWrapper(data []byte, aggrType AggrType) *AggrChunkWrapper {
+	return &AggrChunkWrapper{
+		data:     data,
+		aggrType: aggrType,
+	}
+}
+
+// Bytes returns the underlying byte slice.
+func (c *AggrChunkWrapper) Bytes() []byte {
+	return c.data
+}
+
+// Encoding returns the AggrChunk encoding.
+func (c *AggrChunkWrapper) Encoding() chunkenc.Encoding {
+	return ChunkEncAggr
+}
+
+// Appender returns an error since AggrChunk is read-only.
+func (c *AggrChunkWrapper) Appender() (chunkenc.Appender, error) {
+	return nil, errors.New("AggrChunk is read-only")
+}
+
+// Iterator returns an iterator for the specified aggregate type.
+func (c *AggrChunkWrapper) Iterator(it chunkenc.Iterator) chunkenc.Iterator {
+	return newAggrChunkIterator(c.data, c.aggrType)
+}
+
+// NumSamples returns the number of samples in the aggregate.
+func (c *AggrChunkWrapper) NumSamples() int {
+	chunk := AggrChunk(c.data)
+	subChunk, err := chunk.Get(c.aggrType)
+	if err != nil {
+		return 0
+	}
+	return subChunk.NumSamples()
+}
+
+// Compact is a no-op for read-only AggrChunk.
+func (c *AggrChunkWrapper) Compact() {}
+
+// Reset resets the chunk with new data.
+func (c *AggrChunkWrapper) Reset(stream []byte) {
+	c.data = stream
+}
+
+// AggrChunkPool is a custom Pool that understands AggrChunk encoding (0xff).
+// It delegates standard encodings to the default pool and handles AggrChunk specially.
+type AggrChunkPool struct {
+	defaultPool chunkenc.Pool
+	aggrType    AggrType
+}
+
+// NewAggrChunkPool creates a new pool that handles AggrChunk encoding.
+func NewAggrChunkPool(aggrType AggrType) *AggrChunkPool {
+	return &AggrChunkPool{
+		defaultPool: chunkenc.NewPool(),
+		aggrType:    aggrType,
+	}
+}
+
+// Get returns a chunk for the given encoding and data.
+func (p *AggrChunkPool) Get(e chunkenc.Encoding, b []byte) (chunkenc.Chunk, error) {
+	if e == ChunkEncAggr {
+		return NewAggrChunkWrapper(b, p.aggrType), nil
+	}
+	return p.defaultPool.Get(e, b)
+}
+
+// Put returns a chunk to the pool.
+func (p *AggrChunkPool) Put(c chunkenc.Chunk) error {
+	if c.Encoding() == ChunkEncAggr {
+		// AggrChunk wrappers are not pooled
+		return nil
+	}
+	return p.defaultPool.Put(c)
+}

app/vmctl/thanos/block_meta.go

@@ -0,0 +1,110 @@
+package thanos
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+)
+
+// BlockMeta extends Prometheus BlockMeta with Thanos-specific fields.
+type BlockMeta struct {
+	// Thanos-specific metadata
+	Thanos ThanosMeta `json:"thanos,omitempty"`
+}
+
+// ThanosMeta contains Thanos-specific block metadata.
+type ThanosMeta struct {
+	// Labels are external labels identifying the producer.
+	Labels map[string]string `json:"labels,omitempty"`
+
+	// Downsample contains downsampling information.
+	Downsample ThanosDownsample `json:"downsample,omitempty"`
+
+	// Source indicates where the block came from.
+	Source string `json:"source,omitempty"`
+
+	// SegmentFiles contains list of segment files in the block.
+	SegmentFiles []string `json:"segment_files,omitempty"`
+
+	// Files contains metadata about files in the block.
+	Files []ThanosFile `json:"files,omitempty"`
+}
+
+// ThanosDownsample contains downsampling resolution info.
+type ThanosDownsample struct {
+	// Resolution is the downsampling resolution in milliseconds.
+	// 0 means raw data (no downsampling).
+	// 300000 (5 minutes) or 3600000 (1 hour) for downsampled data.
+	Resolution int64 `json:"resolution"`
+}
+
+// ThanosFile contains metadata about a file in the block.
+type ThanosFile struct {
+	RelPath   string `json:"rel_path"`
+	SizeBytes int64  `json:"size_bytes,omitempty"`
+}
+
+// ResolutionLevel represents the downsampling resolution.
+type ResolutionLevel int64
+
+const (
+	// ResolutionRaw is for raw, non-downsampled data.
+	ResolutionRaw ResolutionLevel = 0
+	// Resolution5m is for 5-minute downsampled data (300000 ms).
+	Resolution5m ResolutionLevel = 300000
+	// Resolution1h is for 1-hour downsampled data (3600000 ms).
+	Resolution1h ResolutionLevel = 3600000
+)
+
+// String returns human-readable resolution string.
+func (r ResolutionLevel) String() string {
+	switch r {
+	case ResolutionRaw:
+		return "raw"
+	case Resolution5m:
+		return "5m"
+	case Resolution1h:
+		return "1h"
+	default:
+		return "unknown"
+	}
+}
+
+// ReadBlockMeta reads Thanos-extended block metadata from meta.json.
+func ReadBlockMeta(blockDir string) (*BlockMeta, error) {
+	metaPath := filepath.Join(blockDir, "meta.json")
+	data, err := os.ReadFile(metaPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var meta BlockMeta
+	if err := json.Unmarshal(data, &meta); err != nil {
+		return nil, err
+	}
+
+	return &meta, nil
+}
+
+// IsDownsampled returns true if the block contains downsampled data.
+func (m *BlockMeta) IsDownsampled() bool {
+	return m.Thanos.Downsample.Resolution > 0
+}
+
+// Resolution returns the block's downsampling resolution.
+func (m *BlockMeta) Resolution() ResolutionLevel {
+	return ResolutionLevel(m.Thanos.Downsample.Resolution)
+}
+
+// ResolutionSuffix returns a suffix string for metric names based on resolution.
+// For example: ":5m" or ":1h" for downsampled data, empty for raw data.
+func (m *BlockMeta) ResolutionSuffix() string {
+	switch m.Resolution() {
+	case Resolution5m:
+		return ":5m"
+	case Resolution1h:
+		return ":1h"
+	default:
+		return ""
+	}
+}

app/vmctl/thanos/block_reader.go

@@ -0,0 +1,83 @@
+package thanos
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/prometheus/prometheus/tsdb"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+)
+
+// BlockInfo contains information about a block including Thanos metadata.
+type BlockInfo struct {
+	Block      tsdb.BlockReader
+	Resolution ResolutionLevel
+	IsThanos   bool
+	// Closer releases the block's resources (file descriptors, mmap).
+	// Must be called only after all queriers on this block have been closed.
+	Closer io.Closer
+}
+
+// OpenBlocksWithInfo opens all blocks and returns them with their metadata.
+// snapshotDir must be a snapshot directory containing block directories.
+func OpenBlocksWithInfo(snapshotDir string, aggrType AggrType) ([]BlockInfo, error) {
+	entries, err := os.ReadDir(snapshotDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read snapshot directory: %w", err)
+	}
+
+	var blocks []BlockInfo
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		blockDir := filepath.Join(snapshotDir, entry.Name())
+		metaPath := filepath.Join(blockDir, "meta.json")
+
+		// Check if this is a valid block directory (has meta.json)
+		if _, err := os.Stat(metaPath); os.IsNotExist(err) {
+			continue
+		}
+
+		meta, err := ReadBlockMeta(blockDir)
+		if err != nil {
+			CloseBlocks(blocks)
+			return nil, fmt.Errorf("failed to read Thanos metadata for block %s: %w", blockDir, err)
+		}
+
+		var pool chunkenc.Pool
+		if meta.IsDownsampled() {
+			// Use AggrChunkPool for downsampled blocks
+			pool = NewAggrChunkPool(aggrType)
+		}
+
+		block, err := tsdb.OpenBlock(nil, blockDir, pool, nil)
+		if err != nil {
+			// Close previously opened blocks before returning error
+			CloseBlocks(blocks)
+			return nil, fmt.Errorf("failed to open block %s: %w", blockDir, err)
+		}
+
+		blocks = append(blocks, BlockInfo{
+			Block:      block,
+			Resolution: meta.Resolution(),
+			IsThanos:   true,
+			Closer:     block,
+		})
+	}
+
+	return blocks, nil
+}
+
+// CloseBlocks closes all blocks in the slice.
+// Must be called only after all queriers on these blocks have been closed.
+func CloseBlocks(blocks []BlockInfo) {
+	for _, bi := range blocks {
+		if bi.Closer != nil {
+			_ = bi.Closer.Close()
+		}
+	}
+}

app/vmctl/thanos/client.go

@@ -0,0 +1,198 @@
+package thanos
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb"
+)
+
+// Config contains parameters for reading Thanos snapshots.
+type Config struct {
+	Snapshot string
+	Filter   Filter
+}
+
+// Filter contains configuration for filtering the timeseries.
+type Filter struct {
+	TimeMin    string
+	TimeMax    string
+	Label      string
+	LabelValue string
+}
+
+// Client reads Thanos snapshot blocks, including downsampled blocks with AggrChunk encoding.
+type Client struct {
+	snapshotPath string
+	filter       filter
+	statsPrinted bool
+}
+
+type filter struct {
+	min, max   int64
+	label      string
+	labelValue string
+}
+
+func (f filter) inRange(minV, maxV int64) bool {
+	fmin, fmax := f.min, f.max
+	if fmin == 0 {
+		fmin = minV
+	}
+	if fmax == 0 {
+		fmax = maxV
+	}
+	return minV <= fmax && fmin <= maxV
+}
+
+// NewClient creates a new Thanos snapshot client.
+func NewClient(cfg Config) (*Client, error) {
+	minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse time in filter: %s", err)
+	}
+	return &Client{
+		snapshotPath: cfg.Snapshot,
+		filter: filter{
+			min:        minTime,
+			max:        maxTime,
+			label:      cfg.Filter.Label,
+			labelValue: cfg.Filter.LabelValue,
+		},
+	}, nil
+}
+
+// Explore fetches all available blocks from the snapshot with support for
+// Thanos AggrChunk (downsampled blocks). It opens blocks with a custom pool
+// that can decode AggrChunk encoding (0xff).
+func (c *Client) Explore(aggrType AggrType) ([]BlockInfo, error) {
+	blockInfos, err := OpenBlocksWithInfo(c.snapshotPath, aggrType)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open blocks: %w", err)
+	}
+
+	s := &Stats{
+		Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
+		Blocks:   len(blockInfos),
+	}
+
+	var blocksToImport []BlockInfo
+	for _, bi := range blockInfos {
+		meta := bi.Block.Meta()
+
+		if s.MinTime == 0 || meta.MinTime < s.MinTime {
+			s.MinTime = meta.MinTime
+		}
+		if s.MaxTime == 0 || meta.MaxTime > s.MaxTime {
+			s.MaxTime = meta.MaxTime
+		}
+
+		if !c.filter.inRange(meta.MinTime, meta.MaxTime) {
+			s.SkippedBlocks++
+			if bi.Closer != nil {
+				_ = bi.Closer.Close()
+			}
+			continue
+		}
+
+		s.Samples += meta.Stats.NumSamples
+		s.Series += meta.Stats.NumSeries
+		blocksToImport = append(blocksToImport, bi)
+	}
+	if !c.statsPrinted {
+		fmt.Println(s)
+		c.statsPrinted = true
+	}
+	return blocksToImport, nil
+}
+
+// querierSeriesSet wraps a SeriesSet and its underlying Querier, ensuring
+// the querier is closed once the SeriesSet has been fully consumed.
+// This releases the querier's read reference on the block, which is required
+// for Block.Close() to complete without hanging.
+type querierSeriesSet struct {
+	storage.SeriesSet
+	q      storage.Querier
+	closed bool
+}
+
+// Next advances the iterator. When the underlying SeriesSet is exhausted,
+// it closes the querier to release resources.
+func (s *querierSeriesSet) Next() bool {
+	if s.SeriesSet.Next() {
+		return true
+	}
+	if !s.closed {
+		_ = s.q.Close()
+		s.closed = true
+	}
+	return false
+}
+
+// Close explicitly closes the underlying querier.
+// This must be called if iteration is stopped early (before Next returns false)
+// to release block read references and prevent Block.Close() from hanging.
+func (s *querierSeriesSet) Close() {
+	if !s.closed {
+		_ = s.q.Close()
+		s.closed = true
+	}
+}
+
+// ClosableSeriesSet extends storage.SeriesSet with a Close method for explicit cleanup.
+type ClosableSeriesSet interface {
+	storage.SeriesSet
+	Close()
+}
+
+// Read reads the given BlockInfo according to configured time and label filters.
+// The returned ClosableSeriesSet automatically closes the underlying querier when fully consumed,
+// but Close() should be called explicitly (e.g., via defer) to handle early returns.
+func (c *Client) Read(bi BlockInfo) (ClosableSeriesSet, error) {
+	minTime, maxTime := bi.Block.Meta().MinTime, bi.Block.Meta().MaxTime
+	if c.filter.min != 0 {
+		minTime = c.filter.min
+	}
+	if c.filter.max != 0 {
+		maxTime = c.filter.max
+	}
+	q, err := tsdb.NewBlockQuerier(bi.Block, minTime, maxTime)
+	if err != nil {
+		return nil, err
+	}
+	ss := q.Select(
+		context.Background(),
+		false,
+		nil,
+		labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue),
+	)
+	return &querierSeriesSet{
+		SeriesSet: ss,
+		q:         q,
+	}, nil
+}
+
+func parseTime(start, end string) (int64, int64, error) {
+	var s, e int64
+	if start == "" && end == "" {
+		return 0, 0, nil
+	}
+	if start != "" {
+		v, err := time.Parse(time.RFC3339, start)
+		if err != nil {
+			return 0, 0, fmt.Errorf("failed to parse %q: %s", start, err)
+		}
+		s = v.UnixNano() / int64(time.Millisecond)
+	}
+	if end != "" {
+		v, err := time.Parse(time.RFC3339, end)
+		if err != nil {
+			return 0, 0, fmt.Errorf("failed to parse %q: %s", end, err)
+		}
+		e = v.UnixNano() / int64(time.Millisecond)
+	}
+	return s, e, nil
+}

app/vmctl/thanos/stats.go

@@ -0,0 +1,38 @@
+package thanos
+
+import (
+	"fmt"
+	"time"
+)
+
+// Stats represents data migration stats for Thanos blocks.
+type Stats struct {
+	Filtered      bool
+	MinTime       int64
+	MaxTime       int64
+	Samples       uint64
+	Series        uint64
+	Blocks        int
+	SkippedBlocks int
+}
+
+// String returns string representation for s.
+func (s Stats) String() string {
+	str := fmt.Sprintf("Thanos snapshot stats:\n"+
+		"  blocks found: %d;\n"+
+		"  blocks skipped by time filter: %d;\n"+
+		"  min time: %d (%v);\n"+
+		"  max time: %d (%v);\n"+
+		"  samples: %d;\n"+
+		"  series: %d.",
+		s.Blocks, s.SkippedBlocks,
+		s.MinTime, time.Unix(s.MinTime/1e3, 0).Format(time.RFC3339),
+		s.MaxTime, time.Unix(s.MaxTime/1e3, 0).Format(time.RFC3339),
+		s.Samples, s.Series)
+
+	if s.Filtered {
+		str += "\n* Stats numbers are based on blocks meta info and don't account for applied filters."
+	}
+
+	return str
+}

app/vmctl/thanos_processor.go

@@ -0,0 +1,309 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+	"sync"
+
+	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/metrics"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/thanos"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+)
+
+type thanosProcessor struct {
+	cl *thanos.Client
+	im *vm.Importer
+	cc int
+
+	isVerbose bool
+	aggrTypes []thanos.AggrType
+}
+
+func (tp *thanosProcessor) run(ctx context.Context) error {
+	if len(tp.aggrTypes) == 0 {
+		tp.aggrTypes = thanos.AllAggrTypes
+	}
+
+	log.Printf("Processing blocks with aggregate types: %v", tp.aggrTypes)
+
+	// Use the first aggregate type to explore blocks (block list is the same for all types)
+	blocks, err := tp.cl.Explore(tp.aggrTypes[0])
+	if err != nil {
+		return fmt.Errorf("explore failed: %s", err)
+	}
+	if len(blocks) < 1 {
+		return fmt.Errorf("found no blocks to import")
+	}
+
+	// Separate blocks into raw (resolution=0) and downsampled (resolution>0)
+	var rawBlocks, downsampledBlocks []thanos.BlockInfo
+	for _, block := range blocks {
+		if block.Resolution == thanos.ResolutionRaw {
+			rawBlocks = append(rawBlocks, block)
+		} else {
+			downsampledBlocks = append(downsampledBlocks, block)
+		}
+	}
+
+	log.Printf("Found %d raw blocks and %d downsampled blocks", len(rawBlocks), len(downsampledBlocks))
+
+	question := fmt.Sprintf("Found %d blocks to import (%d raw + %d downsampled with %d aggregate types). Continue?",
+		len(blocks), len(rawBlocks), len(downsampledBlocks), len(tp.aggrTypes))
+	if !prompt(ctx, question) {
+		return nil
+	}
+
+	// Calculate total number of block processing passes for the progress bar:
+	// raw blocks are processed once, downsampled blocks are processed once per aggregate type.
+	totalPasses := len(rawBlocks) + len(downsampledBlocks)*len(tp.aggrTypes)
+	thanosBlocksTotal.Add(totalPasses)
+	bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), totalPasses)
+	if err := barpool.Start(); err != nil {
+		return err
+	}
+	defer barpool.Stop()
+
+	tp.im.ResetStats()
+
+	type phaseStats struct {
+		name    string
+		series  uint64
+		samples uint64
+	}
+	var phases []phaseStats
+
+	// Process raw blocks first (no aggregate suffix)
+	if len(rawBlocks) > 0 {
+		log.Println("Processing raw blocks (resolution=0)...")
+		stats, err := tp.processBlocks(rawBlocks, thanos.AggrTypeNone, bar)
+		if err != nil {
+			return fmt.Errorf("migration failed for raw blocks: %s", err)
+		}
+		phases = append(phases, phaseStats{
+			name:    "raw",
+			series:  stats.series,
+			samples: stats.samples,
+		})
+	}
+
+	// Close blocks from the initial Explore. The querierSeriesSet wrapper
+	// has already released all querier read references, so Close won't hang.
+	thanos.CloseBlocks(blocks)
+
+	// Process downsampled blocks for each aggregate type.
+	// Each type needs its own AggrChunkPool, so we reopen blocks per type.
+	for _, aggrType := range tp.aggrTypes {
+		if len(downsampledBlocks) < 1 {
+			break
+		}
+
+		log.Printf("Processing downsampled blocks with aggregate type: %s", aggrType)
+
+		aggrBlocks, err := tp.cl.Explore(aggrType)
+		if err != nil {
+			return fmt.Errorf("explore failed for aggr type %s: %s", aggrType, err)
+		}
+
+		var downsampledOnly []thanos.BlockInfo
+		for _, block := range aggrBlocks {
+			if block.Resolution != thanos.ResolutionRaw {
+				downsampledOnly = append(downsampledOnly, block)
+			}
+		}
+
+		if len(downsampledOnly) < 1 {
+			log.Printf("No downsampled blocks found for aggregate type %s, skipping", aggrType)
+			thanos.CloseBlocks(aggrBlocks)
+			continue
+		}
+
+		log.Printf("Processing %d blocks for aggregate type: %s", len(downsampledOnly), aggrType)
+		stats, err := tp.processBlocks(downsampledOnly, aggrType, bar)
+		thanos.CloseBlocks(aggrBlocks)
+		if err != nil {
+			return fmt.Errorf("migration failed for aggr type %s: %s", aggrType, err)
+		}
+		phases = append(phases, phaseStats{
+			name:    aggrType.String(),
+			series:  stats.series,
+			samples: stats.samples,
+		})
+	}
+
+	// Print per-phase and total statistics
+	var totalSeries, totalSamples uint64
+	log.Printf("Migration statistics (%d raw blocks, %d downsampled blocks):", len(rawBlocks), len(downsampledBlocks))
+	for _, p := range phases {
+		log.Printf("  %s: %d series, %d samples", p.name, p.series, p.samples)
+		totalSeries += p.series
+		totalSamples += p.samples
+	}
+	log.Printf("  total: %d series, %d samples", totalSeries, totalSamples)
+
+	// Wait for all buffers to flush
+	tp.im.Close()
+	// Drain import errors channel
+	for vmErr := range tp.im.Errors() {
+		if vmErr.Err != nil {
+			thanosErrorsTotal.Inc()
+			return fmt.Errorf("import process failed: %s", wrapErr(vmErr, tp.isVerbose))
+		}
+	}
+
+	log.Println("Import finished!")
+	log.Println(tp.im.Stats())
+	return nil
+}
+
+// processBlocksStats holds statistics collected during block processing.
+type processBlocksStats struct {
+	blocks  uint64
+	series  uint64
+	samples uint64
+}
+
+func (tp *thanosProcessor) processBlocks(blocks []thanos.BlockInfo, aggrType thanos.AggrType, bar barpool.Bar) (processBlocksStats, error) {
+	blockReadersCh := make(chan thanos.BlockInfo)
+	errCh := make(chan error, tp.cc)
+
+	var processedBlocks, totalSeries, totalSamples uint64
+	var mu sync.Mutex
+
+	var wg sync.WaitGroup
+	for i := range tp.cc {
+		workerID := i
+		wg.Go(func() {
+			for bi := range blockReadersCh {
+				seriesCount, samplesCount, err := tp.do(bi, aggrType)
+				if err != nil {
+					thanosErrorsTotal.Inc()
+					errCh <- fmt.Errorf("read failed for block %q with aggr %s: %s", bi.Block.Meta().ULID, aggrType, err)
+					return
+				}
+
+				mu.Lock()
+				processedBlocks++
+				totalSeries += seriesCount
+				totalSamples += samplesCount
+				log.Printf("[Worker %d] Block %s: %d series, %d samples | Total: %d/%d blocks, %d series, %d samples",
+					workerID, bi.Block.Meta().ULID.String()[:8], seriesCount, samplesCount,
+					processedBlocks, len(blocks), totalSeries, totalSamples)
+				mu.Unlock()
+
+				thanosBlocksProcessed.Inc()
+				bar.Increment()
+			}
+		})
+	}
+
+	// any error breaks the import
+	for _, bi := range blocks {
+		select {
+		case thanosErr := <-errCh:
+			close(blockReadersCh)
+			wg.Wait()
+			return processBlocksStats{}, fmt.Errorf("thanos error: %s", thanosErr)
+		case vmErr := <-tp.im.Errors():
+			close(blockReadersCh)
+			wg.Wait()
+			thanosErrorsTotal.Inc()
+			return processBlocksStats{}, fmt.Errorf("import process failed: %s", wrapErr(vmErr, tp.isVerbose))
+		case blockReadersCh <- bi:
+		}
+	}
+
+	close(blockReadersCh)
+	wg.Wait()
+	close(errCh)
+	for err := range errCh {
+		return processBlocksStats{}, fmt.Errorf("import process failed: %s", err)
+	}
+
+	return processBlocksStats{
+		blocks:  processedBlocks,
+		series:  totalSeries,
+		samples: totalSamples,
+	}, nil
+}
+
+func (tp *thanosProcessor) do(bi thanos.BlockInfo, aggrType thanos.AggrType) (uint64, uint64, error) {
+	ss, err := tp.cl.Read(bi)
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to read block: %s", err)
+	}
+	defer ss.Close() // Ensure querier is closed even on early returns
+
+	var it chunkenc.Iterator
+	var seriesCount, samplesCount uint64
+
+	for ss.Next() {
+		var name string
+		var labelPairs []vm.LabelPair
+		series := ss.At()
+
+		series.Labels().Range(func(label labels.Label) {
+			if label.Name == "__name__" {
+				name = label.Value
+				return
+			}
+			labelPairs = append(labelPairs, vm.LabelPair{
+				Name:  strings.Clone(label.Name),
+				Value: strings.Clone(label.Value),
+			})
+		})
+		if name == "" {
+			return seriesCount, samplesCount, fmt.Errorf("failed to find `__name__` label in labelset for block %v", bi.Block.Meta().ULID)
+		}
+
+		// Add resolution and aggregate type suffix to metric name for downsampled blocks
+		if bi.Resolution != thanos.ResolutionRaw && aggrType != thanos.AggrTypeNone {
+			name = fmt.Sprintf("%s:%s:%s", name, bi.Resolution.String(), aggrType.String())
+		}
+
+		var timestamps []int64
+		var values []float64
+		it = series.Iterator(it)
+		for {
+			typ := it.Next()
+			if typ == chunkenc.ValNone {
+				break
+			}
+			if typ != chunkenc.ValFloat {
+				continue
+			}
+			t, v := it.At()
+			timestamps = append(timestamps, t)
+			values = append(values, v)
+		}
+		if err := it.Err(); err != nil {
+			return seriesCount, samplesCount, err
+		}
+
+		samplesCount += uint64(len(timestamps))
+		seriesCount++
+
+		ts := vm.TimeSeries{
+			Name:       name,
+			LabelPairs: labelPairs,
+			Timestamps: timestamps,
+			Values:     values,
+		}
+		if err := tp.im.Input(&ts); err != nil {
+			return seriesCount, samplesCount, err
+		}
+	}
+	return seriesCount, samplesCount, ss.Err()
+}
+
+var (
+	thanosBlocksTotal     = metrics.NewCounter(`vmctl_thanos_migration_blocks_total`)
+	thanosBlocksProcessed = metrics.NewCounter(`vmctl_thanos_migration_blocks_processed`)
+	thanosErrorsTotal     = metrics.NewCounter(`vmctl_thanos_migration_errors_total`)
+)

app/vmctl/vm/timeseries.go

@@ -76,11 +76,11 @@ func (ts *TimeSeries) write(w io.Writer) (int, error) {
 
 		pointsCount := len(timestampsBatch)
 		cw.printf(`},"timestamps":[`)
-		for i := 0; i < pointsCount-1; i++ {
+		for i := range pointsCount - 1 {
 			cw.printf(`%d,`, timestampsBatch[i])
 		}
 		cw.printf(`%d],"values":[`, timestampsBatch[pointsCount-1])
-		for i := 0; i < pointsCount-1; i++ {
+		for i := range pointsCount - 1 {
 			cw.printf(`%v,`, valuesBatch[i])
 		}
 		cw.printf("%v]}\n", valuesBatch[pointsCount-1])

app/vmctl/vm_native.go

@@ -262,7 +262,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 	errCh := make(chan error, p.cc)
 
 	var wg sync.WaitGroup
-	for i := 0; i < p.cc; i++ {
+	for range p.cc {
 		wg.Go(func() {
 			for f := range filterCh {
 				if !p.disablePerMetricRequests {

app/vminsert/common/streamaggr.go

@@ -55,7 +55,7 @@ var (
 	deduplicator *streamaggr.Deduplicator
 )
 
-// CheckStreamAggrConfig checks config pointed by -stramaggr.config
+// CheckStreamAggrConfig checks config pointed by -streamaggr.config
 func CheckStreamAggrConfig() error {
 	if *streamAggrConfig == "" {
 		return nil

app/vminsert/datadogsketches/request_handler.go

@@ -45,15 +45,14 @@ func insertRows(sketches []*datadogsketches.Sketch, extraLabels []prompb.Label)
 		ms := sketch.ToSummary()
 		for _, m := range ms {
 			ctx.Labels = ctx.Labels[:0]
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10557
+			ctx.AddLabel("host", sketch.Host) // newly added
 			ctx.AddLabel("", m.Name)
 			for _, label := range m.Labels {
 				ctx.AddLabel(label.Name, label.Value)
 			}
 			for _, tag := range sketch.Tags {
 				name, value := datadogutil.SplitTag(tag)
-				if name == "host" {
-					name = "exported_host"
-				}
 				ctx.AddLabel(name, value)
 			}
 			for j := range extraLabels {

app/vminsert/prompush/push.go

@@ -77,7 +77,7 @@ func push(ctx *common.InsertCtx, tss []prompb.TimeSeries) {
 			r := &ts.Samples[i]
 			metricNameRaw, err = ctx.WriteDataPointExt(metricNameRaw, ctx.Labels, r.Timestamp, r.Value)
 			if err != nil {
-				logger.Errorf("cannot write promscape data to storage: %s", err)
+				logger.Errorf("cannot write promscrape data to storage: %s", err)
 				return
 			}
 		}

app/vmrestore/main.go

@@ -30,6 +30,7 @@ var (
 	concurrency             = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce restore duration")
 	maxBytesPerSecond       = flagutil.NewBytes("maxBytesPerSecond", 0, "The maximum download speed. There is no limit if it is set to 0")
 	skipBackupCompleteCheck = flag.Bool("skipBackupCompleteCheck", false, "Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file")
+	SkipPreallocation       = flag.Bool("skipFilePreallocation", false, "Whether to skip pre-allocated files. This will likely be slower in most cases, but allows restores to resume mid file on failure")
 )
 
 func main() {
@@ -63,6 +64,7 @@ func main() {
 		Src:                     srcFS,
 		Dst:                     dstFS,
 		SkipBackupCompleteCheck: *skipBackupCompleteCheck,
+		SkipPreallocation:       *SkipPreallocation,
 	}
 	pushmetrics.Init()
 	if err := a.Run(ctx); err != nil {

app/vmselect/graphite/aggr_state.go

@@ -142,7 +142,7 @@ type aggrStatePercentile struct {
 
 func newAggrStatePercentile(pointsLen int, n float64) aggrState {
 	hs := make([]*histogram.Fast, pointsLen)
-	for i := 0; i < pointsLen; i++ {
+	for i := range pointsLen {
 		hs[i] = histogram.NewFast()
 	}
 	return &aggrStatePercentile{

app/vmselect/graphite/eval.go

@@ -50,7 +50,7 @@ func (ec *evalConfig) newTimestamps(step int64) []int64 {
 	pointsLen := ec.pointsLen(step)
 	timestamps := make([]int64, pointsLen)
 	ts := ec.startTime
-	for i := 0; i < pointsLen; i++ {
+	for i := range pointsLen {
 		timestamps[i] = ts
 		ts += step
 	}

app/vmselect/graphite/natural_compare.go

@@ -25,7 +25,7 @@ func naturalLess(a, b string) bool {
 }
 
 func getNonNumPrefix(s string) (prefix string, tail string) {
-	for i := 0; i < len(s); i++ {
+	for i := range len(s) {
 		ch := s[i]
 		if ch >= '0' && ch <= '9' {
 			return s[:i], s[i:]

app/vmselect/graphite/render_api.go

@@ -82,7 +82,7 @@ func RenderHandler(startTime time.Time, w http.ResponseWriter, r *http.Request)
 	if s := r.FormValue("maxDataPoints"); len(s) > 0 {
 		n, err := strconv.ParseFloat(s, 64)
 		if err != nil {
-			return fmt.Errorf("cannot parse maxDataPoints=%q: %w", maxDataPoints, err)
+			return fmt.Errorf("cannot parse maxDataPoints=%d: %w", maxDataPoints, err)
 		}
 		if n <= 0 {
 			return fmt.Errorf("maxDataPoints must be greater than 0; got %f", n)
@@ -209,7 +209,7 @@ func parseInterval(s string) (int64, error) {
 	s = strings.TrimSpace(s)
 	prefix := s
 	var suffix string
-	for i := 0; i < len(s); i++ {
+	for i := range len(s) {
 		ch := s[i]
 		if ch != '-' && ch != '+' && ch != '.' && (ch < '0' || ch > '9') {
 			prefix = s[:i]

app/vmselect/graphite/transform.go

@@ -1228,7 +1228,7 @@ func transformDelay(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
 				stepsLocal = len(values)
 			}
 			copy(values[stepsLocal:], values[:len(values)-stepsLocal])
-			for i := 0; i < stepsLocal; i++ {
+			for i := range stepsLocal {
 				values[i] = nan
 			}
 		}
@@ -1740,7 +1740,7 @@ func transformGroup(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, er
 
 func groupSeriesLists(ec *evalConfig, args []*graphiteql.ArgExpr, expr graphiteql.Expr) (nextSeriesFunc, error) {
 	var nextSeriess []nextSeriesFunc
-	for i := 0; i < len(args); i++ {
+	for i := range args {
 		nextSeries, err := evalSeriesList(ec, args, "seriesList", i)
 		if err != nil {
 			for _, f := range nextSeriess {
@@ -3233,7 +3233,7 @@ func transformSeriesByTag(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFu
 		return nil, fmt.Errorf("at least one tagExpression must be passed to seriesByTag")
 	}
 	var tagExpressions []string
-	for i := 0; i < len(args); i++ {
+	for i := range args {
 		te, err := getString(args, "tagExpressions", i)
 		if err != nil {
 			return nil, err
@@ -3633,7 +3633,7 @@ var graphiteToGolangRe = regexp.MustCompile(`\\(\d+)`)
 
 func getNodes(args []*graphiteql.ArgExpr) ([]graphiteql.Expr, error) {
 	var nodes []graphiteql.Expr
-	for i := 0; i < len(args); i++ {
+	for i := range args {
 		expr := args[i].Expr
 		switch expr.(type) {
 		case *graphiteql.NumberExpr, *graphiteql.StringExpr:
@@ -4052,7 +4052,7 @@ func formatPathsFromSeriesExpressions(seriesExpressions []string, sortPaths bool
 
 func newNaNSeries(ec *evalConfig, step int64) *series {
 	values := make([]float64, ec.pointsLen(step))
-	for i := 0; i < len(values); i++ {
+	for i := range values {
 		values[i] = nan
 	}
 	return &series{
@@ -5244,7 +5244,7 @@ func transformLinearRegression(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSer
 
 func linearRegressionForSeries(ec *evalConfig, fe *graphiteql.FuncExpr, ss, sourceSeries []*series) (nextSeriesFunc, error) {
 	var resp []*series
-	for i := 0; i < len(ss); i++ {
+	for i := range ss {
 		source := sourceSeries[i]
 		s := ss[i]
 		s.Tags["linearRegressions"] = fmt.Sprintf("%d, %d", ec.startTime/1e3, ec.endTime/1e3)
@@ -5258,7 +5258,7 @@ func linearRegressionForSeries(ec *evalConfig, fe *graphiteql.FuncExpr, ss, sour
 			continue
 		}
 		values := s.Values
-		for j := 0; j < len(values); j++ {
+		for j := range values {
 			values[j] = offset + (float64(int(s.Timestamps[0])+j*int(s.step)))*factor
 		}
 		resp = append(resp, s)
@@ -5370,7 +5370,7 @@ func holtWinterConfidenceBands(ec *evalConfig, fe *graphiteql.FuncExpr, args []*
 		valuesLen := len(forecastValues)
 		upperBand := make([]float64, 0, valuesLen)
 		lowerBand := make([]float64, 0, valuesLen)
-		for i := 0; i < valuesLen; i++ {
+		for i := range valuesLen {
 			forecastItem := forecastValues[i]
 			deviationItem := deviationValues[i]
 			if math.IsNaN(forecastItem) || math.IsNaN(deviationItem) {
@@ -5464,7 +5464,7 @@ func transformHoltWintersAberration(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
 			return nil, fmt.Errorf("bug, len mismatch for series: %d and upperBand values: %d or lowerBand values: %d", len(values), len(upperBand), len(lowerBand))
 		}
 		aberration := make([]float64, 0, len(values))
-		for i := 0; i < len(values); i++ {
+		for i := range values {
 			v := values[i]
 			upperValue := upperBand[i]
 			lowerValue := lowerBand[i]

app/vmselect/graphiteql/lexer.go

@@ -280,7 +280,7 @@ func isMetricExprChar(ch byte) bool {
 }
 
 func appendEscapedIdent(dst []byte, s string) []byte {
-	for i := 0; i < len(s); i++ {
+	for i := range len(s) {
 		ch := s[i]
 		if isIdentChar(ch) || isMetricExprChar(ch) {
 			if i == 0 && !isFirstIdentChar(ch) {

app/vmselect/main.go

@@ -321,19 +321,23 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/tags/tagSeries":
 		graphiteTagsTagSeriesRequests.Inc()
-		if err := graphite.TagsTagSeriesHandler(startTime, w, r); err != nil {
-			graphiteTagsTagSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
+		err := &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
+				"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
+			StatusCode: http.StatusNotImplemented,
 		}
+		graphiteTagsTagSeriesErrors.Inc()
+		httpserver.Errorf(w, r, "%s", err)
 		return true
 	case "/tags/tagMultiSeries":
 		graphiteTagsTagMultiSeriesRequests.Inc()
-		if err := graphite.TagsTagMultiSeriesHandler(startTime, w, r); err != nil {
-			graphiteTagsTagMultiSeriesErrors.Inc()
-			httpserver.Errorf(w, r, "%s", err)
-			return true
+		err := &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("graphite tag registration has been disabled and is planned to be removed in future. " +
+				"See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544"),
+			StatusCode: http.StatusNotImplemented,
 		}
+		graphiteTagsTagMultiSeriesErrors.Inc()
+		httpserver.Errorf(w, r, "%s", err)
 		return true
 	case "/tags":
 		graphiteTagsRequests.Inc()
@@ -739,6 +743,26 @@ func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request, path string) {
 	req := r.Clone(r.Context())
 	req.URL.Path = strings.TrimPrefix(path, "prometheus")
 	req.Host = vmalertProxyHost
+
+	if strings.HasPrefix(r.Header.Get(`User-Agent`), `Grafana`) {
+		// Grafana currently supports only Prometheus-style alerts. If other alert types
+		// (e.g. logs or traces) are returned, it may fail with "Error loading alerts".
+		//
+		// Grafana queries the vmalert API directly, bypassing the VictoriaMetrics datasource,
+		// so query params (such as datasource_type) cannot be enforced on the Grafana side.
+		//
+		// To ensure compatibility, we detect Grafana requests via the User-Agent and enforce
+		// `datasource_type=prometheus`.
+		//
+		// See:
+		// - https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329#issuecomment-3847585443
+		// - https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/59
+		q := req.URL.Query()
+		q.Set("datasource_type", "prometheus")
+		req.URL.RawQuery = q.Encode()
+		req.RequestURI = ""
+	}
+
 	vmalertProxy.ServeHTTP(w, req)
 }
 

app/vmselect/netstorage/netstorage.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"math"
+	"slices"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -21,6 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
 )
 
@@ -491,10 +493,7 @@ func (pts *packedTimeseries) unpackTo(dst []*sortBlock, tbf *tmpBlocksFile, tr s
 	}
 
 	// Prepare worker channels.
-	workers := min(len(upws), gomaxprocs)
-	if workers < 1 {
-		workers = 1
-	}
+	workers := max(min(len(upws), gomaxprocs), 1)
 	itemsPerWorker := (len(upws) + workers - 1) / workers
 	workChs := make([]chan *unpackWork, workers)
 	for i := range workChs {
@@ -832,12 +831,7 @@ func GraphiteTags(qt *querytracer.Tracer, filter string, limit int, deadline sea
 }
 
 func hasString(a []string, s string) bool {
-	for _, x := range a {
-		if x == s {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(a, s)
 }
 
 // LabelValues returns label values matching the given labelName and sq until the given deadline.
@@ -1369,7 +1363,7 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 const maxFastAllocBlockSize = 32 * 1024
 
 // GetMetricNamesStats returns statistic for timeseries metric names usage.
-func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
+func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (metricnamestats.StatsResult, error) {
 	qt = qt.NewChild("get metric names usage statistics with limit: %d, less or equal to: %d, match pattern=%q", limit, le, matchPattern)
 	defer qt.Done()
 	return vmstorage.GetMetricNamesStats(qt, limit, le, matchPattern)

app/vmselect/netstorage/netstorage_timing_test.go

@@ -10,14 +10,14 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
 		b.Run(fmt.Sprintf("replicationFactor-%d", replicationFactor), func(b *testing.B) {
 			const samplesPerBlock = 8192
 			var blocks []*sortBlock
-			for j := 0; j < 10; j++ {
+			for j := range 10 {
 				timestamps := make([]int64, samplesPerBlock)
 				values := make([]float64, samplesPerBlock)
 				for i := range timestamps {
 					timestamps[i] = int64(j*samplesPerBlock + i)
 					values[i] = float64(j*samplesPerBlock + i)
 				}
-				for i := 0; i < replicationFactor; i++ {
+				for range replicationFactor {
 					blocks = append(blocks, &sortBlock{
 						Timestamps: timestamps,
 						Values:     values,
@@ -30,7 +30,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
 	b.Run("overlapped-blocks-bestcase", func(b *testing.B) {
 		const samplesPerBlock = 8192
 		var blocks []*sortBlock
-		for j := 0; j < 10; j++ {
+		for j := range 10 {
 			timestamps := make([]int64, samplesPerBlock)
 			values := make([]float64, samplesPerBlock)
 			for i := range timestamps {
@@ -45,7 +45,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
 		for j := 1; j < len(blocks); j++ {
 			prev := blocks[j-1].Timestamps
 			curr := blocks[j].Timestamps
-			for i := 0; i < samplesPerBlock/2; i++ {
+			for i := range samplesPerBlock / 2 {
 				prev[i+samplesPerBlock/2], curr[i] = curr[i], prev[i+samplesPerBlock/2]
 			}
 		}
@@ -54,7 +54,7 @@ func BenchmarkMergeSortBlocks(b *testing.B) {
 	b.Run("overlapped-blocks-worstcase", func(b *testing.B) {
 		const samplesPerBlock = 8192
 		var blocks []*sortBlock
-		for j := 0; j < 5; j++ {
+		for j := range 5 {
 			timestamps := make([]int64, samplesPerBlock)
 			values := make([]float64, samplesPerBlock)
 			for i := range timestamps {

app/vmselect/prometheus/export.qtpl

@@ -11,6 +11,16 @@
 
 {% stripspace %}
 
+{% func ExportCSVHeader(fieldNames []string) %}
+	{% if len(fieldNames) == 0 %}{% return %}{% endif %}
+	{%s= fieldNames[0] %}
+	{% for _, fieldName := range fieldNames[1:] %}
+		,
+		{%s= fieldName %}
+	{% endfor %}
+	{% newline %}
+{% endfunc %}
+
 {% func ExportCSVLine(xb *exportBlock, fieldNames []string) %}
 	{% if len(xb.timestamps) == 0 || len(fieldNames) == 0 %}{% return %}{% endif %}
 	{% for i, timestamp := range xb.timestamps %}

app/vmselect/prometheus/export_test.go

@@ -0,0 +1,132 @@
+package prometheus
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+func TestExportCSVHeader(t *testing.T) {
+	f := func(fieldNames []string, expected string) {
+		t.Helper()
+		got := ExportCSVHeader(fieldNames)
+		if got != expected {
+			t.Fatalf("ExportCSVHeader(%v): got %q; want %q", fieldNames, got, expected)
+		}
+	}
+
+	f(nil, "")
+	f([]string{}, "")
+
+	f([]string{"__value__"}, "__value__\n")
+	f([]string{"__timestamp__"}, "__timestamp__\n")
+	f([]string{"__timestamp__:rfc3339"}, "__timestamp__:rfc3339\n")
+	f([]string{"__name__"}, "__name__\n")
+	f([]string{"job"}, "job\n")
+
+	f([]string{"__timestamp__:rfc3339", "__value__"}, "__timestamp__:rfc3339,__value__\n")
+	f([]string{"__value__", "__timestamp__"}, "__value__,__timestamp__\n")
+	f([]string{"job", "instance"}, "job,instance\n")
+
+	f([]string{"__name__", "__value__", "__timestamp__:unix_s"}, "__name__,__value__,__timestamp__:unix_s\n")
+	f([]string{"job", "instance", "__value__", "__timestamp__:unix_ms"}, "job,instance,__value__,__timestamp__:unix_ms\n")
+	f([]string{"__timestamp__:custom:2006-01-02", "__value__", "host", "dc", "env"},
+		"__timestamp__:custom:2006-01-02,__value__,host,dc,env\n")
+
+	// duplicate fields
+	f([]string{"__value__", "__value__"}, "__value__,__value__\n")
+	f([]string{"__timestamp__", "__timestamp__:rfc3339"}, "__timestamp__,__timestamp__:rfc3339\n")
+}
+
+func TestExportCSVLine(t *testing.T) {
+	localBak := time.Local
+	time.Local = time.UTC
+	defer func() { time.Local = localBak }()
+
+	f := func(mn *storage.MetricName, timestamps []int64, values []float64, fieldNames []string, expected string) {
+		t.Helper()
+		xb := &exportBlock{
+			mn:         mn,
+			timestamps: timestamps,
+			values:     values,
+		}
+		got := ExportCSVLine(xb, fieldNames)
+		if got != expected {
+			t.Fatalf("ExportCSVLine: got %q; want %q", got, expected)
+		}
+	}
+
+	mn := &storage.MetricName{
+		MetricGroup: []byte("cpu_usage"),
+		Tags: []storage.Tag{
+			{Key: []byte("job"), Value: []byte("node")},
+			{Key: []byte("instance"), Value: []byte("localhost:9090")},
+		},
+	}
+
+	// empty inputs
+	f(mn, nil, nil, []string{"__value__"}, "")
+	f(mn, []int64{}, []float64{}, []string{"__value__"}, "")
+	f(mn, []int64{1000}, []float64{1.5}, nil, "")
+	f(mn, []int64{1000}, []float64{1.5}, []string{}, "")
+
+	f(mn, []int64{1000}, []float64{42.5}, []string{"__value__"}, "42.5\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__"}, "1704067200000\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:unix_s"}, "1704067200\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:unix_ms"}, "1704067200000\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:unix_ns"}, "1704067200000000000\n")
+	f(mn, []int64{1704067200000}, []float64{1}, []string{"__timestamp__:rfc3339"}, "2024-01-01T00:00:00Z\n")
+
+	f(mn, []int64{1000}, []float64{1}, []string{"__name__"}, "cpu_usage\n")
+	f(mn, []int64{1000}, []float64{1}, []string{"job"}, "node\n")
+	f(mn, []int64{1000}, []float64{1}, []string{"instance"}, "localhost:9090\n")
+	f(mn, []int64{1000}, []float64{1}, []string{"missing_label"}, "\n")
+
+	// multiple fields
+	f(mn, []int64{1704067200000}, []float64{99.9},
+		[]string{"__timestamp__:unix_s", "__value__", "job"},
+		"1704067200,99.9,node\n")
+
+	// multiple rows
+	f(mn, []int64{1000, 2000}, []float64{1.1, 2.2},
+		[]string{"__value__", "__timestamp__"},
+		"1.1,1000\n2.2,2000\n")
+	f(mn, []int64{1000, 2000, 3000}, []float64{10, 20, 30},
+		[]string{"__timestamp__:unix_s", "__value__"},
+		"1,10\n2,20\n3,30\n")
+
+	// escaping for special characters in tag values
+	f(&storage.MetricName{
+		MetricGroup: []byte("m"),
+		Tags:        []storage.Tag{{Key: []byte("desc"), Value: []byte("a,b")}},
+	}, []int64{1000}, []float64{1}, []string{"desc"}, "\"a,b\"\n")
+
+	f(&storage.MetricName{
+		MetricGroup: []byte("m"),
+		Tags:        []storage.Tag{{Key: []byte("desc"), Value: []byte(`say "hello"`)}},
+	}, []int64{1000}, []float64{1}, []string{"desc"}, "\"say \\\"hello\\\"\"\n")
+
+	f(&storage.MetricName{
+		MetricGroup: []byte("m"),
+		Tags:        []storage.Tag{{Key: []byte("desc"), Value: []byte("line1\nline2")}},
+	}, []int64{1000}, []float64{1}, []string{"desc"}, "\"line1\\nline2\"\n")
+
+	// header and data line field counts must match
+	fieldNames := []string{"__name__", "job", "instance", "__value__", "__timestamp__:unix_s"}
+	header := ExportCSVHeader(fieldNames)
+	line := ExportCSVLine(&exportBlock{
+		mn:         mn,
+		timestamps: []int64{1704067200000},
+		values:     []float64{99.9},
+	}, fieldNames)
+	headerCommas := strings.Count(header, ",")
+	lineCommas := strings.Count(line, ",")
+	if headerCommas != lineCommas {
+		t.Fatalf("header has %d commas, data line has %d commas", headerCommas, lineCommas)
+	}
+	if headerCommas != len(fieldNames)-1 {
+		t.Fatalf("expected %d commas in header, got %d", len(fieldNames)-1, headerCommas)
+	}
+}

app/vmselect/prometheus/prometheus.go

@@ -6,11 +6,13 @@ import (
 	"math"
 	"net/http"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
+	"unicode/utf8"
 
 	"github.com/VictoriaMetrics/metrics"
 	"github.com/VictoriaMetrics/metricsql"
@@ -173,6 +175,7 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 	w.Header().Set("Content-Type", "text/csv; charset=utf-8")
 	bw := bufferedwriter.Get(w)
 	defer bufferedwriter.Put(bw)
+	WriteExportCSVHeader(bw, fieldNames)
 	sw := newScalableWriter(bw)
 	writeCSVLine := func(xb *exportBlock, workerID uint) error {
 		if len(xb.timestamps) == 0 {
@@ -527,6 +530,14 @@ func LabelValuesHandler(qt *querytracer.Tracer, startTime time.Time, labelName s
 		return err
 	}
 	sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxLabelsAPISeries)
+
+	if strings.HasPrefix(labelName, "U__") {
+		// This label seems to be Unicode-encoded according to the Prometheus spec.
+		// See https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values
+		// Spec: https://github.com/prometheus/proposals/blob/main/proposals/0028-utf8.md
+		labelName = unescapePrometheusLabelName(labelName)
+	}
+
 	labelValues, err := netstorage.LabelValues(qt, labelName, sq, limit, cp.deadline)
 	if err != nil {
 		return fmt.Errorf("cannot obtain values for label %q: %w", labelName, err)
@@ -1004,14 +1015,7 @@ func removeEmptyValuesAndTimeseries(tss []netstorage.Result) []netstorage.Result
 	dst := tss[:0]
 	for i := range tss {
 		ts := &tss[i]
-		hasNaNs := false
-		for _, v := range ts.Values {
-			if math.IsNaN(v) {
-				hasNaNs = true
-				break
-			}
-		}
-		if !hasNaNs {
+		if !slices.ContainsFunc(ts.Values, math.IsNaN) {
 			// Fast path: nothing to remove.
 			if len(ts.Values) > 0 {
 				dst = append(dst, *ts)
@@ -1336,3 +1340,70 @@ func calculateMaxUniqueTimeSeriesForResource(maxConcurrentRequests, remainingMem
 func GetMaxUniqueTimeSeries() int {
 	return maxUniqueTimeseriesValue
 }
+
+// copied from https://github.com/prometheus/common/blob/adea6285c1c7447fcb7bfdeb6abfc6eff893e0a7/model/metric.go#L483
+// it's not possible to use direct import due to increased binary size
+func unescapePrometheusLabelName(name string) string {
+	// lower function taken from strconv.atoi.
+	lower := func(c byte) byte {
+		return c | ('x' - 'X')
+	}
+	if len(name) == 0 {
+		return name
+	}
+	escapedName, found := strings.CutPrefix(name, "U__")
+	if !found {
+		return name
+	}
+
+	var unescaped strings.Builder
+TOP:
+	for i := 0; i < len(escapedName); i++ {
+		// All non-underscores are treated normally.
+		if escapedName[i] != '_' {
+			unescaped.WriteByte(escapedName[i])
+			continue
+		}
+		i++
+		if i >= len(escapedName) {
+			return name
+		}
+		// A double underscore is a single underscore.
+		if escapedName[i] == '_' {
+			unescaped.WriteByte('_')
+			continue
+		}
+		// We think we are in a UTF-8 code, process it.
+		var utf8Val uint
+		for j := 0; i < len(escapedName); j++ {
+			// This is too many characters for a utf8 value based on the MaxRune
+			// value of '\U0010FFFF'.
+			if j >= 6 {
+				return name
+			}
+			// Found a closing underscore, convert to a rune, check validity, and append.
+			if escapedName[i] == '_' {
+				utf8Rune := rune(utf8Val)
+				if !utf8.ValidRune(utf8Rune) {
+					return name
+				}
+				unescaped.WriteRune(utf8Rune)
+				continue TOP
+			}
+			r := lower(escapedName[i])
+			utf8Val *= 16
+			switch {
+			case r >= '0' && r <= '9':
+				utf8Val += uint(r) - '0'
+			case r >= 'a' && r <= 'f':
+				utf8Val += uint(r) - 'a' + 10
+			default:
+				return name
+			}
+			i++
+		}
+		// Didn't find closing underscore, invalid.
+		return name
+	}
+	return unescaped.String()
+}

app/vmselect/prometheus/tsdb_status_response.qtpl

@@ -1,6 +1,7 @@
 {% import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
 ) %}
 
 {% stripspace %}
@@ -34,9 +35,9 @@ TSDBStatusResponse generates response for /api/v1/status/tsdb .
 ]
 {% endfunc %}
 
-{% func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) %}
+{% func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []metricnamestats.StatRecord) %}
 {% code
-  queryStatsByMetricName := make(map[string]storage.MetricNamesStatsRecord,len(queryStats))
+  queryStatsByMetricName := make(map[string]metricnamestats.StatRecord,len(queryStats))
   for _, record := range queryStats{
       queryStatsByMetricName[record.MetricName] = record
     }

app/vmselect/prometheus/tsdb_status_response.qtpl.go

@@ -8,228 +8,229 @@ package prometheus
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
 )
 
 // TSDBStatusResponse generates response for /api/v1/status/tsdb .
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
 func StreamTSDBStatusResponse(qw422016 *qt422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:8
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:9
 	qw422016.N().S(`{"status":"success","data":{"totalSeries":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:12
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:13
 	qw422016.N().DUL(status.TotalSeries)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:12
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:13
 	qw422016.N().S(`,"totalLabelValuePairs":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:13
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
 	qw422016.N().DUL(status.TotalLabelValuePairs)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:13
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
 	qw422016.N().S(`,"seriesCountByMetricName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
 	streamtsdbStatusMetricNameEntries(qw422016, status.SeriesCountByMetricName, status.SeriesQueryStatsByMetricName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:14
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
 	qw422016.N().S(`,"seriesCountByLabelName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:15
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
 	qw422016.N().S(`,"seriesCountByFocusLabelValue":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByFocusLabelValue)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:16
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
 	qw422016.N().S(`,"seriesCountByLabelValuePair":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
 	streamtsdbStatusEntries(qw422016, status.SeriesCountByLabelValuePair)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:17
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
 	qw422016.N().S(`,"labelValueCountByLabelName":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
 	streamtsdbStatusEntries(qw422016, status.LabelValueCountByLabelName)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:18
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:19
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:20
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
 	qt.Done()
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	streamdumpQueryTrace(qw422016, qt)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:21
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:22
 	qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 func WriteTSDBStatusResponse(qq422016 qtio422016.Writer, status *storage.TSDBStatus, qt *querytracer.Tracer) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	StreamTSDBStatusResponse(qw422016, status, qt)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 func TSDBStatusResponse(status *storage.TSDBStatus, qt *querytracer.Tracer) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	WriteTSDBStatusResponse(qb422016, status, qt)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:23
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:24
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
 func streamtsdbStatusEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:25
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:26
 	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
 	for i, e := range a {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:27
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:28
 		qw422016.N().S(`{"name":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
 		qw422016.N().Q(e.Name)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:29
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
 		qw422016.N().S(`,"value":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
 		qw422016.N().D(int(e.Count))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:30
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:31
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
 		if i+1 < len(a) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:32
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
 		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:33
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:34
 	qw422016.N().S(`]`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 func writetsdbStatusEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	streamtsdbStatusEntries(qw422016, a)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 func tsdbStatusEntries(a []storage.TopHeapEntry) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	writetsdbStatusEntries(qb422016, a)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:35
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:36
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:37
-func streamtsdbStatusMetricNameEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:39
-	queryStatsByMetricName := make(map[string]storage.MetricNamesStatsRecord, len(queryStats))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:38
+func streamtsdbStatusMetricNameEntries(qw422016 *qt422016.Writer, a []storage.TopHeapEntry, queryStats []metricnamestats.StatRecord) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:40
+	queryStatsByMetricName := make(map[string]metricnamestats.StatRecord, len(queryStats))
 	for _, record := range queryStats {
 		queryStatsByMetricName[record.MetricName] = record
 	}
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:43
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:44
 	qw422016.N().S(`[`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:45
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:46
 	for i, e := range a {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:45
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:46
 		qw422016.N().S(`{`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:48
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:49
 		entry, ok := queryStatsByMetricName[e.Name]
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:49
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:50
 		qw422016.N().S(`"name":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:50
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
 		qw422016.N().Q(e.Name)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:50
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
 		qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
-		if !ok {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:51
-			qw422016.N().S(`"value":`)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:52
-			qw422016.N().D(int(e.Count))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:53
-		} else {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:53
+		if !ok {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:52
 			qw422016.N().S(`"value":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:54
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:53
 			qw422016.N().D(int(e.Count))
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:54
+		} else {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:54
+			qw422016.N().S(`"value":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
+			qw422016.N().D(int(e.Count))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
 			qw422016.N().S(`,"requestsCount":`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
-			qw422016.N().D(int(entry.RequestsCount))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:55
-			qw422016.N().S(`,"lastRequestTimestamp":`)
 //line app/vmselect/prometheus/tsdb_status_response.qtpl:56
+			qw422016.N().D(int(entry.RequestsCount))
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:56
+			qw422016.N().S(`,"lastRequestTimestamp":`)
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:57
 			qw422016.N().D(int(entry.LastRequestTs))
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:57
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:58
 		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:57
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:58
 		qw422016.N().S(`}`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:59
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
 		if i+1 < len(a) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:59
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
 			qw422016.N().S(`,`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:59
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
 		}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:61
 	}
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:60
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:61
 	qw422016.N().S(`]`)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
-func writetsdbStatusMetricNameEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+func writetsdbStatusMetricNameEntries(qq422016 qtio422016.Writer, a []storage.TopHeapEntry, queryStats []metricnamestats.StatRecord) {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	streamtsdbStatusMetricNameEntries(qw422016, a, queryStats)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 }
 
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
-func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []storage.MetricNamesStatsRecord) string {
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
+func tsdbStatusMetricNameEntries(a []storage.TopHeapEntry, queryStats []metricnamestats.StatRecord) string {
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	writetsdbStatusMetricNameEntries(qb422016, a, queryStats)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	qs422016 := string(qb422016.B)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 	return qs422016
-//line app/vmselect/prometheus/tsdb_status_response.qtpl:62
+//line app/vmselect/prometheus/tsdb_status_response.qtpl:63
 }

app/vmselect/promql/aggr.go

@@ -742,7 +742,7 @@ func getRangeTopKTimeseries(tss []*timeseries, modifier *metricsql.ModifierExpr,
 
 func reverseSeries(tss []*timeseries) {
 	j := len(tss)
-	for i := 0; i < len(tss)/2; i++ {
+	for i := range len(tss) / 2 {
 		j--
 		tss[i], tss[j] = tss[j], tss[i]
 	}
@@ -983,7 +983,7 @@ func getPerPointIQRBounds(tss []*timeseries) ([]float64, []float64) {
 	var qs []float64
 	lower := make([]float64, pointsLen)
 	upper := make([]float64, pointsLen)
-	for i := 0; i < pointsLen; i++ {
+	for i := range pointsLen {
 		values = values[:0]
 		for _, ts := range tss {
 			v := ts.Values[i]

app/vmselect/promql/aggr_incremental_test.go

@@ -53,7 +53,7 @@ func TestIncrementalAggr(t *testing.T) {
 			Values:     valuesExpected,
 		}}
 		// run the test multiple times to make sure there are no side effects on concurrency
-		for i := 0; i < 10; i++ {
+		for i := range 10 {
 			iafc := newIncrementalAggrFuncContext(ae, callbacks)
 			tssSrcCopy := copyTimeseries(tssSrc)
 			if err := testIncrementalParallelAggr(iafc, tssSrcCopy, tssExpected); err != nil {

app/vmselect/promql/eval.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"regexp"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -1165,6 +1166,61 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
 			},
 		}
 		return evalExpr(qt, ec, be)
+	// the cached rate result could be inaccurate in edge cases, see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098
+	case "rate":
+		if iafc != nil {
+			if !strings.EqualFold(iafc.ae.Name, "sum") {
+				qt.Printf("do not apply instant rollup optimization for incremental aggregate %s()", iafc.ae.Name)
+				return evalAt(qt, timestamp, window)
+			}
+			qt.Printf("optimized calculation for sum(rate(m[d])) as (sum(increase(m[d])) / d)")
+			afe := expr.(*metricsql.AggrFuncExpr)
+			fe := afe.Args[0].(*metricsql.FuncExpr)
+			feIncrease := *fe
+			feIncrease.Name = "increase"
+			// copy RollupExpr to drop possible offset,
+			// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
+			newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
+			newArg.Offset = nil
+			feIncrease.Args = []metricsql.Expr{newArg}
+			d := newArg.Window.Duration(ec.Step)
+			if d == 0 {
+				d = ec.Step
+			}
+			afeIncrease := *afe
+			afeIncrease.Args = []metricsql.Expr{&feIncrease}
+			be := &metricsql.BinaryOpExpr{
+				Op:              "/",
+				KeepMetricNames: true,
+				Left:            &afeIncrease,
+				Right: &metricsql.NumberExpr{
+					N: float64(d) / 1000,
+				},
+			}
+			return evalExpr(qt, ec, be)
+		}
+		qt.Printf("optimized calculation for instant rollup rate(m[d]) as (increase(m[d]) / d)")
+		fe := expr.(*metricsql.FuncExpr)
+		feIncrease := *fe
+		feIncrease.Name = "increase"
+		// copy RollupExpr to drop possible offset,
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
+		newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
+		newArg.Offset = nil
+		feIncrease.Args = []metricsql.Expr{newArg}
+		d := newArg.Window.Duration(ec.Step)
+		if d == 0 {
+			d = ec.Step
+		}
+		be := &metricsql.BinaryOpExpr{
+			Op:              "/",
+			KeepMetricNames: fe.KeepMetricNames,
+			Left:            &feIncrease,
+			Right: &metricsql.NumberExpr{
+				N: float64(d) / 1000,
+			},
+		}
+		return evalExpr(qt, ec, be)
 	case "max_over_time":
 		if iafc != nil {
 			if !strings.EqualFold(iafc.ae.Name, "max") {
@@ -1935,14 +1991,7 @@ func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]flo
 		return values, timestamps
 	}
 	// Remove Prometheus staleness marks, so non-default rollup functions don't hit NaN values.
-	hasStaleSamples := false
-	for _, v := range values {
-		if decimal.IsStaleNaN(v) {
-			hasStaleSamples = true
-			break
-		}
-	}
-	if !hasStaleSamples {
+	if !slices.ContainsFunc(values, decimal.IsStaleNaN) {
 		// Fast path: values have no Prometheus staleness marks.
 		return values, timestamps
 	}

app/vmselect/promql/exec.go

@@ -313,7 +313,7 @@ func escapeDots(s string) string {
 		return s
 	}
 	result := make([]byte, 0, len(s)+2*dotsCount)
-	for i := 0; i < len(s); i++ {
+	for i := range len(s) {
 		if s[i] == '.' && (i == 0 || s[i-1] != '\\') && (i+1 == len(s) || i+1 < len(s) && s[i+1] != '*' && s[i+1] != '+' && s[i+1] != '{') {
 			// Escape a dot if the following conditions are met:
 			// - if it isn't escaped already, i.e. if there is no `\` char before the dot.

app/vmselect/promql/exec_test.go

@@ -67,7 +67,7 @@ func TestExecSuccess(t *testing.T) {
 			Deadline:           searchutil.NewDeadline(time.Now(), time.Minute, ""),
 			RoundDigits:        100,
 		}
-		for i := 0; i < 5; i++ {
+		for range 5 {
 			result, err := Exec(nil, ec, q, false)
 			if err != nil {
 				t.Fatalf(`unexpected error when executing %q: %s`, q, err)
@@ -4018,6 +4018,12 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(scalar)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(123, 456, time())`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-no-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "foo", "bar"))`
@@ -4030,6 +4036,12 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(single-value-no-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(123,456, label_set(100, "foo", "bar"))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-invalid-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "le", "foobar"))`
@@ -4042,6 +4054,12 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(single-value-invalid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(50, 60, label_set(100, "le", "foobar"))`
+		resultExpected := []netstorage.Result{}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-inf-le)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6, label_set(100, "le", "+Inf"))`
@@ -4183,6 +4201,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(single-value-valid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(0, 100, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_fraction(single-value-valid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(200, 300, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(histogram_quantile(0.6, label_set(100, "le", "200"), "foobar"))`
@@ -4212,7 +4252,7 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
 	})
-	t.Run(`histogram_quantile(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
+	t.Run(`histogram_share(single-value-valid-le, boundsLabel)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(histogram_share(120, label_set(100, "le", "200"), "foobar"))`
 		r1 := netstorage.Result{
@@ -4311,7 +4351,37 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
-	t.Run(`histogram_share(single-value-valid-le-mid-le)`, func(t *testing.T) {
+	t.Run(`histogram_fraction(single-value-valid-le-max-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(0,100, (
+			label_set(100, "le", "100"),
+			label_set(40, "le", "50"),
+			label_set(0, "le", "10"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_fraction(single-value-valid-le-min-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(0,10, (
+			label_set(100, "le", "100"),
+			label_set(40, "le", "50"),
+			label_set(0, "le", "10"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_share(single-value-valid-le-mid-le-1)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_share(105, (
 			label_set(100, "le", "200"),
@@ -4325,6 +4395,34 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_share(single-value-valid-le-mid-le-2)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_share(55, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, 0, 0, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`histogram_fraction(single-value-valid-le-mid-le)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(55,105, (
+			label_set(100, "le", "200"),
+			label_set(0, "le", "55"),
+		))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966, 0.3448275862068966},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(single-value-valid-le-min-phi-no-zero-bucket)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0, label_set(100, "le", "200"))`
@@ -4358,6 +4456,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(scalar-phi)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(25, time() / 8, label_set(100, "le", "200"))`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.5, 0.625, 0.75, 0.875, 0.875, 0.875},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(duplicate-le)`, func(t *testing.T) {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3225
 		t.Parallel()
@@ -4439,6 +4548,36 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(valid)`, func(t *testing.T) {
+		t.Parallel()
+		q := `sort(histogram_fraction(0, 25,
+			label_set(90, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+			or label_set(200, "tag", "xx", "le", "10")
+			or label_set(300, "tag", "xx", "le", "30")
+		))`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.325, 0.325, 0.325, 0.325, 0.325, 0.325},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666, 0.9166666666666666},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("tag"),
+			Value: []byte("xx"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(negative-bucket-count)`, func(t *testing.T) {
 		t.Parallel()
 		q := `histogram_quantile(0.6,
@@ -4555,6 +4694,25 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`histogram_fraction(normal-bucket-count)`, func(t *testing.T) {
+		t.Parallel()
+		q := `histogram_fraction(22,35,
+			label_set(0, "foo", "bar", "le", "10")
+			or label_set(100, "foo", "bar", "le", "30")
+			or label_set(300, "foo", "bar", "le", "+Inf")
+		)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0.1333333333333333, 0.1333333333333333, 0.1333333333333333, 0.1333333333333333, 0.1333333333333333, 0.1333333333333333},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run(`histogram_quantile(normal-bucket-count, boundsLabel)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort(histogram_quantile(0.2,
@@ -9827,7 +9985,7 @@ func TestExecError(t *testing.T) {
 			Deadline:           searchutil.NewDeadline(time.Now(), time.Minute, ""),
 			RoundDigits:        100,
 		}
-		for i := 0; i < 4; i++ {
+		for range 4 {
 			rv, err := Exec(nil, ec, q, false)
 			if err == nil {
 				t.Fatalf(`expecting non-nil error on %q`, q)

app/vmselect/promql/parse_cache.go

@@ -55,7 +55,7 @@ type parseCache struct {
 
 func newParseCache() *parseCache {
 	pc := new(parseCache)
-	for i := 0; i < parseBucketCount; i++ {
+	for i := range parseBucketCount {
 		pc.buckets[i] = newParseBucket()
 	}
 	return pc
@@ -75,7 +75,7 @@ func (pc *parseCache) get(q string) *parseCacheValue {
 
 func (pc *parseCache) requests() uint64 {
 	var n uint64
-	for i := 0; i < parseBucketCount; i++ {
+	for i := range parseBucketCount {
 		n += pc.buckets[i].requests.Load()
 	}
 	return n
@@ -83,7 +83,7 @@ func (pc *parseCache) requests() uint64 {
 
 func (pc *parseCache) misses() uint64 {
 	var n uint64
-	for i := 0; i < parseBucketCount; i++ {
+	for i := range parseBucketCount {
 		n += pc.buckets[i].misses.Load()
 	}
 	return n
@@ -91,7 +91,7 @@ func (pc *parseCache) misses() uint64 {
 
 func (pc *parseCache) len() uint64 {
 	var n uint64
-	for i := 0; i < parseBucketCount; i++ {
+	for i := range parseBucketCount {
 		n += pc.buckets[i].len()
 	}
 	return n

app/vmselect/promql/parse_cache_test.go

@@ -17,7 +17,7 @@ func testGetParseCacheValue(q string) *parseCacheValue {
 
 func testGenerateQueries(items int) []string {
 	queries := make([]string, items)
-	for i := 0; i < items; i++ {
+	for i := range items {
 		queries[i] = fmt.Sprintf(`node_time_seconds{instance="node%d", job="job%d"}`, i, i)
 	}
 	return queries
@@ -102,7 +102,7 @@ func TestParseCacheBucketOverflow(t *testing.T) {
 	v := testGetParseCacheValue(queries[0])
 
 	// Fill bucket
-	for i := 0; i < parseBucketMaxLen; i++ {
+	for i := range parseBucketMaxLen {
 		b.put(queries[i], v)
 	}
 	expectedLen = uint64(parseBucketMaxLen)

app/vmselect/promql/parse_cache_timing_test.go

@@ -15,7 +15,7 @@ func BenchmarkCachePutNoOverFlow(b *testing.B) {
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			for i := 0; i < items; i++ {
+			for i := range items {
 				pc.put(queries[i], v)
 			}
 		}
@@ -32,14 +32,14 @@ func BenchmarkCacheGetNoOverflow(b *testing.B) {
 	queries := testGenerateQueries(items)
 	v := testGetParseCacheValue(queries[0])
 
-	for i := 0; i < len(queries); i++ {
+	for i := range queries {
 		pc.put(queries[i], v)
 	}
 	b.ResetTimer()
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			for i := 0; i < items; i++ {
+			for i := range items {
 				if v := pc.get(queries[i]); v == nil {
 					b.Errorf("unexpected nil value obtained from cache for query: %s ", queries[i])
 				}
@@ -59,7 +59,7 @@ func BenchmarkCachePutGetNoOverflow(b *testing.B) {
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			for i := 0; i < items; i++ {
+			for i := range items {
 				pc.put(queries[i], v)
 				if res := pc.get(queries[i]); res == nil {
 					b.Errorf("unexpected nil value obtained from cache for query: %s ", queries[i])
@@ -79,7 +79,7 @@ func BenchmarkCachePutOverflow(b *testing.B) {
 	queries := testGenerateQueries(items)
 	v := testGetParseCacheValue(queries[0])
 
-	for i := 0; i < parseCacheMaxLen; i++ {
+	for i := range parseCacheMaxLen {
 		c.put(queries[i], v)
 	}
 
@@ -105,7 +105,7 @@ func BenchmarkCachePutGetOverflow(b *testing.B) {
 	queries := testGenerateQueries(items)
 	v := testGetParseCacheValue(queries[0])
 
-	for i := 0; i < parseCacheMaxLen; i++ {
+	for i := range parseCacheMaxLen {
 		c.put(queries[i], v)
 	}
 
@@ -141,8 +141,8 @@ var testSimpleQueries = []string{
 
 func BenchmarkParsePromQLWithCacheSimple(b *testing.B) {
 	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		for j := 0; j < len(testSimpleQueries); j++ {
+	for range b.N {
+		for j := range testSimpleQueries {
 			_, err := parsePromQLWithCache(testSimpleQueries[j])
 			if err != nil {
 				b.Errorf("unexpected error: %s", err)
@@ -155,7 +155,7 @@ func BenchmarkParsePromQLWithCacheSimpleParallel(b *testing.B) {
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			for i := 0; i < len(testSimpleQueries); i++ {
+			for i := range testSimpleQueries {
 				_, err := parsePromQLWithCache(testSimpleQueries[i])
 				if err != nil {
 					b.Errorf("unexpected error: %s", err)
@@ -210,8 +210,8 @@ var testComplexQueries = []string{
 
 func BenchmarkParsePromQLWithCacheComplex(b *testing.B) {
 	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		for j := 0; j < len(testComplexQueries); j++ {
+	for range b.N {
+		for j := range testComplexQueries {
 			_, err := parsePromQLWithCache(testComplexQueries[j])
 			if err != nil {
 				b.Errorf("unexpected error: %s", err)
@@ -224,7 +224,7 @@ func BenchmarkParsePromQLWithCacheComplexParallel(b *testing.B) {
 	b.ReportAllocs()
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
-			for i := 0; i < len(testComplexQueries); i++ {
+			for i := range testComplexQueries {
 				_, err := parsePromQLWithCache(testComplexQueries[i])
 				if err != nil {
 					b.Errorf("unexpected error: %s", err)

app/vmselect/promql/rollup_result_cache.go

@@ -739,7 +739,7 @@ func (mi *rollupResultCacheMetainfo) Unmarshal(src []byte) error {
 	entriesLen := int(encoding.UnmarshalUint32(src))
 	src = src[4:]
 	mi.entries = slicesutil.SetLength(mi.entries, entriesLen)
-	for i := 0; i < entriesLen; i++ {
+	for i := range entriesLen {
 		tail, err := mi.entries[i].Unmarshal(src)
 		if err != nil {
 			return fmt.Errorf("cannot unmarshal entry #%d: %w", i, err)