Fix version mismatch

Signed-off-by: Arkadii Yakovets <ark@victoriametrics.com>

.github/workflows/benchmark.yml

@@ -0,0 +1,56 @@
+name: benchmark
+on:
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+      - "dashboards/**"
+      - "deployment/**.yml"
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-streamaggr-benchmark-image:
+    name: build
+    runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'streamaggr-benchmark')
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Package VMAgent Docker image for benchmark
+        run: |
+          SKIP_SCRATCH_BUILD=true \
+          DOCKER_BUILD_OPTS='--cache-to type=gha,mode=max --cache-from type=gha' \
+          PKG_TAG=${{ github.event.pull_request.head.sha }} \
+          DOCKER_REGISTRY=ghcr.io \
+          TARGET_PLATFORM=linux/amd64 make publish-vmagent

.github/workflows/main.yml

@@ -6,12 +6,14 @@ on:
       - cluster
       - master
     paths:
+      - '.github/workflows/main.yml'
       - '**.go'
   pull_request:
     branches:
       - cluster
       - master
     paths:
+      - '.github/workflows/main.yml'
       - '**.go'
 
 permissions:
@@ -46,10 +48,25 @@ jobs:
           key: go-artifacts-${{ runner.os }}-check-all-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.sum', 'Makefile', 'app/**/Makefile') }}
           restore-keys: go-artifacts-${{ runner.os }}-check-all-
 
-      - name: Run check-all
-        run: |
-          make check-all
-          git diff --exit-code
+      - name: Run fmt
+        run: make fmt
+
+      - name: Run vet
+        run: make vet
+      
+        # Use action instead of `make golangci-lint` to speed up the process
+        # as it caches data between builds.
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          # The version must match "install-golangci-lint" Makefile target version.
+          version: v1.59.1
+
+      - name: Run govulncheck
+        run: make govulncheck
+
+      - name: Check diff
+        run: git diff --exit-code
 
   test:
     name: test

.gitignore

@@ -7,7 +7,6 @@
 .vscode
 *.test
 *.swp
-/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -24,5 +23,4 @@ Gemfile.lock
 _site
 *.tmp
 /docs/.jekyll-metadata
-coverage.txt
-cspell.json
+coverage.txt

CODE_OF_CONDUCT_RU.md

@@ -0,0 +1,120 @@
+
+# Кодекс Поведения участника
+
+## Наши обязательства
+
+Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
+свободным от притеснений для всех, независимо от возраста, телосложения,
+видимых или невидимых ограничений способности, этнической принадлежности,
+половых признаков, гендерной идентичности и выражения, уровня опыта,
+образования, социо-экономического статуса, национальности, внешности,
+расы, религии, или сексуальной идентичности и ориентации.
+
+Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
+дружелюбное, многообразное, инклюзивное и здоровое сообщество.
+
+## Наши стандарты
+
+Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
+
+* Проявление доброты и эмпатии к другим участникам проекта
+* Уважение к чужой точке зрения и опыту
+* Конструктивная критика и принятие конструктивной критики
+* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
+  и извлечение уроков из опыта
+* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
+
+Примеры неприемлемого поведения участников включают в себя:
+
+* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
+* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
+* Публичное или приватное домогательство
+* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
+* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
+
+## Обязанности
+
+Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
+поведения и будут предпринимать соответствующие и честные меры по исправлению положения
+в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
+
+Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
+комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
+с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
+
+## Область применения
+
+Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
+а также когда человек официально представляет сообщество в публичных местах.
+Примеры представления проекта или сообщества включают использование официальной электронной почты,
+публикации в официальном аккаунте в социальных сетях,
+или упоминания как представителя в онлайн или оффлайн мероприятии.
+
+## Приведение в исполнение
+
+О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
+поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
+Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
+
+Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
+неприкосновенность автора сообщения.
+
+## Руководство по исполнению
+
+Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
+чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
+
+### 1. Исправление
+
+**Общественное влияние**: Использование недопустимой лексики или другое поведение,
+считающиеся непрофессиональным или нежелательным в сообществе.
+
+**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
+объясняющее суть нарушения и почему такое поведение
+было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
+
+### 2. Предупреждение
+
+**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
+
+**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
+На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
+включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
+в публичных пространствах, а так же во внешних каналах,
+таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
+
+### 3. Временный бан
+
+**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
+включая продолжительное неуместное поведение.
+
+**Последствия**: Временный запрет (бан) на любое взаимодействие
+или публичное общение с сообществом на определенный период времени.
+На этот период не допускается публичное или личное взаимодействие с людьми,
+вовлеченными в инцидент, включая незапрошенное взаимодействие
+с теми, кто обеспечивает соблюдение Кодекса.
+Нарушение этих правил влечет за собой вечный бан.
+
+### 4. Вечный бан
+
+**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
+включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
+или проявление агрессии либо пренебрежительного отношения к категориям лиц.
+
+**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
+
+## Атрибуция
+
+Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
+версии 2.0, доступной по адресу
+<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
+
+Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
+<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
+<https://www.contributor-covenant.org/translations>.

Makefile

@@ -13,12 +13,10 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
-TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
-include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
@@ -246,7 +244,7 @@ release-victoria-metrics-windows-amd64:
 
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
@@ -303,7 +301,7 @@ release-victoria-logs-windows-amd64:
 
 release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
@@ -367,7 +365,7 @@ release-vmutils-goos-goarch: \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
 	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
 			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
@@ -495,7 +493,8 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
+	# The version must match GitHub main.yml lint job "Run golangci-lint" step version.
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -514,3 +513,33 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
+
+copy-docs:
+# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
+# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
+	echo "---" > ${DST}
+	@if [ ${ORDER} -ne 0 ]; then \
+		echo "sort: ${ORDER}" >> ${DST}; \
+		echo "weight: ${ORDER}" >> ${DST}; \
+		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
+	fi
+
+	echo "title: ${TITLE}" >> ${DST}
+	@if [ ${OLD_URL} ]; then \
+		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
+	fi
+	echo "---" >> ${DST}
+	cat ${SRC} >> ${DST}
+	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
+	rm -rf docs/*.tmp
+
+# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
+# For ORDER=0 it adds no order tag/weight tag.
+# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
+# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
+# Images starting with <img src="docs/ are replaced with <img src="
+# Cluster docs are supposed to be ordered as 2nd.
+# The rest of docs is ordered manually.
+docs-sync:
+	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
+	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
-| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to <security@victoriametrics.com>
+Please report any security issues to security@victoriametrics.com

app/victoria-logs/Makefile

@@ -81,9 +81,6 @@ victoria-logs-linux-ppc64le:
 victoria-logs-linux-s390x:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-victoria-logs-linux-loong64:
-	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 victoria-logs-linux-386:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/victoria-metrics/main_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -118,7 +117,7 @@ func (q *Query) metrics() ([]Metric, error) {
 type QueryInstant struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TV     [2]any            `json:"value"`
+		TV     [2]interface{}    `json:"value"`
 	} `json:"result"`
 }
 
@@ -141,7 +140,7 @@ func (q QueryInstant) metrics() ([]Metric, error) {
 type QueryRange struct {
 	Result []struct {
 		Metric map[string]string `json:"metric"`
-		Values [][]any           `json:"values"`
+		Values [][]interface{}   `json:"values"`
 	} `json:"result"`
 }
 
@@ -250,24 +249,24 @@ func TestWriteRead(t *testing.T) {
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
-		for _, test := range readIn("prometheus", insertionTime) {
+		for _, test := range readIn("prometheus", t, insertionTime) {
 			if test.Data == nil {
 				continue
 			}
+			s := newSuite(t)
 			r := testutil.WriteRequest{}
-			testData := strings.Join(test.Data, "\n")
-			if err := json.Unmarshal([]byte(testData), &r.Timeseries); err != nil {
-				panic(fmt.Errorf("BUG: cannot unmarshal TimeSeries: %s\ntest data\n%s", err, testData))
+			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
+			data, err := testutil.Compress(r)
+			s.greaterThan(len(r.Timeseries), 0)
+			if err != nil {
+				t.Errorf("error compressing %v %s", r, err)
+				t.Fail()
 			}
-			if n := len(r.Timeseries); n <= 0 {
-				panic(fmt.Errorf("BUG: expecting non-empty Timeseries in test data:\n%s", testData))
-			}
-			data := testutil.Compress(r)
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
 	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", insertionTime) {
+		for _, test := range readIn("csv", t, insertionTime) {
 			if test.Data == nil {
 				continue
 			}
@@ -276,7 +275,7 @@ func testWrite(t *testing.T) {
 	})
 
 	t.Run("influxdb", func(t *testing.T) {
-		for _, x := range readIn("influxdb", insertionTime) {
+		for _, x := range readIn("influxdb", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -285,7 +284,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("graphite", func(t *testing.T) {
-		for _, x := range readIn("graphite", insertionTime) {
+		for _, x := range readIn("graphite", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -294,7 +293,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdb", func(t *testing.T) {
-		for _, x := range readIn("opentsdb", insertionTime) {
+		for _, x := range readIn("opentsdb", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -303,7 +302,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdbhttp", func(t *testing.T) {
-		for _, x := range readIn("opentsdbhttp", insertionTime) {
+		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -317,7 +316,7 @@ func testWrite(t *testing.T) {
 func testRead(t *testing.T) {
 	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
-			for _, x := range readIn(engine, insertionTime) {
+			for _, x := range readIn(engine, t, insertionTime) {
 				test := x
 				t.Run(test.Name, func(t *testing.T) {
 					t.Parallel()
@@ -366,10 +365,11 @@ func testRead(t *testing.T) {
 	}
 }
 
-func readIn(readFor string, insertTime time.Time) []test {
-	testDir := filepath.Join(testFixturesDir, readFor)
+func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
+	t.Helper()
+	s := newSuite(t)
 	var tt []test
-	err := filepath.Walk(testDir, func(path string, _ os.FileInfo, err error) error {
+	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -377,129 +377,84 @@ func readIn(readFor string, insertTime time.Time) []test {
 			return nil
 		}
 		b, err := os.ReadFile(path)
-		if err != nil {
-			panic(fmt.Errorf("BUG: cannot read %s: %s", path, err))
-		}
+		s.noError(err)
 		item := test{}
-		if err := json.Unmarshal(b, &item); err != nil {
-			panic(fmt.Errorf("cannot parse %T from %s: %s; data:\n%s", &item, path, err, b))
-		}
+		s.noError(json.Unmarshal(b, &item))
 		for i := range item.Data {
 			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
 		}
 		tt = append(tt, item)
 		return nil
-	})
-	if err != nil {
-		panic(fmt.Errorf("BUG: cannot read test data at %s: %w", testDir, err))
-	}
+	}))
 	if len(tt) == 0 {
-		panic(fmt.Errorf("BUG: no tests found in %s", testDir))
+		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
 	}
 	return tt
 }
 
 func httpWrite(t *testing.T, address, query string, r io.Reader) {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Post(requestURL, "", r)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
-	_ = resp.Body.Close()
-	if resp.StatusCode != 204 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 204", requestURL, resp.StatusCode)
-	}
+	s := newSuite(t)
+	resp, err := http.Post(address+query, "", r)
+	s.noError(err)
+	s.noError(resp.Body.Close())
+	s.equalInt(resp.StatusCode, 204)
 }
 
-func tcpWrite(t *testing.T, address, data string) {
+func tcpWrite(t *testing.T, address string, data string) {
 	t.Helper()
-
+	s := newSuite(t)
 	conn, err := net.Dial("tcp", address)
-	if err != nil {
-		t.Fatalf("cannot dial %s: %s", address, err)
-	}
+	s.noError(err)
 	defer func() {
 		_ = conn.Close()
 	}()
 	n, err := conn.Write([]byte(data))
-	if err != nil {
-		t.Fatalf("cannot write %d bytes to %s: %s", len(data), address, err)
-	}
-	if n != len(data) {
-		panic(fmt.Errorf("BUG: conn.Write() returned unexpected number of written bytes to %s; got %d; want %d", address, n, len(data)))
-	}
+	s.noError(err)
+	s.equalInt(n, len(data))
 }
 
 func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Get(requestURL)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	if resp.StatusCode != 200 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
-	}
-
+	s.equalInt(resp.StatusCode, 200)
 	var rows []Metric
-	dec := json.NewDecoder(resp.Body)
-	for {
+	for dec := json.NewDecoder(resp.Body); dec.More(); {
 		var row Metric
-		err := dec.Decode(&row)
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return rows
-			}
-			t.Fatalf("cannot decode %T from response received from %s: %s", &row, requestURL, err)
-		}
+		s.noError(dec.Decode(&row))
 		rows = append(rows, row)
 	}
+	return rows
 }
 
-func httpReadStruct(t *testing.T, address, query string, dst any) {
+func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Get(requestURL)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	if resp.StatusCode != 200 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
-	}
-	err = json.NewDecoder(resp.Body).Decode(dst)
-	if err != nil {
-		t.Fatalf("cannot decode %T from response received from %s: %s", dst, requestURL, err)
-	}
+	s.equalInt(resp.StatusCode, 200)
+	s.noError(json.NewDecoder(resp.Body).Decode(dst))
 }
 
 func httpReadData(t *testing.T, address, query string) []byte {
 	t.Helper()
-
-	requestURL := address + query
-	resp, err := http.Get(requestURL)
-	if err != nil {
-		t.Fatalf("cannot send request to %s: %s", requestURL, err)
-	}
+	s := newSuite(t)
+	resp, err := http.Get(address + query)
+	s.noError(err)
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	if resp.StatusCode != 200 {
-		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
-	}
+	s.equalInt(resp.StatusCode, 200)
 	data, err := io.ReadAll(resp.Body)
-	if err != nil {
-		t.Fatalf("cannot read response from %s: %s", requestURL, err)
-	}
+	s.noError(err)
 	return data
 }
 
@@ -548,6 +503,34 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
+type suite struct{ t *testing.T }
+
+func newSuite(t *testing.T) *suite { return &suite{t: t} }
+
+func (s *suite) noError(err error) {
+	s.t.Helper()
+	if err != nil {
+		s.t.Errorf("unexpected error %v", err)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) equalInt(a, b int) {
+	s.t.Helper()
+	if a != b {
+		s.t.Errorf("%d not equal %d", a, b)
+		s.t.FailNow()
+	}
+}
+
+func (s *suite) greaterThan(a, b int) {
+	s.t.Helper()
+	if a <= b {
+		s.t.Errorf("%d less or equal then %d", a, b)
+		s.t.FailNow()
+	}
+}
+
 func TestImportJSONLines(t *testing.T) {
 	f := func(labelsCount, labelLen int) {
 		t.Helper()

app/victoria-metrics/test/prom_writer.go

@@ -1,16 +1,12 @@
 package test
 
-import (
-	"fmt"
-
-	"github.com/golang/snappy"
-)
+import "github.com/golang/snappy"
 
 // Compress marshals and compresses wr.
-func Compress(wr WriteRequest) []byte {
+func Compress(wr WriteRequest) ([]byte, error) {
 	data, err := wr.Marshal()
 	if err != nil {
-		panic(fmt.Errorf("BUG: cannot compress WriteRequest: %s", err))
+		return nil, err
 	}
-	return snappy.Encode(nil, data)
+	return snappy.Encode(nil, data), nil
 }

app/vlinsert/elasticsearch/elasticsearch.go

@@ -57,12 +57,6 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
-	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
-		// Return fake response for Logstash APIs requests.
-		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
-		fmt.Fprintf(w, `{}`)
-		return true
-	}
 	switch path {
 	case "/":
 		switch r.Method {

app/vlinsert/loki/loki_protobuf.go

@@ -79,14 +79,12 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.UnmarshalProtobuf(bb.B)
+	err = req.Unmarshal(bb.B)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
-	fields := getFields()
-	defer putFields(fields)
-
+	var commonFields []logstorage.Field
 	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
@@ -94,60 +92,30 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
-		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
+		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
 		if err != nil {
 			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
-		commonFieldsLen := len(fields.fields)
+		fields := commonFields
 
 		entries := stream.Entries
 		for j := range entries {
-			e := &entries[j]
-			fields.fields = fields.fields[:commonFieldsLen]
-
-			for _, lp := range e.StructuredMetadata {
-				fields.fields = append(fields.fields, logstorage.Field{
-					Name:  lp.Name,
-					Value: lp.Value,
-				})
-			}
-
-			fields.fields = append(fields.fields, logstorage.Field{
+			entry := &entries[j]
+			fields = append(fields[:len(commonFields)], logstorage.Field{
 				Name:  "_msg",
-				Value: e.Line,
+				Value: entry.Line,
 			})
-
-			ts := e.Timestamp.UnixNano()
+			ts := entry.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
-
-			lmp.AddRow(ts, fields.fields)
+			lmp.AddRow(ts, fields)
 		}
 		rowsIngested += len(stream.Entries)
 	}
 	return rowsIngested, nil
 }
 
-func getFields() *fields {
-	v := fieldsPool.Get()
-	if v == nil {
-		return &fields{}
-	}
-	return v.(*fields)
-}
-
-func putFields(f *fields) {
-	f.fields = f.fields[:0]
-	fieldsPool.Put(f)
-}
-
-var fieldsPool sync.Pool
-
-type fields struct {
-	fields []logstorage.Field
-}
-
 // parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
 //
 // See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
@@ -213,6 +181,6 @@ func getPushRequest() *PushRequest {
 }
 
 func putPushRequest(req *PushRequest) {
-	req.reset()
+	req.Reset()
 	pushReqsPool.Put(req)
 }

app/vlinsert/loki/loki_protobuf_test.go

@@ -36,7 +36,7 @@ func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.
 		Entries: []Entry{
 			{
 				Timestamp: time.Unix(0, timestamp),
-				Line:      strings.Clone(msg),
+				Line:      msg,
 			},
 		},
 	})
@@ -58,7 +58,10 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
-		data := tlp.pr.MarshalProtobuf(nil)
+		data, err := tlp.pr.Marshal()
+		if err != nil {
+			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
+		}
 		encodedData := snappy.Encode(nil, data)
 
 		tlp2 := &insertutils.TestLogMessageProcessor{}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {
@@ -39,47 +38,29 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	})
 }
 
-func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
-	var b []byte
-	var entries []Entry
-	streams := make([]Stream, streamsCount)
-	for i := range streams {
-		b = b[:0]
-		b = append(b, '{')
-		for j := 0; j < labelsCount; j++ {
-			b = append(b, "label_"...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, `="value_`...)
-			b = strconv.AppendInt(b, int64(j), 10)
-			b = append(b, '"')
-			if j < labelsCount-1 {
-				b = append(b, ',')
+func getProtobufBody(streams, rows, labels int) []byte {
+	var pr PushRequest
+
+	for i := 0; i < streams; i++ {
+		var st Stream
+
+		st.Labels = `{`
+		for j := 0; j < labels; j++ {
+			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
+			if j < labels-1 {
+				st.Labels += `,`
 			}
 		}
-		b = append(b, '}')
-		labels := string(b)
+		st.Labels += `}`
 
-		var rowsBuf []byte
-		entriesLen := len(entries)
-		for j := 0; j < rowsCount; j++ {
-			rowsBufLen := len(rowsBuf)
-			rowsBuf = append(rowsBuf, "value_"...)
-			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
-			entries = append(entries, Entry{
-				Timestamp: time.Now(),
-				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
-			})
+		for j := 0; j < rows; j++ {
+			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
 		}
 
-		st := &streams[i]
-		st.Labels = labels
-		st.Entries = entries[entriesLen:]
-	}
-	pr := PushRequest{
-		Streams: streams,
+		pr.Streams = append(pr.Streams, st)
 	}
 
-	body := pr.MarshalProtobuf(nil)
+	body, _ := pr.Marshal()
 	encodedBody := snappy.Encode(nil, body)
 
 	return encodedBody

app/vlinsert/loki/pb.go

@@ -1,302 +0,0 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: push_request.proto
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package loki
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/VictoriaMetrics/easyproto"
-)
-
-var mp easyproto.MarshalerPool
-
-// PushRequest represents Loki PushRequest
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
-type PushRequest struct {
-	Streams []Stream
-
-	entriesBuf   []Entry
-	labelPairBuf []LabelPair
-}
-
-func (pr *PushRequest) reset() {
-	pr.Streams = pr.Streams[:0]
-
-	pr.entriesBuf = pr.entriesBuf[:0]
-	pr.labelPairBuf = pr.labelPairBuf[:0]
-}
-
-// UnmarshalProtobuf unmarshals pr from protobuf message at src.
-//
-// pr remains valid until src is modified.
-func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
-	pr.reset()
-	var err error
-	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
-	return err
-}
-
-// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
-func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
-	m := mp.Get()
-	pr.marshalProtobuf(m.MessageMarshaler())
-	dst = m.Marshal(dst)
-	mp.Put(m)
-	return dst
-}
-
-func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	for _, s := range pr.Streams {
-		s.marshalProtobuf(mm.AppendMessage(1))
-	}
-}
-
-func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message PushRequest {
-	//   repeated Stream streams = 1;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
-			}
-			pr.Streams = append(pr.Streams, Stream{})
-			s := &pr.Streams[len(pr.Streams)-1]
-			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
-			}
-		}
-	}
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Stream represents Loki stream.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
-type Stream struct {
-	Labels  string
-	Entries []Entry
-}
-
-func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, s.Labels)
-	for _, e := range s.Entries {
-		e.marshalProtobuf(mm.AppendMessage(2))
-	}
-}
-
-func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
-	// message Stream {
-	//   string labels = 1;
-	//   repeated Entry entries = 2;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	entriesBufLen := len(entriesBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			labels, ok := fc.String()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
-			}
-			s.Labels = labels
-		case 2:
-			data, ok := fc.MessageData()
-			if !ok {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
-			}
-			entriesBuf = append(entriesBuf, Entry{})
-			e := &entriesBuf[len(entriesBuf)-1]
-			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
-			if err != nil {
-				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
-			}
-		}
-	}
-	s.Entries = entriesBuf[entriesBufLen:]
-	return entriesBuf, labelPairBuf, nil
-}
-
-// Entry represents Loki entry.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
-type Entry struct {
-	Timestamp          time.Time
-	Line               string
-	StructuredMetadata []LabelPair
-}
-
-func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	marshalTime(mm, 1, e.Timestamp)
-	mm.AppendString(2, e.Line)
-	for _, lp := range e.StructuredMetadata {
-		lp.marshalProtobuf(mm.AppendMessage(3))
-	}
-}
-
-func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
-	// message Entry {
-	//   Timestamp timestamp = 1;
-	//   string line = 2;
-	//   repeated LabelPair structuredMetadata = 3;
-	// }
-	var err error
-	var fc easyproto.FieldContext
-	labelPairBufLen := len(labelPairBuf)
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
-			}
-			timestamp, err := unmarshalTime(data)
-			if err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
-			}
-			e.Timestamp = timestamp
-		case 2:
-			line, ok := fc.String()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read Line")
-			}
-			e.Line = line
-		case 3:
-			data, ok := fc.MessageData()
-			if !ok {
-				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
-			}
-			labelPairBuf = append(labelPairBuf, LabelPair{})
-			lp := &labelPairBuf[len(labelPairBuf)-1]
-			if err := lp.unmarshalProtobuf(data); err != nil {
-				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
-			}
-		}
-	}
-	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
-	return labelPairBuf, nil
-}
-
-// LabelPair represents Loki label pair.
-//
-// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
-type LabelPair struct {
-	Name  string
-	Value string
-}
-
-func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendString(1, lp.Name)
-	mm.AppendString(2, lp.Value)
-}
-
-func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
-	// message LabelPair {
-	//   string name = 1;
-	//   string value = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			name, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot read name")
-			}
-			lp.Name = name
-		case 2:
-			value, ok := fc.String()
-			if !ok {
-				return fmt.Errorf("cannot unmarshal value")
-			}
-			lp.Value = value
-		}
-	}
-	return nil
-}
-
-func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
-	nsecs := timestamp.UnixNano()
-	ts := Timestamp{
-		Seconds: nsecs / 1e9,
-		Nanos:   int32(nsecs % 1e9),
-	}
-	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
-}
-
-func unmarshalTime(src []byte) (time.Time, error) {
-	var ts Timestamp
-	if err := ts.unmarshalProtobuf(src); err != nil {
-		return time.Time{}, err
-	}
-	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	return timestamp, nil
-}
-
-// Timestamp is protobuf well-known timestamp type.
-type Timestamp struct {
-	Seconds int64
-	Nanos   int32
-}
-
-func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
-	mm.AppendInt64(1, ts.Seconds)
-	mm.AppendInt32(2, ts.Nanos)
-}
-
-func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
-	// message Timestamp {
-	//   int64 seconds = 1;
-	//   int32 nanos = 2;
-	// }
-	var fc easyproto.FieldContext
-	for len(src) > 0 {
-		src, err = fc.NextField(src)
-		if err != nil {
-			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
-		}
-		switch fc.FieldNum {
-		case 1:
-			seconds, ok := fc.Int64()
-			if !ok {
-				return fmt.Errorf("cannot read Seconds")
-			}
-			ts.Seconds = seconds
-		case 2:
-			nanos, ok := fc.Int32()
-			if !ok {
-				return fmt.Errorf("cannot read Nanos")
-			}
-			ts.Nanos = nanos
-		}
-	}
-	return nil
-}

app/vlinsert/loki/push_request.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package logproto;
+
+import "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
+
+message PushRequest {
+  repeated StreamAdapter streams = 1 [
+    (gogoproto.jsontag) = "streams",
+    (gogoproto.customtype) = "Stream"
+  ];
+}
+
+message StreamAdapter {
+  string labels = 1 [(gogoproto.jsontag) = "labels"];
+  repeated EntryAdapter entries = 2 [
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "entries"
+  ];
+  // hash contains the original hash of the stream.
+  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
+}
+
+message EntryAdapter {
+  google.protobuf.Timestamp timestamp = 1 [
+    (gogoproto.stdtime) = true,
+    (gogoproto.nullable) = false,
+    (gogoproto.jsontag) = "ts"
+  ];
+  string line = 2 [(gogoproto.jsontag) = "line"];
+}

app/vlinsert/loki/timestamp.go

@@ -0,0 +1,110 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/gogo/protobuf/types"
+)
+
+const (
+	// Seconds field of the earliest valid Timestamp.
+	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	minValidSeconds = -62135596800
+	// Seconds field just after the latest valid Timestamp.
+	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
+	maxValidSeconds = 253402300800
+)
+
+// validateTimestamp determines whether a Timestamp is valid.
+// A valid timestamp represents a time in the range
+// [0001-01-01, 10000-01-01) and has a Nanos field
+// in the range [0, 1e9).
+//
+// If the Timestamp is valid, validateTimestamp returns nil.
+// Otherwise, it returns an error that describes
+// the problem.
+//
+// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
+func validateTimestamp(ts *types.Timestamp) error {
+	if ts == nil {
+		return errors.New("timestamp: nil Timestamp")
+	}
+	if ts.Seconds < minValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
+	}
+	if ts.Seconds >= maxValidSeconds {
+		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
+	}
+	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
+		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
+	}
+	return nil
+}
+
+// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
+// but avoids the escape incurred by using fmt.Sprintf, eliminating
+// unnecessary heap allocations.
+func formatTimestamp(ts *types.Timestamp) string {
+	if ts == nil {
+		return "nil"
+	}
+
+	seconds := strconv.FormatInt(ts.Seconds, 10)
+	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
+	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
+}
+
+func sizeOfStdTime(t time.Time) int {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0
+	}
+	return ts.Size()
+}
+
+func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
+	ts, err := timestampProto(t)
+	if err != nil {
+		return 0, err
+	}
+	return ts.MarshalTo(data)
+}
+
+func stdTimeUnmarshal(t *time.Time, data []byte) error {
+	ts := &types.Timestamp{}
+	if err := ts.Unmarshal(data); err != nil {
+		return err
+	}
+	tt, err := timestampFromProto(ts)
+	if err != nil {
+		return err
+	}
+	*t = tt
+	return nil
+}
+
+func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
+	// Don't return the zero value on error, because corresponds to a valid
+	// timestamp. Instead return whatever time.Unix gives us.
+	var t time.Time
+	if ts == nil {
+		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
+	} else {
+		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	}
+	return t, validateTimestamp(ts)
+}
+
+func timestampProto(t time.Time) (types.Timestamp, error) {
+	ts := types.Timestamp{
+		Seconds: t.Unix(),
+		Nanos:   int32(t.Nanosecond()),
+	}
+	return ts, validateTimestamp(&ts)
+}

app/vlinsert/loki/types.go

@@ -0,0 +1,481 @@
+package loki
+
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+import (
+	"fmt"
+	"io"
+	"time"
+)
+
+// Stream contains a unique labels set as a string and a set of entries for it.
+// We are not using the proto generated version but this custom one so that we
+// can improve serialization see benchmark.
+type Stream struct {
+	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
+	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
+	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
+}
+
+// Entry is a log entry with a timestamp.
+type Entry struct {
+	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
+	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Stream) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.Hash != 0 {
+		i = encodeVarintPush(dAtA, i, m.Hash)
+		i--
+		dAtA[i] = 0x18
+	}
+	if len(m.Entries) > 0 {
+		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPush(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0x12
+		}
+	}
+	if len(m.Labels) > 0 {
+		i -= len(m.Labels)
+		copy(dAtA[i:], m.Labels)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+// Marshal implements the proto.Marshaler interface.
+func (m *Entry) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+// MarshalTo marshals m to dst.
+func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+// MarshalToSizedBuffer marshals m to the sized buffer.
+func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Line) > 0 {
+		i -= len(m.Line)
+		copy(dAtA[i:], m.Line)
+		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
+		i--
+		dAtA[i] = 0x12
+	}
+	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
+	if err7 != nil {
+		return 0, err7
+	}
+	i -= n7
+	i = encodeVarintPush(dAtA, i, uint64(n7))
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Stream) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Labels = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Entries = append(m.Entries, Entry{})
+			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
+			}
+			m.Hash = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.Hash |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Unmarshal unmarshals the given data into m.
+func (m *Entry) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPush
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPush
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPush
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPush
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Line = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPush(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthPush
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+
+// Size returns the size of the serialized Stream.
+func (m *Stream) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Labels)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	if len(m.Entries) > 0 {
+		for _, e := range m.Entries {
+			l = e.Size()
+			n += 1 + l + sovPush(uint64(l))
+		}
+	}
+	if m.Hash != 0 {
+		n += 1 + sovPush(m.Hash)
+	}
+	return n
+}
+
+// Size returns the size of the serialized Entry
+func (m *Entry) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = sizeOfStdTime(m.Timestamp)
+	n += 1 + l + sovPush(uint64(l))
+	l = len(m.Line)
+	if l > 0 {
+		n += 1 + l + sovPush(uint64(l))
+	}
+	return n
+}
+
+// Equal returns true if the two Streams are equal.
+func (m *Stream) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Stream)
+	if !ok {
+		that2, ok := that.(Stream)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if m.Labels != that1.Labels {
+		return false
+	}
+	if len(m.Entries) != len(that1.Entries) {
+		return false
+	}
+	for i := range m.Entries {
+		if !m.Entries[i].Equal(that1.Entries[i]) {
+			return false
+		}
+	}
+	return m.Hash == that1.Hash
+}
+
+// Equal returns true if the two Entries are equal.
+func (m *Entry) Equal(that interface{}) bool {
+	if that == nil {
+		return m == nil
+	}
+
+	that1, ok := that.(*Entry)
+	if !ok {
+		that2, ok := that.(Entry)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return m == nil
+	} else if m == nil {
+		return false
+	}
+	if !m.Timestamp.Equal(that1.Timestamp) {
+		return false
+	}
+	if m.Line != that1.Line {
+		return false
+	}
+	return true
+}

app/vlogsgenerator/main.go

@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 	for i := 0; i < activeStreams; i++ {
 		ip := toIPv4(rand.Uint32())
 		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
 			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
 		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
 		for j := 0; j < *constFieldsPerLog; j++ {

app/vlselect/logsql/logsql.go

@@ -70,10 +70,9 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		fieldsLimit = 0
 	}
 
-	// Prepare the query for hits count.
-	q.Optimize()
-	q.DropAllPipes()
+	// Prepare the query
 	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	q.Optimize()
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.c9cc37dd.css",
-    "main.js": "./static/js/main.7ae2e2c4.js",
+    "main.css": "./static/css/main.1041c3d4.css",
+    "main.js": "./static/js/main.8988988c.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.8c2e588d62b87f90dbf0.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.c9cc37dd.css",
-    "static/js/main.7ae2e2c4.js"
+    "static/css/main.1041c3d4.css",
+    "static/js/main.8988988c.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.7ae2e2c4.js"></script><link href="./static/css/main.c9cc37dd.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8988988c.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/media/MetricsQL.aaabf95f2c9bf356bde4.md

@@ -1,4 +1,5 @@
 ---
+sort: 23
 weight: 23
 title: MetricsQL
 menu:
@@ -9,6 +10,9 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
+
+# MetricsQL
+
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -216,11 +220,9 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
+See also [median_over_time](#median_over_time).
 
 #### changes
 
@@ -260,8 +262,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,8 +272,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -284,8 +282,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -296,9 +292,7 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
-See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
+See also [count_over_time](#count_over_time).
 
 #### count_over_time
 
@@ -319,8 +313,6 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -440,8 +432,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -453,15 +443,11 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -469,8 +455,6 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -478,9 +462,8 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`.
-
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
+returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -512,14 +495,13 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -528,13 +510,12 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -543,10 +524,6 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
-
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -563,18 +540,15 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -613,8 +587,6 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -622,11 +594,9 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
+See also [tmax_over_time](#tmax_over_time).
 
 #### median_over_time
 
@@ -634,8 +604,6 @@ See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -643,11 +611,9 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
+See also [tmin_over_time](#tmin_over_time).
 
 #### mode_over_time
 
@@ -656,8 +622,6 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -668,8 +632,6 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -697,8 +659,6 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -710,8 +670,6 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -722,12 +680,11 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -746,8 +703,6 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -755,11 +710,10 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
 This function is supported by PromQL.
 
 #### rollup
@@ -771,10 +725,6 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
-See also [rollup_rate](#rollup_rate).
-
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -786,8 +736,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -814,8 +762,6 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
-
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -828,17 +774,12 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
-
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -846,12 +787,10 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
-
-See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
-
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -885,8 +824,6 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -901,8 +838,6 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -914,8 +849,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -933,8 +866,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -946,43 +877,35 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
 
 #### sum_eq_over_time
 
-`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
 
-`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
 
-`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -992,8 +915,6 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -1003,8 +924,6 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1082,8 +1001,6 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
-
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2313,8 +2230,7 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2347,6 +2263,5 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
-  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
+  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.

app/vlselect/vmui/victorialogs.html

@@ -1,55 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta
-    name="description"
-    content="UI for VictoriaMetrics"
-  />
-  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
-  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
-  <!--
-    Notice the use of %PUBLIC_URL% in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>VM UI</title>
-
-  <meta name="twitter:card" content="summary_large_image">
-  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
-  <meta name="twitter:title" content="UI for VictoriaMetrics">
-  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta name="twitter:site" content="@VictoriaMetrics">
-
-  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
-  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
-  <meta property="og:type" content="website">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-</body>
-</html>

app/vlselect/vmui/victoriametrics.html

@@ -1,56 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta
-    name="description"
-    content="UI for VictoriaMetrics"
-  />
-  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
-  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
-  <!--
-    Notice the use of %PUBLIC_URL% in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>VM UI</title>
-  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
-
-  <meta name="twitter:card" content="summary_large_image">
-  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
-  <meta name="twitter:title" content="UI for VictoriaMetrics">
-  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta name="twitter:site" content="@VictoriaMetrics">
-
-  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
-  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
-  <meta property="og:type" content="website">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-</body>
-</html>

app/vlselect/vmui/vmanomaly.html

@@ -1,56 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="utf-8"/>
-  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
-  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
-  <meta name="theme-color" content="#000000"/>
-  <meta
-    name="description"
-    content="UI for VictoriaMetrics"
-  />
-  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
-  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
-  <!--
-    manifest.json provides metadata used when your web app is installed on a
-    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-  -->
-  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
-  <!--
-    Notice the use of %PUBLIC_URL% in the tags above.
-    It will be replaced with the URL of the `public` folder during the build.
-    Only files inside the `public` folder can be referenced from the HTML.
-
-    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
-    work correctly both with client-side routing and a non-root public URL.
-    Learn how to configure a non-root public URL by running `npm run build`.
-  -->
-  <title>VM UI</title>
-  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
-
-  <meta name="twitter:card" content="summary_large_image">
-  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
-  <meta name="twitter:title" content="UI for VictoriaMetrics">
-  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta name="twitter:site" content="@VictoriaMetrics">
-
-  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
-  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
-  <meta property="og:type" content="website">
-</head>
-<body>
-<noscript>You need to enable JavaScript to run this app.</noscript>
-<div id="root"></div>
-<!--
-  This HTML file is a template.
-  If you open it directly in the browser, you will see an empty page.
-
-  You can add webfonts, meta tags, or analytics to this file.
-  The build step will place the bundled scripts into the <body> tag.
-
-  To begin the development, run `npm start` or `yarn start`.
-  To create a production bundle, use `npm run build` or `yarn build`.
--->
-</body>
-</html>

app/vlstorage/main.go

@@ -80,7 +80,7 @@ func Init() {
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics, true)
+	metrics.UnregisterSet(storageMetrics)
 	storageMetrics = nil
 
 	strg.MustClose()

app/vmagent/main.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -44,7 +42,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -72,8 +70,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -116,7 +114,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -318,10 +316,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "/influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -440,7 +434,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -449,17 +443,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()
@@ -568,10 +562,6 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
-	case "influx/health":
-		influxHealthRequests.Inc()
-		influxutils.WriteHealthCheckResponse(w)
-		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -682,8 +672,7 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
-	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
+	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Fatalf("unxepected error %s", err)
+		t.Errorf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/remotewrite/client.go

@@ -15,8 +15,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -34,10 +34,8 @@ var (
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
-	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
-	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
 	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
@@ -92,9 +90,6 @@ type client struct {
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
-	retryMinInterval time.Duration
-	retryMaxTime     time.Duration
-
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
@@ -125,7 +120,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
+		DialContext:         httputils.GetStatDialFunc("vmagent_remotewrite"),
 		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
@@ -148,15 +143,13 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
-		sanitizedURL:     sanitizedURL,
-		remoteWriteURL:   remoteWriteURL,
-		authCfg:          authCfg,
-		awsCfg:           awsCfg,
-		fq:               fq,
-		hc:               hc,
-		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
-		retryMaxTime:     retryMaxTime.GetOptionalArg(argIdx),
-		stopCh:           make(chan struct{}),
+		sanitizedURL:   sanitizedURL,
+		remoteWriteURL: remoteWriteURL,
+		authCfg:        authCfg,
+		awsCfg:         awsCfg,
+		fq:             fq,
+		hc:             hc,
+		stopCh:         make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
 
@@ -403,11 +396,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
 // The function returns false only if c.stopCh is closed.
-// Otherwise, it tries sending the block to remote storage indefinitely.
+// Otherwise it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
-	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
+	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
+	retryDuration := timeutil.AddJitterToDuration(time.Second)
 	retriesCount := 0
 
 again:

app/vmagent/remotewrite/relabel.go

@@ -181,7 +181,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		return &relabelCtx{}
 	},
 }

app/vmagent/remotewrite/remotewrite.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
-	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -89,15 +88,15 @@ var (
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
-		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
+		"See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
-	rwctxsGlobal []*remoteWriteCtx
+	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
+	rwctxs []*remoteWriteCtx
 
 	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
 	defaultAuthToken = &auth.Token{}
@@ -110,11 +109,8 @@ var (
 		StatusCode: http.StatusTooManyRequests,
 	}
 
-	// disableOnDiskQueueAny is set to true if at least a single -remoteWrite.url is configured with -remoteWrite.disableOnDiskQueue
-	disableOnDiskQueueAny bool
-
-	// dropSamplesOnFailureGlobal is set to true if -remoteWrite.dropSamplesOnOverload is set or if multiple -remoteWrite.disableOnDiskQueue options are set.
-	dropSamplesOnFailureGlobal bool
+	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
+	disableOnDiskQueueAll bool
 )
 
 // MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
@@ -207,18 +203,28 @@ func Init() {
 	relabelConfigSuccess.Set(1)
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
-	initStreamAggrConfigGlobal()
+	sasFile, sasOpts := getStreamAggrOpts(-1)
+	if sasFile != "" {
+		sas, err := newStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -streamAggr.config=%q: %s", sasFile, err)
+		}
+		sasGlobal.Store(sas)
+	} else if sasOpts.DedupInterval > 0 {
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesDropFailed, sasOpts.DedupInterval, sasOpts.DropInputLabels, sasOpts.Alias)
+	}
 
-	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	if len(*remoteWriteURLs) > 0 {
+		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
+	}
 
-	disableOnDiskQueues := []bool(*disableOnDiskQueue)
-	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
-
-	// Samples must be dropped if multiple -remoteWrite.disableOnDiskQueue options are configured and at least a single is set to true.
-	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
-	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
-	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
-	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
+	disableOnDiskQueueAll = true
+	for _, v := range *disableOnDiskQueue {
+		if !v {
+			disableOnDiskQueueAll = false
+			break
+		}
+	}
 
 	dropDanglingQueues()
 
@@ -228,9 +234,9 @@ func Init() {
 		defer configReloaderWG.Done()
 		for {
 			select {
+			case <-sighupCh:
 			case <-configReloaderStopCh:
 				return
-			case <-sighupCh:
 			}
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
@@ -249,8 +255,8 @@ func dropDanglingQueues() {
 	// In case if there were many persistent queues with identical *remoteWriteURLs
 	// the queue with the last index will be dropped.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
+	existingQueues := make(map[string]struct{}, len(rwctxs))
+	for _, rwctx := range rwctxs {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -267,7 +273,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
 	}
 }
 
@@ -376,10 +382,10 @@ func Stop() {
 		deduplicatorGlobal = nil
 	}
 
-	for _, rwctx := range rwctxsGlobal {
+	for _, rwctx := range rwctxs {
 		rwctx.MustStop()
 	}
-	rwctxsGlobal = nil
+	rwctxs = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -391,8 +397,6 @@ func Stop() {
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
 //
-// PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
-//
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
@@ -405,7 +409,7 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, dropSamplesOnFailureGlobal)
+	return tryPush(at, wr, false)
 }
 
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
@@ -422,30 +426,33 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
 	}
+	rowsCount := getRowsCount(tss)
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
 	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	rwctxs, ok := getEligibleRemoteWriteCtxs(tss, forceDropSamplesOnFailure)
-	if !ok {
-		// At least a single remote write queue is blocked and dropSamplesOnFailure isn't set.
-		// Return false to the caller, so it could re-send samples again.
-		return false
-	}
-	if len(rwctxs) == 0 {
-		// All the remote write queues are skipped because they are blocked and dropSamplesOnFailure is set to true.
-		// Return true to the caller, so it doesn't re-send the samples again.
-		return true
+	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
+	if disableOnDiskQueueAll {
+		for _, rwctx := range rwctxs {
+			if rwctx.fq.IsWriteBlocked() {
+				rwctx.pushFailures.Inc()
+				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+					// Just drop samples
+					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+					continue
+				}
+				return false
+			}
+		}
 	}
 
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
+	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
-	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -498,46 +505,20 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
 	return true
 }
 
-func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	if !disableOnDiskQueueAny {
-		return rwctxsGlobal, true
-	}
-
-	// This code is applicable if at least a single remote storage has -disableOnDiskQueue
-	rwctxs := make([]*remoteWriteCtx, 0, len(rwctxsGlobal))
-	for _, rwctx := range rwctxsGlobal {
-		if !rwctx.fq.IsWriteBlocked() {
-			rwctxs = append(rwctxs, rwctx)
-		} else {
-			rwctx.pushFailures.Inc()
-			if !forceDropSamplesOnFailure {
-				return nil, false
-			}
-			rowsCount := getRowsCount(tss)
-			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-		}
-	}
-	return rwctxs, true
-}
-
-func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
-	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
-	if len(rwctxs) == 0 {
+func pushToRemoteStoragesDropFailed(tss []prompbmarshal.TimeSeries) {
+	if tryPushBlockToRemoteStorages(tss, true) {
 		return
 	}
-	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
-		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
-	}
 }
 
-func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -556,7 +537,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
@@ -577,7 +558,7 @@ func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmar
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -764,8 +745,10 @@ type remoteWriteCtx struct {
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
 
-	streamAggrKeepInput bool
-	streamAggrDropInput bool
+	streamAggrKeepInput   bool
+	streamAggrDropInput   bool
+	disableOnDiskQueue    bool
+	dropSamplesOnOverload bool
 
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
@@ -790,7 +773,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
-
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -835,13 +817,31 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
+		disableOnDiskQueue:    isPQDisabled,
 
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
+	}
+
+	// Initialize sas
+	sasFile, sasOpts := getStreamAggrOpts(argIdx)
+	if sasFile != "" {
+		sas, err := newStreamAggrConfig(argIdx, rwctx.pushInternalTrackDropped)
+		if err != nil {
+			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
+		}
+		rwctx.sas.Store(sas)
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
+	} else if sasOpts.DedupInterval > 0 {
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, sasOpts.DedupInterval, sasOpts.DropInputLabels, sasOpts.Alias)
 	}
-	rwctx.initStreamAggrConfig()
 
 	return rwctx
 }
@@ -934,9 +934,8 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 
 	// Couldn't push tss to remote storage
 	rwctx.pushFailures.Inc()
-	if forceDropSamplesOnFailure {
-		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
+		rwctx.rowsDroppedOnPushFailure.Add(len(tss))
 		return true
 	}
 	return false
@@ -963,12 +962,14 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.fq.IsPersistentQueueDisabled() {
+	if !rwctx.disableOnDiskQueue {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
 	rwctx.pushFailures.Inc()
-	rowsCount := getRowsCount(tss)
-	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
+		rowsCount := getRowsCount(tss)
+		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+	}
 }
 
 func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
@@ -999,7 +1000,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 }
 
 var tssPool = &sync.Pool{
-	New: func() any {
+	New: func() interface{} {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},

app/vmagent/remotewrite/remotewrite_test.go

@@ -77,16 +77,14 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "global")
 		}
 
-		if streamAggrConfig != "" {
-			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
+		if len(streamAggrConfig) > 0 {
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), nil, streamaggr.Options{})
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
-			defer sas.MustStop()
 			rwctx.sas.Store(sas)
 		}
 
@@ -96,9 +94,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		if !rwctx.TryPush(inputTss, false) {
-			t.Fatalf("cannot push samples to rwctx")
-		}
+		rwctx.TryPush(inputTss, false)
 
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)

app/vmagent/remotewrite/streamaggr.go

@@ -50,7 +50,7 @@ var (
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
@@ -61,182 +61,104 @@ var (
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
 func CheckStreamAggrConfigs() error {
-	// Check global config
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		return err
-	}
-	sas.MustStop()
-
-	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
-	}
-
 	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+
+	if _, err := newStreamAggrConfig(-1, pushNoop); err != nil {
+		return fmt.Errorf("could not load -streamAggr.config stream aggregation config: %w", err)
+	}
+	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
+		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
+			len(*streamAggrConfig), len(*remoteWriteURLs))
+	}
 	for idx := range *streamAggrConfig {
-		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
-		if err != nil {
+		if _, err := newStreamAggrConfig(idx, pushNoop); err != nil {
 			return err
 		}
-		sas.MustStop()
 	}
 	return nil
 }
 
 func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfigGlobal()
-	for _, rwctx := range rwctxsGlobal {
-		rwctx.reloadStreamAggrConfig()
+	reloadStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
+	for idx, rwctx := range rwctxs {
+		reloadStreamAggrConfig(idx, rwctx.pushInternalTrackDropped)
 	}
 }
 
-func reloadStreamAggrConfigGlobal() {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return
-	}
-
-	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
-		return
-	}
-
-	sas := sasGlobal.Load()
-	if !sasNew.Equal(sas) {
-		sasOld := sasGlobal.Swap(sasNew)
-		sasOld.MustStop()
-		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
-	} else {
-		sasNew.MustStop()
-		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
-	}
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
-}
-
-func initStreamAggrConfigGlobal() {
-	sas, err := newStreamAggrConfigGlobal()
-	if err != nil {
-		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		sasGlobal.Store(sas)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrGlobalDedupInterval.Duration()
-		if dedupInterval > 0 {
-			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
-		}
-	}
-}
-
-func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
-	idx := rwctx.idx
-
-	sas, err := rwctx.newStreamAggrConfig()
-	if err != nil {
-		logger.Fatalf("cannot initialize stream aggregators: %s", err)
-	}
-	if sas != nil {
-		filePath := sas.FilePath()
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
-	} else {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
-		if dedupInterval > 0 {
-			alias := fmt.Sprintf("dedup-%d", idx+1)
-			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
-		}
-	}
-}
-
-func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
-	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
-	if path == "" {
-		return
-	}
-
+func reloadStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) {
+	path, opts := getStreamAggrOpts(idx)
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
 
-	sasNew, err := rwctx.newStreamAggrConfig()
+	sasNew, err := newStreamAggrConfigWithOpts(pushFunc, path, opts)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		logger.Errorf("cannot reload stream aggregation config at %q; continue using the previously loaded config; error: %s", path, err)
 		return
 	}
 
-	sas := rwctx.sas.Load()
+	var sas *streamaggr.Aggregators
+	if idx < 0 {
+		sas = sasGlobal.Load()
+	} else {
+		sas = rwctxs[idx].sas.Load()
+	}
+
 	if !sasNew.Equal(sas) {
-		sasOld := rwctx.sas.Swap(sasNew)
+		var sasOld *streamaggr.Aggregators
+		if idx < 0 {
+			sasOld = sasGlobal.Swap(sasNew)
+		} else {
+			sasOld = rwctxs[idx].sas.Swap(sasNew)
+		}
 		sasOld.MustStop()
-		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
+		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
 	} else {
 		sasNew.MustStop()
-		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
+		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
 }
 
-func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
-	path := *streamAggrGlobalConfig
-	if path == "" {
-		return nil, nil
+func getStreamAggrOpts(idx int) (string, streamaggr.Options) {
+	if idx < 0 {
+		return *streamAggrGlobalConfig, streamaggr.Options{
+			DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+			DropInputLabels:      *streamAggrGlobalDropInputLabels,
+			IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
+			IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+			Alias:                "global",
+		}
 	}
-
-	opts := &streamaggr.Options{
-		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
-		DropInputLabels:      *streamAggrGlobalDropInputLabels,
-		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-		KeepInput:            *streamAggrGlobalKeepInput,
-	}
-
-	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
-	}
-	return sas, nil
-}
-
-func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
-	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
-}
-
-func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path := streamAggrConfig.GetOptionalArg(idx)
-	if path == "" {
-		return nil, nil
-	}
-
-	alias := fmt.Sprintf("%d:secret-url", idx+1)
+	url := fmt.Sprintf("%d:secret-url", idx+1)
 	if *showRemoteWriteURL {
-		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+		url = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
 	}
-	opts := &streamaggr.Options{
+	opts := streamaggr.Options{
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
-		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
+		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		Alias:                url,
 	}
 
-	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
-	if err != nil {
-		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
+	if len(*streamAggrConfig) == 0 {
+		return "", opts
 	}
-	return sas, nil
+	return streamAggrConfig.GetOptionalArg(idx), opts
+}
+
+func newStreamAggrConfigWithOpts(pushFunc streamaggr.PushFunc, path string, opts streamaggr.Options) (*streamaggr.Aggregators, error) {
+	if len(path) == 0 {
+		// Skip empty stream aggregation config.
+		return nil, nil
+	}
+	return streamaggr.LoadFromFile(path, pushFunc, opts)
+}
+
+func newStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path, opts := getStreamAggrOpts(idx)
+	return newStreamAggrConfigWithOpts(pushFunc, path, opts)
 }

app/vmalert-tool/Makefile

@@ -81,9 +81,6 @@ vmalert-tool-linux-ppc64le:
 vmalert-tool-linux-s390x:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
-vmalert-tool-linux-loong64:
-	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
-
 vmalert-tool-linux-386:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert-tool/main.go

@@ -41,19 +41,9 @@ Examples:
 						Usage:    "disable adding group's Name as label to generated alerts and time series.",
 						Required: false,
 					},
-					&cli.StringSliceFlag{
-						Name:     "external.label",
-						Usage:    `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
-						Required: false,
-					},
-					&cli.StringFlag{
-						Name:     "external.url",
-						Usage:    `Optional external URL to template in rule's labels or annotations.`,
-						Required: false,
-					},
 				},
 				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url")); failed {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
 						return fmt.Errorf("unittest failed")
 					}
 					return nil

app/vmalert-tool/unittest/input.go

@@ -74,7 +74,10 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
 		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
 	}
 
-	data := testutil.Compress(r)
+	data, err := testutil.Compress(r)
+	if err != nil {
+		return fmt.Errorf("failed to compress data: %v", err)
+	}
 	// write input series to vm
 	httpWrite(dst, bytes.NewBuffer(data))
 	vmstorage.Storage.DebugFlush()

app/vmalert-tool/unittest/input_test.go

@@ -6,61 +6,88 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
-func TestParseInputValue_Failure(t *testing.T) {
-	f := func(input string) {
-		t.Helper()
-
-		_, err := parseInputValue(input, true)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
+func TestParseInputValue(t *testing.T) {
+	testCases := []struct {
+		input  string
+		exp    []sequenceValue
+		failed bool
+	}{
+		{
+			"",
+			nil,
+			true,
+		},
+		{
+			"testfailed",
+			nil,
+			true,
+		},
+		// stale doesn't support operations
+		{
+			"stalex3",
+			nil,
+			true,
+		},
+		{
+			"-4",
+			[]sequenceValue{{Value: -4}},
+			false,
+		},
+		{
+			"_",
+			[]sequenceValue{{Omitted: true}},
+			false,
+		},
+		{
+			"stale",
+			[]sequenceValue{{Value: decimal.StaleNaN}},
+			false,
+		},
+		{
+			"-4x1",
+			[]sequenceValue{{Value: -4}, {Value: -4}},
+			false,
+		},
+		{
+			"_x1",
+			[]sequenceValue{{Omitted: true}},
+			false,
+		},
+		{
+			"1+1x2 0.1 0.1+0.3x2 3.14",
+			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}},
+			false,
+		},
+		{
+			"2-1x4",
+			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
+			false,
+		},
+		{
+			"1+1x1 _ -4 stale 3+20x1",
+			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
+			false,
+		},
 	}
 
-	f("")
-	f("testfailed")
-
-	// stale doesn't support operations
-	f("stalex3")
-}
-
-func TestParseInputValue_Success(t *testing.T) {
-	f := func(input string, outputExpected []sequenceValue) {
-		t.Helper()
-
-		output, err := parseInputValue(input, true)
-		if err != nil {
-			t.Fatalf("unexpected error in parseInputValue: %s", err)
+	for _, tc := range testCases {
+		output, err := parseInputValue(tc.input, true)
+		if err != nil != tc.failed {
+			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
 		}
-
-		if len(outputExpected) != len(output) {
-			t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
+		if len(tc.exp) != len(output) {
+			t.Fatalf("expect %v, got %v", tc.exp, output)
 		}
-		for i := 0; i < len(outputExpected); i++ {
-			if outputExpected[i].Omitted != output[i].Omitted {
-				t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
+		for i := 0; i < len(tc.exp); i++ {
+			if tc.exp[i].Omitted != output[i].Omitted {
+				t.Fatalf("expect %v, got %v", tc.exp, output)
 			}
-			if outputExpected[i].Value != output[i].Value {
-				if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
+			if tc.exp[i].Value != output[i].Value {
+				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
 					continue
 				}
-				t.Fatalf("unexpeccted Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
+				t.Fatalf("expect %v, got %v", tc.exp, output)
 			}
 		}
 	}
-
-	f("-4", []sequenceValue{{Value: -4}})
-
-	f("_", []sequenceValue{{Omitted: true}})
-
-	f("stale", []sequenceValue{{Value: decimal.StaleNaN}})
-
-	f("-4x1", []sequenceValue{{Value: -4}, {Value: -4}})
-
-	f("_x1", []sequenceValue{{Omitted: true}})
-
-	f("1+1x2 0.1 0.1+0.3x2 3.14", []sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}})
-
-	f("2-1x4", []sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}})
-
-	f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
 }

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -24,16 +24,20 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
+              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 in cluster  has been down for more than 5 minutes."
-              dashboard: "/d/dashboard?orgId=1"
+              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
 
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - {}
+          - exp_labels:
+              datacenter: dc-123
 
       - eval_time: 0
         alertname: InstanceDown
         exp_alerts: []
+
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -8,8 +8,7 @@ groups:
           severity: page
         annotations:
           summary: "Instance {{ $labels.instance }} down"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
-          dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
       - alert: AlwaysFiring
         expr: 1
       - alert: SameAlertNameWithDifferentGroup

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -16,8 +16,7 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - exp_labels:
-              cluster: prod
+          - {}
       - eval_time: 1m
         groupname: group2
         alertname: SameAlertNameWithDifferentGroup
@@ -26,8 +25,7 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - exp_labels:
-              cluster: prod
+          - {}
       - eval_time: 6m
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
@@ -63,18 +61,18 @@ tests:
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t1", cluster="prod"}'
+            labels: '{__name__="t1", datacenter="dc-123"}'
       - expr: t2
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t2", cluster="prod"}'
+            labels: '{__name__="t2", datacenter="dc-123"}'
       - expr: t3
         eval_time: 4m
         exp_samples:
           # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
           - value: 3
-            labels: '{__name__="t3", cluster="prod"}'
+            labels: '{__name__="t3", datacenter="dc-123"}'
 
     alert_rule_test:
       - eval_time: 10m
@@ -85,21 +83,22 @@ tests:
               job: vmagent1
               severity: page
               instance: localhost:9090
-              cluster: prod
+              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
-              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
+              description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              cluster: prod
+              datacenter: dc-123
 
       - eval_time: 0
         groupname: alerts
         alertname: InstanceDown
         exp_alerts: []
 
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test", cluster="prod", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -25,21 +25,22 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
-              cluster: prod
+              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 in cluster prod has been down for more than 5 minutes."
-              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
+              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              cluster: prod
+              datacenter: dc-123
 
       - eval_time: 0
         groupname: group1
         alertname: InstanceDown
         exp_alerts: []
 
+    external_labels:
+      datacenter: dc-123

app/vmalert-tool/unittest/unittest.go

@@ -55,7 +55,7 @@ const (
 )
 
 // UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL string) bool {
+func UnitTest(files []string, disableGroupLabel bool) bool {
 	if err := templates.Load([]string{}, true); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
@@ -71,34 +71,14 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 
 	testfiles, err := config.ReadFromFS(files)
 	if err != nil {
-		logger.Fatalf("failed to load test files %q: %v", files, err)
+		fmt.Println("  FAILED")
+		fmt.Printf("\nfailed to read test files: \n%v", err)
 	}
-	if len(testfiles) == 0 {
-		fmt.Println("no test file found")
-		return false
-	}
-
-	labels := make(map[string]string)
-	for _, s := range externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		labels[s[:n]] = s[n+1:]
-	}
-	_, err = notifier.Init(nil, labels, externalURL)
-	if err != nil {
-		logger.Fatalf("failed to init notifier: %v", err)
-	}
-
 	var failed bool
 	for fileName, file := range testfiles {
-		if err := ruleUnitTest(fileName, file, labels); err != nil {
+		if err := ruleUnitTest(fileName, file); err != nil {
 			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
@@ -108,7 +88,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	return failed
 }
 
-func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
+func ruleUnitTest(filename string, content []byte) []error {
 	fmt.Println("\nUnit Testing: ", filename)
 	var unitTestInp unitTestFile
 	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
@@ -146,7 +126,7 @@ func ruleUnitTest(filename string, content []byte, externalLabels map[string]str
 			errs = append(errs, err)
 			continue
 		}
-		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups, externalLabels)
+		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
 		errs = append(errs, testErrs...)
 	}
 
@@ -183,10 +163,6 @@ func verifyTestGroup(group testGroup) error {
 			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
-	if group.ExternalLabels != nil {
-		fmt.Printf("\n%s    warning: filed `external_labels` will be deprecated soon, please use `-external.label` cmd-line flag instead. "+
-			"Check https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735 for details.\n", testGroupName)
-	}
 	return nil
 }
 
@@ -203,7 +179,6 @@ func processFlags() {
 		{flag: "retentionPeriod", value: "100y"},
 		{flag: "datasource.url", value: testDataSourcePath},
 		{flag: "remoteWrite.url", value: testRemoteWritePath},
-		{flag: "notifier.blackhole", value: "true"},
 	} {
 		// panics if flag doesn't exist
 		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
@@ -264,7 +239,7 @@ func tearDown() {
 	fs.MustRemoveAll(storagePath)
 }
 
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
 	// set up vmstorage and http server for ingest and read queries
 	setUp()
 	// tear down vmstorage and clean the data dir
@@ -313,14 +288,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	// create groups with given rule
 	var groups []*rule.Group
 	for _, group := range testGroups {
-		mergedExternalLabels := make(map[string]string)
-		for k, v := range tg.ExternalLabels {
-			mergedExternalLabels[k] = v
-		}
-		for k, v := range externalLabels {
-			mergedExternalLabels[k] = v
-		}
-		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
+		ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
 		groups = append(groups, ng)
 	}
 

app/vmalert-tool/unittest/unittest_test.go

@@ -14,34 +14,34 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestUnitTest_Failure(t *testing.T) {
-	f := func(files []string) {
-		t.Helper()
-
-		failed := UnitTest(files, false, nil, "")
-		if !failed {
-			t.Fatalf("expecting failed test")
+func TestUnitRule(t *testing.T) {
+	testCases := []struct {
+		name              string
+		disableGroupLabel bool
+		files             []string
+		failed            bool
+	}{
+		{
+			name:   "run multi files",
+			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
+			failed: false,
+		},
+		{
+			name:              "disable group label",
+			disableGroupLabel: true,
+			files:             []string{"./testdata/disable-group-label.yaml"},
+			failed:            false,
+		},
+		{
+			name:   "failing test",
+			files:  []string{"./testdata/failed-test.yaml"},
+			failed: true,
+		},
+	}
+	for _, tc := range testCases {
+		fail := UnitTest(tc.files, tc.disableGroupLabel)
+		if fail != tc.failed {
+			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
 		}
 	}
-
-	// failing test
-	f([]string{"./testdata/failed-test.yaml"})
-}
-
-func TestUnitTest_Success(t *testing.T) {
-	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
-		t.Helper()
-
-		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL)
-		if failed {
-			t.Fatalf("unexpected failed test")
-		}
-	}
-
-	// run multi files
-	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
-
-	// disable group label
-	// template with null external values
-	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
 }

app/vmalert/config/config.go

@@ -45,11 +45,11 @@ type Group struct {
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
+func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type group Group
 	if err := unmarshal((*group)(g)); err != nil {
 		return err
@@ -142,11 +142,11 @@ type Rule struct {
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (r *Rule) UnmarshalYAML(unmarshal func(any) error) error {
+func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rule Rule
 	if err := unmarshal((*rule)(r)); err != nil {
 		return err
@@ -301,7 +301,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
-		XXX map[string]any `yaml:",inline"`
+		XXX map[string]interface{} `yaml:",inline"`
 	}{}
 	err = yaml.Unmarshal(data, &g)
 	if err != nil {
@@ -310,7 +310,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
-func checkOverflow(m map[string]any, ctx string) error {
+func checkOverflow(m map[string]interface{}, ctx string) error {
 	if len(m) > 0 {
 		var keys []string
 		for k := range m {

app/vmalert/config/config_test.go

@@ -23,6 +23,12 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
 func TestParseFromURL(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
@@ -49,353 +55,438 @@ groups:
 	defer srv.Close()
 
 	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
-		t.Fatalf("error parsing URLs %s", err)
+		t.Errorf("error parsing URLs %s", err)
 	}
 
 	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
-		t.Fatalf("expected parsing error: %s", err)
+		t.Errorf("expected parsing error: %s", err)
 	}
 }
 
-func TestParse_Success(t *testing.T) {
-	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
-	if err != nil {
-		t.Fatalf("error parsing files %s", err)
+func TestParseBad(t *testing.T) {
+	testCases := []struct {
+		path   []string
+		expErr string
+	}{
+		{
+			[]string{"testdata/rules/rules_interval_bad.rules"},
+			"eval_offset should be smaller than interval",
+		},
+		{
+			[]string{"testdata/rules/rules0-bad.rules"},
+			"unexpected token",
+		},
+		{
+			[]string{"testdata/dir/rules0-bad.rules"},
+			"error parsing annotation",
+		},
+		{
+			[]string{"testdata/dir/rules1-bad.rules"},
+			"duplicate in file",
+		},
+		{
+			[]string{"testdata/dir/rules2-bad.rules"},
+			"function \"unknown\" not defined",
+		},
+		{
+			[]string{"testdata/dir/rules3-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/dir/rules4-bad.rules"},
+			"either `record` or `alert` must be set",
+		},
+		{
+			[]string{"testdata/rules/rules1-bad.rules"},
+			"bad graphite expr",
+		},
+		{
+			[]string{"testdata/dir/rules6-bad.rules"},
+			"missing ':' in header",
+		},
+		{
+			[]string{"http://unreachable-url"},
+			"failed to",
+		},
 	}
-}
-
-func TestParse_Failure(t *testing.T) {
-	f := func(paths []string, errStrExpected string) {
-		t.Helper()
-
-		_, err := Parse(paths, notifier.ValidateTemplates, true)
+	for _, tc := range testCases {
+		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
 		if err == nil {
-			t.Fatalf("expected to get error")
+			t.Errorf("expected to get error")
+			return
 		}
-		if !strings.Contains(err.Error(), errStrExpected) {
-			t.Fatalf("expected err to contain %q; got %q instead", errStrExpected, err)
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
 		}
 	}
-
-	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
-	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
-	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
-	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
-	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
-	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
-	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
-	f([]string{"http://unreachable-url"}, "failed to")
 }
 
-func TestRuleValidate(t *testing.T) {
+func TestRule_Validate(t *testing.T) {
 	if err := (&Rule{}).Validate(); err == nil {
-		t.Fatalf("expected empty name error")
+		t.Errorf("expected empty name error")
 	}
 	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
-		t.Fatalf("expected empty expr error")
+		t.Errorf("expected empty expr error")
 	}
 	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Fatalf("expected valid rule; got %s", err)
+		t.Errorf("expected valid rule; got %s", err)
 	}
 }
 
-func TestGroupValidate_Failure(t *testing.T) {
-	f := func(group *Group, validateExpressions bool, errStrExpected string) {
-		t.Helper()
-
-		err := group.Validate(nil, validateExpressions)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the returned error %q", errStrExpected, errStr)
-		}
-	}
-
-	f(&Group{}, false, "group name must be set")
-
-	f(&Group{
-		Name:     "negative interval",
-		Interval: promutils.NewDuration(-1),
-	}, false, "interval shouldn't be lower than 0")
-
-	f(&Group{
-		Name:       "wrong eval_offset",
-		Interval:   promutils.NewDuration(time.Minute),
-		EvalOffset: promutils.NewDuration(2 * time.Minute),
-	}, false, "eval_offset should be smaller than interval")
-
-	f(&Group{
-		Name:  "wrong limit",
-		Limit: -1,
-	}, false, "invalid limit")
-
-	f(&Group{
-		Name:        "wrong concurrency",
-		Concurrency: -1,
-	}, false, "invalid concurrency")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
+func TestGroup_Validate(t *testing.T) {
+	testCases := []struct {
+		group               *Group
+		rules               []Rule
+		validateAnnotations bool
+		validateExpressions bool
+		expErr              string
+	}{
+		{
+			group:  &Group{},
+			expErr: "group name must be set",
+		},
+		{
+			group: &Group{
+				Name:     "negative interval",
+				Interval: promutils.NewDuration(-1),
 			},
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
+			expErr: "interval shouldn't be lower than 0",
+		},
+		{
+			group: &Group{
+				Name:       "wrong eval_offset",
+				Interval:   promutils.NewDuration(time.Minute),
+				EvalOffset: promutils.NewDuration(2 * time.Minute),
 			},
+			expErr: "eval_offset should be smaller than interval",
 		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Record: "record", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"summary": "{{ value|query }}",
-			}},
-		},
-	}, false, "duplicate")
-
-	f(&Group{
-		Name: "test graphite prometheus bad expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
+		{
+			group: &Group{
+				Name:  "wrong limit",
+				Limit: -1,
 			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
+			expErr: "invalid limit",
+		},
+		{
+			group: &Group{
+				Name:        "wrong concurrency",
+				Concurrency: -1,
 			},
+			expErr: "invalid concurrency",
 		},
-	}, false, "invalid rule")
-
-	f(&Group{
-		Name: "test graphite inherit",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sum(up == 0 ) by (host)",
-			},
-		},
-	}, false, "either `record` or `alert` must be set")
-
-	// validate expressions
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "invalid expression")
-
-	f(&Group{
-		Name: "test thanos",
-		Type: NewRawType("thanos"),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
-		},
-	}, true, "unknown datasource type")
-
-	f(&Group{
-		Name: "test graphite",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "some-description",
-			}},
-		},
-	}, true, "bad graphite expr")
-}
-
-func TestGroupValidate_Success(t *testing.T) {
-	f := func(group *Group, validateAnnotations, validateExpressions bool) {
-		t.Helper()
-
-		var validateTplFn ValidateTplFn
-		if validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := group.Validate(validateTplFn, validateExpressions)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, false, false)
-
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": "{{ value|query }}",
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
 				},
 			},
+			expErr: "",
 		},
-	}, false, false)
-
-	// validate annotiations
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Alert: "alert",
-				Expr:  "up == 1",
-				Labels: map[string]string{
-					"summary": `
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Record: "record",
+						Expr:   "up | 0",
+					},
+				},
+			},
+			expErr:              "invalid expression",
+			validateExpressions: true,
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": "{{ value|query }}",
+						},
+					},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+						Labels: map[string]string{
+							"summary": `
 {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
   {{ . | first | value | humanize1024 }}B
 {{ end }}`,
+						},
+					},
+				},
+			},
+			validateAnnotations: true,
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+					{
+						Alert: "alert",
+						Expr:  "up == 1",
+					},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Record: "record", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "duplicate",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test",
+				Rules: []Rule{
+					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"summary": "{{ value|query }}",
+					}},
+				},
+			},
+			expErr: "",
+		},
+		{
+			group: &Group{
+				Name: "test thanos",
+				Type: NewRawType("thanos"),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "unknown datasource type",
+		},
+		{
+			group: &Group{
+				Name: "test graphite",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "some-description",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test prometheus",
+				Type: NewPrometheusType(),
+				Rules: []Rule{
+					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+						"description": "{{ value|query }}",
+					}},
+				},
+			},
+			validateExpressions: true,
+			expErr:              "",
+		},
+		{
+			group: &Group{
+				Name: "test graphite inherit",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sum(up == 0 ) by (host)",
+					},
 				},
 			},
 		},
-	}, true, false)
-
-	// validate expressions
-	f(&Group{
-		Name: "test prometheus",
-		Type: NewPrometheusType(),
-		Rules: []Rule{
-			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"description": "{{ value|query }}",
-			}},
+		{
+			group: &Group{
+				Name: "test graphite prometheus bad expr",
+				Type: NewGraphiteType(),
+				Rules: []Rule{
+					{
+						Expr: "sum(up == 0 ) by (host)",
+						For:  promutils.NewDuration(10 * time.Millisecond),
+					},
+					{
+						Expr: "sumSeries(time('foo.bar',10))",
+					},
+				},
+			},
+			expErr: "invalid rule",
 		},
-	}, false, true)
-}
-
-func TestHashRule_NotEqual(t *testing.T) {
-	f := func(a, b Rule) {
-		t.Helper()
-
-		aID, bID := HashRule(a), HashRule(b)
-		if aID == bID {
-			t.Fatalf("rule hashes mustn't be equal; got %d", aID)
-		}
 	}
 
-	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
-
-	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 2"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-		"foo": "baz",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1"})
-}
-
-func TestHashRule_Equal(t *testing.T) {
-	f := func(a, b Rule) {
-		t.Helper()
-
-		aID, bID := HashRule(a), HashRule(b)
-		if aID != bID {
-			t.Fatalf("rule hashes must be equal; got %d and %d", aID, bID)
+	for _, tc := range testCases {
+		var validateTplFn ValidateTplFn
+		if tc.validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
+		if err == nil {
+			if tc.expErr != "" {
+				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
+			}
+			continue
+		}
+		if !strings.Contains(err.Error(), tc.expErr) {
+			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
 		}
 	}
+}
 
-	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1"}, Rule{Alert: "alert", Expr: "up == 1"})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}})
-
-	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"foo": "bar",
-		"baz": "foo",
-	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-		"baz": "foo",
-		"foo": "bar",
-	}})
-
-	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
-
-	f(Rule{
-		Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
-	}, Rule{Alert: "alert", Expr: "up == 1"})
+func TestHashRule(t *testing.T) {
+	testCases := []struct {
+		a, b  Rule
+		equal bool
+	}{
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1"},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "bar",
+			}},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Alert: "record", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			true,
+		},
+		{
+			Rule{Alert: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 1"},
+			false,
+		},
+		{
+			Rule{Record: "record", Expr: "up == 1"},
+			Rule{Record: "record", Expr: "up == 2"},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+				"foo": "baz",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"baz": "foo",
+			}},
+			false,
+		},
+		{
+			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"foo": "bar",
+				"baz": "foo",
+			}},
+			Rule{Alert: "alert", Expr: "up == 1"},
+			false,
+		},
+	}
+	for i, tc := range testCases {
+		aID, bID := HashRule(tc.a), HashRule(tc.b)
+		if tc.equal != (aID == bID) {
+			t.Fatalf("missmatch for rule %d", i)
+		}
+	}
 }
 
 func TestGroupChecksum(t *testing.T) {

app/vmalert/config/log/logger.go

@@ -29,7 +29,7 @@ func (l *Logger) isDisabled() bool {
 }
 
 // Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...any) {
+func (l *Logger) Errorf(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -37,7 +37,7 @@ func (l *Logger) Errorf(format string, args ...any) {
 }
 
 // Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...any) {
+func (l *Logger) Warnf(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -45,7 +45,7 @@ func (l *Logger) Warnf(format string, args ...any) {
 }
 
 // Infof logs info message.
-func (l *Logger) Infof(format string, args ...any) {
+func (l *Logger) Infof(format string, args ...interface{}) {
 	if l.isDisabled() {
 		return
 	}
@@ -54,6 +54,6 @@ func (l *Logger) Infof(format string, args ...any) {
 
 // Panicf logs panic message and panics.
 // Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...any) {
+func (l *Logger) Panicf(format string, args ...interface{}) {
 	logger.Panicf(format, args...)
 }

app/vmalert/config/log/logger_test.go

@@ -18,14 +18,14 @@ func TestOutput(t *testing.T) {
 
 	mustMatch := func(exp string) {
 		t.Helper()
-
 		if exp == "" {
 			if testOutput.String() != "" {
-				t.Fatalf("expected output to be empty; got %q", testOutput.String())
+				t.Errorf("expected output to be empty; got %q", testOutput.String())
+				return
 			}
 		}
 		if !strings.Contains(testOutput.String(), exp) {
-			t.Fatalf("output %q should contain %q", testOutput.String(), exp)
+			t.Errorf("output %q should contain %q", testOutput.String(), exp)
 		}
 		fmt.Println(testOutput.String())
 		testOutput.Reset()

app/vmalert/config/types.go

@@ -69,7 +69,7 @@ func (t *Type) ValidateExpr(expr string) error {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
+func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err
@@ -87,7 +87,7 @@ func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
-func (t Type) MarshalYAML() (any, error) {
+func (t Type) MarshalYAML() (interface{}, error) {
 	return t.Name, nil
 }
 
@@ -98,7 +98,7 @@ type Header struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (h *Header) UnmarshalYAML(unmarshal func(any) error) error {
+func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err

app/vmalert/datasource/init.go

@@ -12,12 +12,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -99,9 +98,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
-	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
+	tr.DialContext = httputils.GetStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {

app/vmalert/datasource/vm_prom_api.go

@@ -57,23 +57,20 @@ var jsonParserPool fastjson.ParserPool
 //	[{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]},
 //	{"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}]
 func (pi *promInstant) Unmarshal(b []byte) error {
-	var metrics []json.RawMessage
-	// metrics slice could be large, so parsing it with fastjson could consume a lot of memory.
-	// We parse the slice with standard lib to keep mem usage low.
-	// And each metric object will be parsed with fastjson to reduce allocations.
-	if err := json.Unmarshal(b, &metrics); err != nil {
-		return fmt.Errorf("cannot unmarshal metrics: %w", err)
-	}
-
 	p := jsonParserPool.Get()
 	defer jsonParserPool.Put(p)
 
-	pi.ms = make([]Metric, len(metrics))
-	for i, data := range metrics {
-		row, err := p.ParseBytes(data)
-		if err != nil {
-			return fmt.Errorf("cannot parse metric object: %w", err)
-		}
+	v, err := p.ParseBytes(b)
+	if err != nil {
+		return err
+	}
+
+	rows, err := v.Array()
+	if err != nil {
+		return fmt.Errorf("cannot find the top-level array of result objects: %w", err)
+	}
+	pi.ms = make([]Metric, len(rows))
+	for i, row := range rows {
 		metric := row.Get("metric")
 		if metric == nil {
 			return fmt.Errorf("can't find `metric` object in %q", row)
@@ -122,7 +119,7 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 type promRange struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TVs    [][2]any          `json:"values"`
+		TVs    [][2]interface{}  `json:"values"`
 	} `json:"result"`
 }
 
@@ -150,7 +147,7 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
-type promScalar [2]any
+type promScalar [2]interface{}
 
 func (r promScalar) metrics() ([]Metric, error) {
 	var m Metric

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -1,24 +1,43 @@
 package datasource
 
 import (
+	"bytes"
+	"io"
+	"net/http"
 	"os"
 	"testing"
 )
 
-func BenchmarkPromInstantUnmarshal(b *testing.B) {
+func BenchmarkMetrics(b *testing.B) {
+	payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
+
+	var pi promInstant
+	if err := pi.Unmarshal(payload); err != nil {
+		b.Fatalf(err.Error())
+	}
+	b.Run("Instant", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, _ = pi.metrics()
+		}
+	})
+}
+
+func BenchmarkParsePrometheusResponse(b *testing.B) {
+	req, _ := http.NewRequest("GET", "", nil)
+	resp := &http.Response{StatusCode: http.StatusOK}
 	data, err := os.ReadFile("testdata/instant_response.json")
 	if err != nil {
 		b.Fatalf("error while reading file: %s", err)
 	}
+	resp.Body = io.NopCloser(bytes.NewReader(data))
 
-	// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10                    1760            668959 ns/op          280147 B/op       5781 allocs/op
-	b.Run("Instant std+fastjson", func(b *testing.B) {
+	b.Run("Instant", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			var pi promInstant
-			err = pi.Unmarshal(data)
+			_, err := parsePrometheusResponse(req, resp)
 			if err != nil {
 				b.Fatalf("unexpected parse err: %s", err)
 			}
+			resp.Body = io.NopCloser(bytes.NewReader(data))
 		}
 	})
 }

app/vmalert/datasource/vm_test.go

@@ -31,26 +31,26 @@ var (
 func TestVMInstantQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Fatalf("expected POST method got %s", r.Method)
+			t.Errorf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		timeParam := r.URL.Query().Get("time")
 		if timeParam == "" {
-			t.Fatalf("expected 'time' in query param, got nil instead")
+			t.Errorf("expected 'time' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, timeParam); err != nil {
-			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
+			t.Errorf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
 		case 0:
@@ -197,13 +197,13 @@ func TestVMInstantQuery(t *testing.T) {
 func TestVMInstantQueryWithRetry(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.URL.Query().Get("query") != query {
-			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		switch c {
 		case 0:
@@ -289,37 +289,37 @@ func metricsEqual(t *testing.T, gotM, expectedM []Metric) {
 func TestVMRangeQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Fatalf("expected POST method got %s", r.Method)
+			t.Errorf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		startTS := r.URL.Query().Get("start")
 		if startTS == "" {
-			t.Fatalf("expected 'start' in query param, got nil instead")
+			t.Errorf("expected 'start' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, startTS); err != nil {
-			t.Fatalf("failed to parse 'start' query param: %s", err)
+			t.Errorf("failed to parse 'start' query param: %s", err)
 		}
 		endTS := r.URL.Query().Get("end")
 		if endTS == "" {
-			t.Fatalf("expected 'end' in query param, got nil instead")
+			t.Errorf("expected 'end' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, endTS); err != nil {
-			t.Fatalf("failed to parse 'end' query param: %s", err)
+			t.Errorf("failed to parse 'end' query param: %s", err)
 		}
 		step := r.URL.Query().Get("step")
 		if step != "15s" {
-			t.Fatalf("expected 'step' query param to be 15s; got %q instead", step)
+			t.Errorf("expected 'step' query param to be 15s; got %q instead", step)
 		}
 		switch c {
 		case 0:
@@ -370,299 +370,368 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
-	query := "up"
-	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
-
-	f := func(isQueryRange bool, vm *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
-		t.Helper()
-
-		req, err := vm.newRequest(ctx)
-		if err != nil {
-			t.Fatalf("error in newRequest: %s", err)
-		}
-
-		switch vm.dataSourceType {
-		case "", datasourcePrometheus:
-			if isQueryRange {
-				vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
-			} else {
-				vm.setPrometheusInstantReqParams(req, query, timestamp)
-			}
-		case datasourceGraphite:
-			vm.setGraphiteReqParams(req, query)
-		}
-
-		checkFn(t, req)
-	}
-
 	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+		t.Fatalf("unexpected: %s", err)
 	}
+	query := "up"
+	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
 	storage := VMStorage{
 		extraParams: url.Values{"round_digits": {"10"}},
 	}
-
-	// prometheus path
-	f(false, &VMStorage{
-		dataSourceType: datasourcePrometheus,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/api/v1/query", r.URL.Path)
-	})
-
-	// prometheus prefix
-	f(false, &VMStorage{
-		dataSourceType:   datasourcePrometheus,
-		appendTypePrefix: true,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
-	})
-
-	// prometheus range path
-	f(true, &VMStorage{
-		dataSourceType: datasourcePrometheus,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/api/v1/query_range", r.URL.Path)
-	})
-
-	// prometheus range prefix
-	f(true, &VMStorage{
-		dataSourceType:   datasourcePrometheus,
-		appendTypePrefix: true,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
-	})
-
-	// graphite path
-	f(false, &VMStorage{
-		dataSourceType: datasourceGraphite,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, graphitePath, r.URL.Path)
-	})
-
-	// graphite prefix
-	f(false, &VMStorage{
-		dataSourceType:   datasourceGraphite,
-		appendTypePrefix: true,
-	}, func(t *testing.T, r *http.Request) {
-		checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
-	})
-
-	// default params
-	f(false, &VMStorage{}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// default range params
-	f(true, &VMStorage{}, func(t *testing.T, r *http.Request) {
-		ts := timestamp.Format(time.RFC3339)
-		exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// basic auth
-	f(false, &VMStorage{
-		authCfg: authCfg,
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "foo", u)
-		checkEqualString(t, "bar", p)
-	})
-
-	// basic auth range
-	f(true, &VMStorage{
-		authCfg: authCfg,
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "foo", u)
-		checkEqualString(t, "bar", p)
-	})
-
-	// evaluation interval
-	f(false, &VMStorage{
-		evaluationInterval: 15 * time.Second,
-	}, func(t *testing.T, r *http.Request) {
-		evalInterval := 15 * time.Second
-		exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// step override
-	f(false, &VMStorage{
-		queryStep: time.Minute,
-	}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{
-			"query": {query},
-			"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
-			"time":  {timestamp.Format(time.RFC3339)},
-		}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	//  step to seconds
-	f(false, &VMStorage{
-		evaluationInterval: 3 * time.Hour,
-	}, func(t *testing.T, r *http.Request) {
-		evalInterval := 3 * time.Hour
-		exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// prometheus extra params
-	f(false, &VMStorage{
-		extraParams: url.Values{"round_digits": {"10"}},
-	}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// prometheus extra params range
-	f(true, &VMStorage{
-		extraParams: url.Values{
-			"nocache":      {"1"},
-			"max_lookback": {"1h"},
+	testCases := []struct {
+		name       string
+		queryRange bool
+		vm         *VMStorage
+		checkFn    func(t *testing.T, r *http.Request)
+	}{
+		{
+			"prometheus path",
+			false,
+			&VMStorage{
+				dataSourceType: datasourcePrometheus,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/api/v1/query", r.URL.Path)
+			},
 		},
-	}, func(t *testing.T, r *http.Request) {
-		exp := url.Values{
-			"query":        {query},
-			"end":          {timestamp.Format(time.RFC3339)},
-			"start":        {timestamp.Format(time.RFC3339)},
-			"nocache":      {"1"},
-			"max_lookback": {"1h"},
-		}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// custom params overrides the original params
-	f(false, storage.Clone().ApplyParams(QuerierParams{
-		QueryParams: url.Values{"round_digits": {"2"}},
-	}), func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// allow duplicates in query params
-	f(false, storage.Clone().ApplyParams(QuerierParams{
-		QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
-	}), func(t *testing.T, r *http.Request) {
-		exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
-		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-	})
-
-	// graphite extra params
-	f(false, &VMStorage{
-		dataSourceType: datasourceGraphite,
-		extraParams: url.Values{
-			"nocache":      {"1"},
-			"max_lookback": {"1h"},
+		{
+			"prometheus prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   datasourcePrometheus,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
+			},
 		},
-	}, func(t *testing.T, r *http.Request) {
-		exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
-		checkEqualString(t, exp, r.URL.RawQuery)
-	})
-
-	// graphite extra params allows to override from
-	f(false, &VMStorage{
-		dataSourceType: datasourceGraphite,
-		extraParams: url.Values{
-			"from": {"-10m"},
+		{
+			"prometheus range path",
+			true,
+			&VMStorage{
+				dataSourceType: datasourcePrometheus,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
+			},
 		},
-	}, func(t *testing.T, r *http.Request) {
-		exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
-		checkEqualString(t, exp, r.URL.RawQuery)
-	})
+		{
+			"prometheus range prefix",
+			true,
+			&VMStorage{
+				dataSourceType:   datasourcePrometheus,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
+			},
+		},
+		{
+			"graphite path",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"graphite prefix",
+			false,
+			&VMStorage{
+				dataSourceType:   datasourceGraphite,
+				appendTypePrefix: true,
+			},
+			func(t *testing.T, r *http.Request) {
+				checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
+			},
+		},
+		{
+			"default params",
+			false,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"default range params",
+			true,
+			&VMStorage{},
+			func(t *testing.T, r *http.Request) {
+				ts := timestamp.Format(time.RFC3339)
+				exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"basic auth",
+			false,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"basic auth range",
+			true,
+			&VMStorage{authCfg: authCfg},
+			func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			"evaluation interval",
+			false,
+			&VMStorage{
+				evaluationInterval: 15 * time.Second,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 15 * time.Second
+				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"step override",
+			false,
+			&VMStorage{
+				queryStep: time.Minute,
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{
+					"query": {query},
+					"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
+					"time":  {timestamp.Format(time.RFC3339)},
+				}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"step to seconds",
+			false,
+			&VMStorage{
+				evaluationInterval: 3 * time.Hour,
+			},
+			func(t *testing.T, r *http.Request) {
+				evalInterval := 3 * time.Hour
+				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params",
+			false,
+			&VMStorage{
+				extraParams: url.Values{"round_digits": {"10"}},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"prometheus extra params range",
+			true,
+			&VMStorage{
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{
+					"query":        {query},
+					"end":          {timestamp.Format(time.RFC3339)},
+					"start":        {timestamp.Format(time.RFC3339)},
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"custom params overrides the original params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"round_digits": {"2"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"allow duplicates in query params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
+		{
+			"graphite extra params",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"nocache":      {"1"},
+					"max_lookback": {"1h"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"graphite extra params allows to override from",
+			false,
+			&VMStorage{
+				dataSourceType: datasourceGraphite,
+				extraParams: url.Values{
+					"from": {"-10m"},
+				},
+			},
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			req, err := tc.vm.newRequest(ctx)
+			if err != nil {
+				t.Fatal(err)
+			}
+			switch tc.vm.dataSourceType {
+			case "", datasourcePrometheus:
+				if tc.queryRange {
+					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
+				} else {
+					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
+				}
+			case datasourceGraphite:
+				tc.vm.setGraphiteReqParams(req, query)
+			}
+			tc.checkFn(t, req)
+		})
+	}
 }
 
 func TestHeaders(t *testing.T) {
-	f := func(vmFn func() *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
-		t.Helper()
-
-		vm := vmFn()
-		req, err := vm.newQueryRequest(ctx, "foo", time.Now())
-		if err != nil {
-			t.Fatalf("error in newQueryRequest: %s", err)
-		}
-		checkFn(t, req)
-	}
-
-	// basic auth
-	f(func() *VMStorage {
-		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-		if err != nil {
-			t.Fatalf("Error get auth config: %s", err)
-		}
-		return &VMStorage{authCfg: cfg}
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "foo", u)
-		checkEqualString(t, "bar", p)
-	})
-
-	// bearer auth
-	f(func() *VMStorage {
-		cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
-		if err != nil {
-			t.Fatalf("Error get auth config: %s", err)
-		}
-		return &VMStorage{authCfg: cfg}
-	}, func(t *testing.T, r *http.Request) {
-		reqToken := r.Header.Get("Authorization")
-		splitToken := strings.Split(reqToken, "Bearer ")
-		if len(splitToken) != 2 {
-			t.Fatalf("expected two items got %d", len(splitToken))
-		}
-		token := splitToken[1]
-		checkEqualString(t, "foo", token)
-	})
-
-	// custom extraHeaders
-	f(func() *VMStorage {
-		return &VMStorage{extraHeaders: []keyValue{
-			{key: "Foo", value: "bar"},
-			{key: "Baz", value: "qux"},
-		}}
-	}, func(t *testing.T, r *http.Request) {
-		h1 := r.Header.Get("Foo")
-		checkEqualString(t, "bar", h1)
-		h2 := r.Header.Get("Baz")
-		checkEqualString(t, "qux", h2)
-	})
-
-	// custom header overrides basic auth
-	f(func() *VMStorage {
-		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-		if err != nil {
-			t.Fatalf("Error get auth config: %s", err)
-		}
-		return &VMStorage{
-			authCfg: cfg,
-			extraHeaders: []keyValue{
-				{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+	testCases := []struct {
+		name    string
+		vmFn    func() *VMStorage
+		checkFn func(t *testing.T, r *http.Request)
+	}{
+		{
+			name: "basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
 			},
-		}
-	}, func(t *testing.T, r *http.Request) {
-		u, p, _ := r.BasicAuth()
-		checkEqualString(t, "Aladdin", u)
-		checkEqualString(t, "open sesame", p)
-	})
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "foo", u)
+				checkEqualString(t, "bar", p)
+			},
+		},
+		{
+			name: "bearer auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{authCfg: cfg}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				reqToken := r.Header.Get("Authorization")
+				splitToken := strings.Split(reqToken, "Bearer ")
+				if len(splitToken) != 2 {
+					t.Errorf("expected two items got %d", len(splitToken))
+				}
+				token := splitToken[1]
+				checkEqualString(t, "foo", token)
+			},
+		},
+		{
+			name: "custom extraHeaders",
+			vmFn: func() *VMStorage {
+				return &VMStorage{extraHeaders: []keyValue{
+					{key: "Foo", value: "bar"},
+					{key: "Baz", value: "qux"},
+				}}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				h1 := r.Header.Get("Foo")
+				checkEqualString(t, "bar", h1)
+				h2 := r.Header.Get("Baz")
+				checkEqualString(t, "qux", h2)
+			},
+		},
+		{
+			name: "custom header overrides basic auth",
+			vmFn: func() *VMStorage {
+				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+				if err != nil {
+					t.Errorf("Error get auth config: %s", err)
+				}
+				return &VMStorage{
+					authCfg: cfg,
+					extraHeaders: []keyValue{
+						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+					},
+				}
+			},
+			checkFn: func(t *testing.T, r *http.Request) {
+				u, p, _ := r.BasicAuth()
+				checkEqualString(t, "Aladdin", u)
+				checkEqualString(t, "open sesame", p)
+			},
+		},
+	}
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			vm := tt.vmFn()
+			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
+			if err != nil {
+				t.Fatal(err)
+			}
+			tt.checkFn(t, req)
+		})
+	}
 }
 
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
-
 	if got != exp {
-		t.Fatalf("expected to get: \n%q; \ngot: \n%q", exp, got)
+		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 
 func expectError(t *testing.T, err error, exp string) {
 	t.Helper()
-
 	if err == nil {
-		t.Fatalf("expected non-nil error")
+		t.Errorf("expected non-nil error")
 	}
 	if !strings.Contains(err.Error(), exp) {
-		t.Fatalf("expected error %q to contain %q", err, exp)
+		t.Errorf("expected error %q to contain %q", err, exp)
 	}
 }

app/vmalert/main_test.go

@@ -25,26 +25,26 @@ func TestGetExternalURL(t *testing.T) {
 	invalidURL := "victoriametrics.com/path"
 	_, err := getExternalURL(invalidURL)
 	if err == nil {
-		t.Fatalf("expected error, got nil")
+		t.Errorf("expected error, got nil")
 	}
 
 	expURL := "https://victoriametrics.com/path"
 	u, err := getExternalURL(expURL)
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Fatalf("unexpected url: want %q, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %q, got %s", expURL, u.String())
 	}
 
 	h, _ := os.Hostname()
 	expURL = fmt.Sprintf("http://%s:8880", h)
 	u, err = getExternalURL("")
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Fatalf("unexpected url: want %s, got %s", expURL, u.String())
+		t.Errorf("unexpected url: want %s, got %s", expURL, u.String())
 	}
 }
 
@@ -53,22 +53,22 @@ func TestGetAlertURLGenerator(t *testing.T) {
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
 	if exp != fn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Fatalf("expected template validation error got nil")
+		t.Errorf("expected template validation error got nil")
 	}
 	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
-		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
 

app/vmalert/manager_test.go

@@ -82,8 +82,9 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	wg.Wait()
 }
 
-// TestManagerUpdate tests sequential configuration updates.
-func TestManagerUpdate_Success(t *testing.T) {
+// TestManagerUpdate tests sequential configuration
+// updates.
+func TestManagerUpdate(t *testing.T) {
 	const defaultEvalInterval = time.Second * 30
 	currentEvalInterval := *evaluationInterval
 	*evaluationInterval = defaultEvalInterval
@@ -119,127 +120,145 @@ func TestManagerUpdate_Success(t *testing.T) {
 		}
 	)
 
-	f := func(initPath, updatePath string, groupsExpected []*rule.Group) {
-		t.Helper()
-
-		ctx, cancel := context.WithCancel(context.TODO())
-		m := &manager{
-			groups:         make(map[uint64]*rule.Group),
-			querierBuilder: &datasource.FakeQuerier{},
-			notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
-		}
-
-		cfgInit := loadCfg(t, []string{initPath}, true, true)
-		if err := m.update(ctx, cfgInit, false); err != nil {
-			t.Fatalf("failed to complete initial rules update: %s", err)
-		}
-
-		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
-		if err == nil { // update can fail and that's expected
-			_ = m.update(ctx, cfgUpdate, false)
-		}
-		if len(groupsExpected) != len(m.groups) {
-			t.Fatalf("unexpected number of groups; got %d; want %d", len(m.groups), len(groupsExpected))
-		}
-
-		for _, wantG := range groupsExpected {
-			gotG, ok := m.groups[wantG.ID()]
-			if !ok {
-				t.Fatalf("expected to have group %q", wantG.Name)
-			}
-			compareGroups(t, wantG, gotG)
-		}
-
-		cancel()
-		m.close()
-	}
-
-	// update good rules
-	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules1-good.rules", []*rule.Group{
+	testCases := []struct {
+		name       string
+		initPath   string
+		updatePath string
+		want       []*rule.Group
+	}{
 		{
-			File:     "config/testdata/dir/rules1-good.rules",
-			Name:     "duplicatedGroupDiffFiles",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules: []rule.Rule{
-				&rule.AlertingRule{
-					Name:   "VMRows",
-					Expr:   "vm_rows > 0",
-					For:    5 * time.Minute,
-					Labels: map[string]string{"dc": "gcp", "label": "bar"},
-					Annotations: map[string]string{
-						"summary":     "{{ $value }}",
-						"description": "{{$labels}}",
+			name:       "update good rules",
+			initPath:   "config/testdata/rules/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules1-good.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/dir/rules1-good.rules",
+					Name:     "duplicatedGroupDiffFiles",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules: []rule.Rule{
+						&rule.AlertingRule{
+							Name:   "VMRows",
+							Expr:   "vm_rows > 0",
+							For:    5 * time.Minute,
+							Labels: map[string]string{"dc": "gcp", "label": "bar"},
+							Annotations: map[string]string{
+								"summary":     "{{ $value }}",
+								"description": "{{$labels}}",
+							},
+						},
 					},
 				},
 			},
 		},
-	})
-
-	// update good rules from 1 to 2 groups
-	f("config/testdata/dir/rules/rules1-good.rules", "config/testdata/rules/rules0-good.rules", []*rule.Group{
 		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Name:     "groupGorSingleAlert",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules:    []rule.Rule{VMRows},
-		},
-		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Interval: defaultEvalInterval,
-			Type:     config.NewPrometheusType(),
-			Name:     "TestGroup",
-			Rules: []rule.Rule{
-				Conns,
-				ExampleAlertAlwaysFiring,
+			name:       "update good rules from 1 to 2 groups",
+			initPath:   "config/testdata/dir/rules/rules1-good.rules",
+			updatePath: "config/testdata/rules/rules0-good.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Type:     config.NewPrometheusType(),
+					Name:     "TestGroup",
+					Rules: []rule.Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
 			},
 		},
-	})
-
-	// update with one bad rule file
-	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules2-bad.rules", []*rule.Group{
 		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Name:     "groupGorSingleAlert",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules:    []rule.Rule{VMRows},
-		},
-		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Interval: defaultEvalInterval,
-			Name:     "TestGroup",
-			Type:     config.NewPrometheusType(),
-			Rules: []rule.Rule{
-				Conns,
-				ExampleAlertAlwaysFiring,
+			name:       "update with one bad rule file",
+			initPath:   "config/testdata/rules/rules0-good.rules",
+			updatePath: "config/testdata/dir/rules2-bad.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Name:     "TestGroup",
+					Type:     config.NewPrometheusType(),
+					Rules: []rule.Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
 			},
 		},
-	})
-
-	// update empty dir rules from 0 to 2 groups
-	f("config/testdata/empty/*", "config/testdata/rules/rules0-good.rules", []*rule.Group{
 		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Name:     "groupGorSingleAlert",
-			Type:     config.NewPrometheusType(),
-			Interval: defaultEvalInterval,
-			Rules:    []rule.Rule{VMRows},
-		},
-		{
-			File:     "config/testdata/rules/rules0-good.rules",
-			Interval: defaultEvalInterval,
-			Type:     config.NewPrometheusType(),
-			Name:     "TestGroup",
-			Rules: []rule.Rule{
-				Conns,
-				ExampleAlertAlwaysFiring,
+			name:       "update empty dir rules from 0 to 2 groups",
+			initPath:   "config/testdata/empty/*",
+			updatePath: "config/testdata/rules/rules0-good.rules",
+			want: []*rule.Group{
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Name:     "groupGorSingleAlert",
+					Type:     config.NewPrometheusType(),
+					Interval: defaultEvalInterval,
+					Rules:    []rule.Rule{VMRows},
+				},
+				{
+					File:     "config/testdata/rules/rules0-good.rules",
+					Interval: defaultEvalInterval,
+					Type:     config.NewPrometheusType(),
+					Name:     "TestGroup",
+					Rules: []rule.Rule{
+						Conns,
+						ExampleAlertAlwaysFiring,
+					},
+				},
 			},
 		},
-	})
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+			m := &manager{
+				groups:         make(map[uint64]*rule.Group),
+				querierBuilder: &datasource.FakeQuerier{},
+				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
+			}
+
+			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
+			if err := m.update(ctx, cfgInit, false); err != nil {
+				t.Fatalf("failed to complete initial rules update: %s", err)
+			}
+
+			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
+			if err == nil { // update can fail and that's expected
+				_ = m.update(ctx, cfgUpdate, false)
+			}
+			if len(tc.want) != len(m.groups) {
+				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
+			}
+
+			for _, wantG := range tc.want {
+				gotG, ok := m.groups[wantG.ID()]
+				if !ok {
+					t.Fatalf("expected to have group %q", wantG.Name)
+				}
+				compareGroups(t, wantG, gotG)
+			}
+
+			cancel()
+			m.close()
+		})
+	}
 }
-
 func compareGroups(t *testing.T, a, b *rule.Group) {
 	t.Helper()
 	if a.Name != b.Name {
@@ -266,59 +285,82 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
 	}
 }
 
-func TestManagerUpdate_Failure(t *testing.T) {
-	f := func(notifiers []notifier.Notifier, rw remotewrite.RWClient, cfg config.Group, errStrExpected string) {
-		t.Helper()
-
-		m := &manager{
-			groups:         make(map[uint64]*rule.Group),
-			querierBuilder: &datasource.FakeQuerier{},
-			rw:             rw,
-		}
-		if notifiers != nil {
-			m.notifiers = func() []notifier.Notifier { return notifiers }
-		}
-		err := m.update(context.Background(), []config.Group{cfg}, false)
-		if err == nil {
-			t.Fatalf("expected to get error; got nil")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
-		}
+func TestManagerUpdateNegative(t *testing.T) {
+	testCases := []struct {
+		notifiers []notifier.Notifier
+		rw        remotewrite.RWClient
+		cfg       config.Group
+		expErr    string
+	}{
+		{
+			nil,
+			nil,
+			config.Group{
+				Name: "Recording rule only",
+				Rules: []config.Rule{
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			nil,
+			config.Group{
+				Name: "Alerting rule only",
+				Rules: []config.Rule{
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
+		{
+			[]notifier.Notifier{&notifier.FakeNotifier{}},
+			nil,
+			config.Group{
+				Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Alert: "alert1", Expr: "up > 0"},
+					{Alert: "alert2", Expr: "up > 0"},
+					{Record: "record", Expr: "max(up)"},
+				},
+			},
+			"contains recording rules",
+		},
+		{
+			nil,
+			&remotewrite.Client{},
+			config.Group{
+				Name: "Recording and alerting rules",
+				Rules: []config.Rule{
+					{Record: "record1", Expr: "max(up)"},
+					{Record: "record2", Expr: "max(up)"},
+					{Alert: "alert", Expr: "up > 0"},
+				},
+			},
+			"contains alerting rules",
+		},
 	}
 
-	f(nil, nil, config.Group{
-		Name: "Recording rule only",
-		Rules: []config.Rule{
-			{Record: "record", Expr: "max(up)"},
-		},
-	}, "contains recording rules")
-
-	f(nil, nil, config.Group{
-		Name: "Alerting rule only",
-		Rules: []config.Rule{
-			{Alert: "alert", Expr: "up > 0"},
-		},
-	}, "contains alerting rules")
-
-	f([]notifier.Notifier{&notifier.FakeNotifier{}}, nil, config.Group{
-		Name: "Recording and alerting rules",
-		Rules: []config.Rule{
-			{Alert: "alert1", Expr: "up > 0"},
-			{Alert: "alert2", Expr: "up > 0"},
-			{Record: "record", Expr: "max(up)"},
-		},
-	}, "contains recording rules")
-
-	f(nil, &remotewrite.Client{}, config.Group{
-		Name: "Recording and alerting rules",
-		Rules: []config.Rule{
-			{Record: "record1", Expr: "max(up)"},
-			{Record: "record2", Expr: "max(up)"},
-			{Alert: "alert", Expr: "up > 0"},
-		},
-	}, "contains alerting rules")
+	for _, tc := range testCases {
+		t.Run(tc.cfg.Name, func(t *testing.T) {
+			m := &manager{
+				groups:         make(map[uint64]*rule.Group),
+				querierBuilder: &datasource.FakeQuerier{},
+				rw:             tc.rw,
+			}
+			if tc.notifiers != nil {
+				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
+			}
+			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
+			if err == nil {
+				t.Fatalf("expected to get error; got nil")
+			}
+			if !strings.Contains(err.Error(), tc.expErr) {
+				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
+			}
+		})
+	}
 }
 
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {

app/vmalert/notifier/alert_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func TestAlertExecTemplate(t *testing.T) {
+func TestAlert_ExecTemplate(t *testing.T) {
 	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
@@ -23,164 +23,201 @@ func TestAlertExecTemplate(t *testing.T) {
 	_, err := Init(nil, extLabels, extURL)
 	checkErr(t, err)
 
-	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
-		t.Helper()
-
-		if err := ValidateTemplates(annotations); err != nil {
-			t.Fatalf("cannot validate annotations: %s", err)
-		}
-
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return []datasource.Metric{
-				{
-					Labels: []datasource.Label{
-						{Name: "foo", Value: "bar"},
-						{Name: "baz", Value: "qux"},
-					},
-					Values:     []float64{1},
-					Timestamps: []int64{1},
+	testCases := []struct {
+		name        string
+		alert       *Alert
+		annotations map[string]string
+		expTpl      map[string]string
+	}{
+		{
+			name:        "empty-alert",
+			alert:       &Alert{},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			name: "no-template",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"instance": "localhost",
 				},
-				{
-					Labels: []datasource.Label{
-						{Name: "foo", Value: "garply"},
-						{Name: "baz", Value: "fred"},
-					},
-					Values:     []float64{2},
-					Timestamps: []int64{1},
+			},
+			annotations: map[string]string{},
+			expTpl:      map[string]string{},
+		},
+		{
+			name: "label-template",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
 				},
-			}, nil
-		}
-
-		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
-		if err != nil {
-			t.Fatalf("cannot execute template: %s", err)
-		}
-		if len(tpl) != len(tplExpected) {
-			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
-		}
-		for k := range tplExpected {
-			got, exp := tpl[k], tplExpected[k]
-			if got != exp {
-				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
-			}
-		}
+				For: 5 * time.Minute,
+			},
+			annotations: map[string]string{
+				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+			},
+			expTpl: map[string]string{
+				"summary":     "Too high connection number for localhost for job staging",
+				"description": "It is 10000 connections for localhost for more than 5m0s",
+			},
+		},
+		{
+			name: "expression-template",
+			alert: &Alert{
+				Expr: `vm_rows{"label"="bar"}<0`,
+			},
+			annotations: map[string]string{
+				"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+				"exprEscapedPath":   "{{ $expr|pathEscape }}",
+				"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+				"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+				"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+			},
+			expTpl: map[string]string{
+				"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+				"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+				"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+				"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+				"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+			},
+		},
+		{
+			name:  "query",
+			alert: &Alert{Expr: `vm_rows{"label"="bar"}>0`},
+			annotations: map[string]string{
+				"summary": `{{ query "foo" | first | value }}`,
+				"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+			},
+			expTpl: map[string]string{
+				"summary": "1",
+				"desc":    "bar 1;garply 2;",
+			},
+		},
+		{
+			name: "external",
+			alert: &Alert{
+				Value: 1e4,
+				Labels: map[string]string{
+					"job":      "staging",
+					"instance": "localhost",
+				},
+			},
+			annotations: map[string]string{
+				"url":         "{{ $externalURL }}",
+				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+			},
+			expTpl: map[string]string{
+				"url":         extURL,
+				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+			},
+		},
+		{
+			name: "alert and group IDs",
+			alert: &Alert{
+				ID:      42,
+				GroupID: 24,
+			},
+			annotations: map[string]string{
+				"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+			},
+			expTpl: map[string]string{
+				"url": "/api/v1/alert?alertID=42&groupID=24",
+			},
+		},
+		{
+			name: "ActiveAt time",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+			},
+			expTpl: map[string]string{
+				"diagram": "![](http://example.com?render=1660941298",
+			},
+		},
+		{
+			name:  "ActiveAt time is nil",
+			alert: &Alert{},
+			annotations: map[string]string{
+				"default_time": "{{$activeAt}}",
+			},
+			expTpl: map[string]string{
+				"default_time": "0001-01-01 00:00:00 +0000 UTC",
+			},
+		},
+		{
+			name: "ActiveAt custom format",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+			},
+			expTpl: map[string]string{
+				"fire_time": "2022/08/19 20:34:58",
+			},
+		},
+		{
+			name: "ActiveAt query range",
+			alert: &Alert{
+				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+			},
+			annotations: map[string]string{
+				"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+			},
+			expTpl: map[string]string{
+				"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+			},
+		},
 	}
 
-	// empty-alert
-	f(&Alert{}, map[string]string{}, map[string]string{})
-
-	// no-template
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"instance": "localhost",
-		},
-	}, map[string]string{}, map[string]string{})
-
-	// label-template
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-		For: 5 * time.Minute,
-	}, map[string]string{
-		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-	}, map[string]string{
-		"summary":     "Too high connection number for localhost for job staging",
-		"description": "It is 10000 connections for localhost for more than 5m0s",
-	})
-
-	// expression-template
-	f(&Alert{
-		Expr: `vm_rows{"label"="bar"}<0`,
-	}, map[string]string{
-		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-		"exprEscapedPath":   "{{ $expr|pathEscape }}",
-		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-	}, map[string]string{
-		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-	})
-
-	// query
-	f(&Alert{
-		Expr: `vm_rows{"label"="bar"}>0`,
-	}, map[string]string{
-		"summary": `{{ query "foo" | first | value }}`,
-		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-	}, map[string]string{
-		"summary": "1",
-		"desc":    "bar 1;garply 2;",
-	})
-
-	// external
-	f(&Alert{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"url":         "{{ $externalURL }}",
-		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-	}, map[string]string{
-		"url":         extURL,
-		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-	})
-
-	// alert and group IDs
-	f(&Alert{
-		ID:      42,
-		GroupID: 24,
-	}, map[string]string{
-		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-	}, map[string]string{
-		"url": "/api/v1/alert?alertID=42&groupID=24",
-	})
-
-	// ActiveAt time
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-	}, map[string]string{
-		"diagram": "![](http://example.com?render=1660941298",
-	})
-
-	// ActiveAt time is nil
-	f(&Alert{}, map[string]string{
-		"default_time": "{{$activeAt}}",
-	}, map[string]string{
-		"default_time": "0001-01-01 00:00:00 +0000 UTC",
-	})
-
-	// ActiveAt custom format
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-	}, map[string]string{
-		"fire_time": "2022/08/19 20:34:58",
-	})
-
-	// ActiveAt query range
-	f(&Alert{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-	}, map[string]string{
-		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-	})
+	qFn := func(_ string) ([]datasource.Metric, error) {
+		return []datasource.Metric{
+			{
+				Labels: []datasource.Label{
+					{Name: "foo", Value: "bar"},
+					{Name: "baz", Value: "qux"},
+				},
+				Values:     []float64{1},
+				Timestamps: []int64{1},
+			},
+			{
+				Labels: []datasource.Label{
+					{Name: "foo", Value: "garply"},
+					{Name: "baz", Value: "fred"},
+				},
+				Values:     []float64{2},
+				Timestamps: []int64{1},
+			},
+		}, nil
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if err := ValidateTemplates(tc.annotations); err != nil {
+				t.Fatal(err)
+			}
+			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(tpl) != len(tc.expTpl) {
+				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
+			}
+			for k := range tc.expTpl {
+				got, exp := tpl[k], tc.expTpl[k]
+				if got != exp {
+					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
+				}
+			}
+		})
+	}
 }
 
 func TestAlert_toPromLabels(t *testing.T) {

app/vmalert/notifier/alertmanager.go

@@ -76,6 +76,9 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
 
 	if am.timeout > 0 {
 		var cancel context.CancelFunc
@@ -91,11 +94,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 			return err
 		}
 	}
-	// external headers have higher priority
-	for key, value := range headers {
-		req.Header.Set(key, value)
-	}
-
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -132,8 +130,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
-
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 
 	ba := new(promauth.BasicAuthConfig)
@@ -148,9 +145,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
-		utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
-	)
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/alertmanager_test.go

@@ -3,7 +3,6 @@ package notifier
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -17,10 +16,10 @@ func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
 	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+		t.Errorf("unexpected error: %s", err)
 	}
 	if am.Addr() != addr {
-		t.Fatalf("expected to have %q; got %q", addr, am.Addr())
+		t.Errorf("expected to have %q; got %q", addr, am.Addr())
 	}
 }
 
@@ -29,29 +28,27 @@ func TestAlertManager_Send(t *testing.T) {
 	const headerKey, headerValue = "TenantID", "foo"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Fatalf("should not be called")
+		t.Errorf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
 		user, pass, ok := r.BasicAuth()
 		if !ok {
-			t.Fatalf("unauthorized request")
+			t.Errorf("unauthorized request")
 		}
 		if user != baUser || pass != baPass {
-			t.Fatalf("wrong creds %q:%q; expected %q:%q", user, pass, baUser, baPass)
+			t.Errorf("wrong creds %q:%q; expected %q:%q",
+				user, pass, baUser, baPass)
 		}
 		c++
 		if r.Method != http.MethodPost {
-			t.Fatalf("expected POST method got %s", r.Method)
+			t.Errorf("expected POST method got %s", r.Method)
 		}
 		switch c {
 		case 0:
 			conn, _, _ := w.(http.Hijacker).Hijack()
 			_ = conn.Close()
 		case 1:
-			if r.Header.Get(headerKey) != headerValue {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
-			}
 			w.WriteHeader(500)
 		case 2:
 			var a []struct {
@@ -62,26 +59,25 @@ func TestAlertManager_Send(t *testing.T) {
 				GeneratorURL string            `json:"generatorURL"`
 			}
 			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
-				t.Fatalf("can not unmarshal data into alert %s", err)
+				t.Errorf("can not unmarshal data into alert %s", err)
+				t.FailNow()
 			}
 			if len(a) != 1 {
-				t.Fatalf("expected 1 alert in array got %d", len(a))
+				t.Errorf("expected 1 alert in array got %d", len(a))
 			}
 			if a[0].GeneratorURL != "0/0" {
-				t.Fatalf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
+				t.Errorf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
 			}
 			if a[0].StartsAt.IsZero() {
-				t.Fatalf("expected non-zero start time")
+				t.Errorf("expected non-zero start time")
 			}
 			if a[0].EndAt.IsZero() {
-				t.Fatalf("expected non-zero end time")
-			}
-			if r.Header.Get(headerKey) != "bar" {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+				t.Errorf("expected non-zero end time")
 			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
-				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+				t.Errorf("expected header %q to be set to %q; got %q instead",
+					headerKey, headerValue, r.Header.Get(headerKey))
 			}
 		}
 	})
@@ -93,19 +89,18 @@ func TestAlertManager_Send(t *testing.T) {
 			Username: baUser,
 			Password: promauth.NewSecret(baPass),
 		},
-		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
 	}, aCfg, nil, 0)
 	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
+		t.Errorf("unexpected error: %s", err)
 	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
-		t.Fatalf("expected connection error got nil")
+		t.Error("expected connection error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
-		t.Fatalf("expected wrong http code error got nil")
+		t.Error("expected wrong http code error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{{
 		GroupID:     0,
@@ -113,13 +108,13 @@ func TestAlertManager_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, map[string]string{headerKey: "bar"}); err != nil {
-		t.Fatalf("unexpected error %s", err)
+	}}, nil); err != nil {
+		t.Errorf("unexpected error %s", err)
 	}
 	if c != 2 {
-		t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
+		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
 	}
 	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 }

app/vmalert/notifier/config.go

@@ -51,7 +51,7 @@ type Config struct {
 	Checksum string
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]any `yaml:",inline"`
+	XXX map[string]interface{} `yaml:",inline"`
 
 	// This is set to the directory from where the config has been loaded.
 	baseDir string
@@ -73,7 +73,7 @@ type StaticConfig struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
+func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type config Config
 	if err := unmarshal((*config)(cfg)); err != nil {
 		return err

app/vmalert/notifier/config_test.go

@@ -5,14 +5,10 @@ import (
 	"testing"
 )
 
-func TestParseConfig_Success(t *testing.T) {
+func TestConfigParseGood(t *testing.T) {
 	f := func(path string) {
-		t.Helper()
-
 		_, err := parseConfig(path)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
+		checkErr(t, err)
 	}
 	f("testdata/mixed.good.yaml")
 	f("testdata/consul.good.yaml")
@@ -20,16 +16,14 @@ func TestParseConfig_Success(t *testing.T) {
 	f("testdata/static.good.yaml")
 }
 
-func TestParseConfig_Failure(t *testing.T) {
+func TestConfigParseBad(t *testing.T) {
 	f := func(path, expErr string) {
-		t.Helper()
-
 		_, err := parseConfig(path)
 		if err == nil {
 			t.Fatalf("expected to get non-nil err for config %q", path)
 		}
 		if !strings.Contains(err.Error(), expErr) {
-			t.Fatalf("expected err to contain %q; got %q instead", expErr, err)
+			t.Errorf("expected err to contain %q; got %q instead", expErr, err)
 		}
 	}
 

app/vmalert/notifier/config_watcher_test.go

@@ -319,41 +319,46 @@ func TestMergeHTTPClientConfigs(t *testing.T) {
 	}
 }
 
-func TestParseLabels_Failure(t *testing.T) {
-	f := func(target string, cfg *Config) {
-		t.Helper()
-
-		_, _, err := parseLabels(target, nil, cfg)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
+func TestParseLabels(t *testing.T) {
+	testCases := []struct {
+		name            string
+		target          string
+		cfg             *Config
+		expectedAddress string
+		expectedErr     bool
+	}{
+		{
+			"invalid address",
+			"invalid:*//url",
+			&Config{},
+			"",
+			true,
+		},
+		{
+			"use some default params",
+			"alertmanager:9093",
+			&Config{PathPrefix: "test"},
+			"http://alertmanager:9093/test/api/v2/alerts",
+			false,
+		},
+		{
+			"use target address",
+			"https://alertmanager:9093/api/v1/alerts",
+			&Config{Scheme: "http", PathPrefix: "test"},
+			"https://alertmanager:9093/api/v1/alerts",
+			false,
+		},
 	}
 
-	// invalid address
-	f("invalid:*//url", &Config{})
-}
-
-func TestParseLabels_Success(t *testing.T) {
-	f := func(target string, cfg *Config, expectedAddress string) {
-		t.Helper()
-
-		address, _, err := parseLabels(target, nil, cfg)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if address != expectedAddress {
-			t.Fatalf("unexpected address; got %q; want %q", address, expectedAddress)
-		}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			address, _, err := parseLabels(tc.target, nil, tc.cfg)
+			if err == nil == tc.expectedErr {
+				t.Fatalf("unexpected error; got %t; want %t", err != nil, tc.expectedErr)
+			}
+			if address != tc.expectedAddress {
+				t.Fatalf("unexpected address; got %q; want %q", address, tc.expectedAddress)
+			}
+		})
 	}
-
-	// use some default params
-	f("alertmanager:9093", &Config{
-		PathPrefix: "test",
-	}, "http://alertmanager:9093/test/api/v2/alerts")
-
-	// use target address
-	f("https://alertmanager:9093/api/v1/alerts", &Config{
-		Scheme:     "http",
-		PathPrefix: "test",
-	}, "https://alertmanager:9093/api/v1/alerts")
 }

app/vmalert/notifier/init.go

@@ -25,9 +25,6 @@ var (
 		"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
 		"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
 
-	headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
-		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
-		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
@@ -174,7 +171,6 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
 				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
 			},
-			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
 		addr = strings.TrimSuffix(addr, "/")

app/vmalert/notifier/notifier_blackhole_test.go

@@ -17,12 +17,12 @@ func TestBlackHoleNotifier_Send(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 
 	alertCount := bh.metrics.alertsSent.Get()
 	if alertCount != 1 {
-		t.Fatalf("expect value 1; instead got %d", alertCount)
+		t.Errorf("expect value 1; instead got %d", alertCount)
 	}
 }
 
@@ -35,7 +35,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Fatalf("unexpected error %s", err)
+		t.Errorf("unexpected error %s", err)
 	}
 
 	bh.Close()
@@ -44,7 +44,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 	alertMetricName := "vmalert_alerts_sent_total{addr=\"blackhole\"}"
 	for _, name := range defaultMetrics.ListMetricNames() {
 		if name == alertMetricName {
-			t.Fatalf("Metric name should have unregistered.But still present")
+			t.Errorf("Metric name should have unregistered.But still present")
 		}
 	}
 }

app/vmalert/remoteread/init.go

@@ -10,14 +10,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 
 	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
@@ -68,10 +67,10 @@ func Init() (datasource.QuerierBuilder, error) {
 	}
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
-	tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")
+	tr.DialContext = httputils.GetStatDialFunc("vmalert_remoteread")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/remotewrite/client_test.go

@@ -44,7 +44,8 @@ func TestClient_Push(t *testing.T) {
 	}
 
 	r := rand.New(rand.NewSource(1))
-	const rowsN = int(1e4)
+	const rowsN = 1e4
+	var sent int
 	for i := 0; i < rowsN; i++ {
 		s := prompbmarshal.TimeSeries{
 			Samples: []prompbmarshal.Sample{{
@@ -56,11 +57,17 @@ func TestClient_Push(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected err: %s", err)
 		}
+		if err == nil {
+			sent++
+		}
 		err = faultyClient.Push(s)
 		if err != nil {
 			t.Fatalf("unexpected err: %s", err)
 		}
 	}
+	if sent == 0 {
+		t.Fatalf("0 series sent")
+	}
 	if err := client.Close(); err != nil {
 		t.Fatalf("failed to close client: %s", err)
 	}
@@ -68,66 +75,77 @@ func TestClient_Push(t *testing.T) {
 		t.Fatalf("failed to close faulty client: %s", err)
 	}
 	got := testSrv.accepted()
-	if got != rowsN {
-		t.Fatalf("expected to have %d series; got %d", rowsN, got)
+	if got != sent {
+		t.Fatalf("expected to have %d series; got %d", sent, got)
 	}
 	got = faultySrv.accepted()
-	if got != rowsN {
-		t.Fatalf("expected to have %d series for faulty client; got %d", rowsN, got)
+	if got != sent {
+		t.Fatalf("expected to have %d series for faulty client; got %d", sent, got)
 	}
 }
 
 func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
-	const batchSize = 20
+	batchSize := 20
 
-	f := func(pushCnt, batchCntExpected int) {
-		t.Helper()
-
-		// run new server
-		bcServer := newBatchCntRWServer()
-
-		// run new client
-		rwClient, err := NewClient(context.Background(), Config{
-			MaxBatchSize: batchSize,
-
-			// Set everything to 1 to simplify the calculation.
-			Concurrency:   1,
-			MaxQueueSize:  1000,
-			FlushInterval: time.Minute,
-
-			// batch count server
-			Addr: bcServer.URL,
-		})
-		if err != nil {
-			t.Fatalf("cannot create remote write client: %s", err)
-		}
-
-		// push time series to the client.
-		for i := 0; i < pushCnt; i++ {
-			if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
-				t.Fatalf("cannot time series to the client: %s", err)
-			}
-		}
-
-		// close the client so the rest ts will be flushed in `shutdown`
-		if err = rwClient.Close(); err != nil {
-			t.Fatalf("cannot shutdown client: %s", err)
-		}
-
-		// finally check how many batches is sent.
-		if bcServer.acceptedBatches() != batchCntExpected {
-			t.Fatalf("client sent batch count incorrect; got %d; want %d", bcServer.acceptedBatches(), batchCntExpected)
-		}
-		if pushCnt != bcServer.accepted() {
-			t.Fatalf("client sent time series count incorrect; got %d; want %d", bcServer.accepted(), pushCnt)
-		}
+	testTable := []struct {
+		name     string // name of the test case
+		pushCnt  int    // how many time series is pushed to the client
+		batchCnt int    // the expected batch count sent by the client
+	}{
+		{
+			name:     "pushCnt % batchSize == 0",
+			pushCnt:  batchSize * 40,
+			batchCnt: 40,
+		},
+		{
+			name:     "pushCnt % batchSize != 0",
+			pushCnt:  batchSize*40 + 1,
+			batchCnt: 40 + 1,
+		},
 	}
 
-	// pushCnt % batchSize == 0
-	f(batchSize*40, 40)
+	for _, tt := range testTable {
+		t.Run(tt.name, func(t *testing.T) {
+			// run new server
+			bcServer := newBatchCntRWServer()
 
-	//pushCnt % batchSize != 0
-	f(batchSize*40+1, 40+1)
+			// run new client
+			rwClient, err := NewClient(context.Background(), Config{
+				MaxBatchSize: batchSize,
+
+				// Set everything to 1 to simplify the calculation.
+				Concurrency:   1,
+				MaxQueueSize:  1000,
+				FlushInterval: time.Minute,
+
+				// batch count server
+				Addr: bcServer.URL,
+			})
+			if err != nil {
+				t.Fatalf("new remote write client failed, err: %v", err)
+			}
+
+			// push time series to the client.
+			for i := 0; i < tt.pushCnt; i++ {
+				if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
+					t.Fatalf("push time series to the client failed, err: %v", err)
+				}
+			}
+
+			// close the client so the rest ts will be flushed in `shutdown`
+			if err = rwClient.Close(); err != nil {
+				t.Fatalf("shutdown client failed, err: %v", err)
+			}
+
+			// finally check how many batches is sent.
+			if tt.batchCnt != bcServer.acceptedBatches() {
+				t.Errorf("client sent batch count incorrect, want: %d, get: %d", tt.batchCnt, bcServer.acceptedBatches())
+			}
+			if tt.pushCnt != bcServer.accepted() {
+				t.Errorf("client sent time series count incorrect, want: %d, get: %d", tt.pushCnt, bcServer.accepted())
+			}
+		})
+	}
 }
 
 func newRWServer() *rwServer {

app/vmalert/remotewrite/debug_client.go

@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	c := &DebugClient{
 		c: &http.Client{

app/vmalert/remotewrite/init.go

@@ -9,13 +9,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. "+
-		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -72,10 +71,10 @@ func Init(ctx context.Context) (*Client, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
+		return nil, fmt.Errorf("failed to create transport: %w", err)
 	}
 	t.IdleConnTimeout = *idleConnectionTimeout
-	t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")
+	t.DialContext = httputils.GetStatDialFunc("vmalert_remotewrite")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/replay_test.go

@@ -39,102 +39,135 @@ func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to ti
 }
 
 func TestReplay(t *testing.T) {
-	f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
-		t.Helper()
-
-		fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
-		retriesOrig, delayOrig := *replayRuleRetryAttempts, *replayRulesDelay
-		defer func() {
-			*replayFrom, *replayTo = fromOrig, toOrig
-			*replayMaxDatapoints, *replayRuleRetryAttempts = maxDatapointsOrig, retriesOrig
-			*replayRulesDelay = delayOrig
-		}()
-
-		*replayRuleRetryAttempts = 1
-		*replayRulesDelay = time.Millisecond
-		rwb := &remotewrite.DebugClient{}
-		*replayFrom = from
-		*replayTo = to
-		*replayMaxDatapoints = maxDP
-		if err := replay(cfg, qb, rwb); err != nil {
-			t.Fatalf("replay failed: %s", err)
-		}
-		if len(qb.registry) > 0 {
-			t.Fatalf("not all requests were sent: %#v", qb.registry)
-		}
+	testCases := []struct {
+		name     string
+		from, to string
+		maxDP    int
+		cfg      []config.Group
+		qb       *fakeReplayQuerier
+	}{
+		{
+			name:  "one rule + one response",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:00.000Z",
+			maxDP: 10,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {"12:00:00+12:02:00": {}},
+				},
+			},
+		},
+		{
+			name:  "one rule + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "datapoints per step",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T15:02:30.000Z",
+			maxDP: 60,
+			cfg: []config.Group{
+				{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+13:00:00": {},
+						"13:00:00+14:00:00": {},
+						"14:00:00+15:00:00": {},
+						"15:00:00+15:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "multiple recording rules + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+				{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+					"max(up)": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
+		{
+			name:  "multiple alerting rules + multiple responses",
+			from:  "2021-01-01T12:00:00.000Z",
+			to:    "2021-01-01T12:02:30.000Z",
+			maxDP: 1,
+			cfg: []config.Group{
+				{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
+				{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
+			},
+			qb: &fakeReplayQuerier{
+				registry: map[string]map[string]struct{}{
+					"sum(up) > 1": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+					"max(up) < 1": {
+						"12:00:00+12:01:00": {},
+						"12:01:00+12:02:00": {},
+						"12:02:00+12:02:30": {},
+					},
+				},
+			},
+		},
 	}
 
-	// one rule + one response
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
-		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {"12:00:00+12:02:00": {}},
-		},
-	})
+	from, to, maxDP := *replayFrom, *replayTo, *replayMaxDatapoints
+	retries, delay := *replayRuleRetryAttempts, *replayRulesDelay
+	defer func() {
+		*replayFrom, *replayTo = from, to
+		*replayMaxDatapoints, *replayRuleRetryAttempts = maxDP, retries
+		*replayRulesDelay = delay
+	}()
 
-	// one rule + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
-		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-		},
-	})
-
-	// datapoints per step
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
-		{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {
-				"12:00:00+13:00:00": {},
-				"13:00:00+14:00:00": {},
-				"14:00:00+15:00:00": {},
-				"15:00:00+15:02:30": {},
-			},
-		},
-	})
-
-	// multiple recording rules + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
-		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-		{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up)": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-			"max(up)": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-		},
-	})
-
-	// multiple alerting rules + multiple responses
-	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
-		{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
-		{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
-	}, &fakeReplayQuerier{
-		registry: map[string]map[string]struct{}{
-			"sum(up) > 1": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-			"max(up) < 1": {
-				"12:00:00+12:01:00": {},
-				"12:01:00+12:02:00": {},
-				"12:02:00+12:02:30": {},
-			},
-		},
-	})
+	*replayRuleRetryAttempts = 1
+	*replayRulesDelay = time.Millisecond
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			*replayFrom = tc.from
+			*replayTo = tc.to
+			*replayMaxDatapoints = tc.maxDP
+			if err := replay(tc.cfg, tc.qb, &remotewrite.DebugClient{}); err != nil {
+				t.Fatalf("replay failed: %s", err)
+			}
+			if len(tc.qb.registry) > 0 {
+				t.Fatalf("not all requests were sent: %#v", tc.qb.registry)
+			}
+		})
+	}
 }

app/vmalert/rule/alerting.go

@@ -183,7 +183,7 @@ func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
 	return ar.alerts[id]
 }
 
-func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...any) {
+func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
 	if !ar.Debug {
 		return
 	}
@@ -441,6 +441,9 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			}
 			a.Value = m.Values[0]
 			a.Annotations = annotations
+			if err != nil {
+				return nil, err
+			}
 			a.KeepFiringSince = time.Time{}
 			continue
 		}

app/vmalert/rule/group.go

@@ -475,7 +475,7 @@ func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *
 	return randSleep
 }
 
-func (g *Group) infof(format string, args ...any) {
+func (g *Group) infof(format string, args ...interface{}) {
 	msg := fmt.Sprintf(format, args...)
 	logger.Infof("group %q %s; interval=%v; eval_offset=%v; concurrency=%d",
 		g.Name, msg, g.Interval, g.EvalOffset, g.Concurrency)

app/vmalert/rule/group_test.go

@@ -37,142 +37,158 @@ func TestMain(m *testing.M) {
 }
 
 func TestUpdateWith(t *testing.T) {
-	f := func(currentRules, newRules []config.Rule) {
-		t.Helper()
-
-		g := &Group{
-			Name: "test",
-		}
-		qb := &datasource.FakeQuerier{}
-		for _, r := range currentRules {
-			r.ID = config.HashRule(r)
-			g.Rules = append(g.Rules, g.newRule(qb, r))
-		}
-
-		ng := &Group{
-			Name: "test",
-		}
-		for _, r := range newRules {
-			r.ID = config.HashRule(r)
-			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
-		}
-
-		err := g.updateWith(ng)
-		if err != nil {
-			t.Fatalf("cannot update rule: %s", err)
-		}
-
-		if len(g.Rules) != len(newRules) {
-			t.Fatalf("expected to have %d rules; got: %d", len(g.Rules), len(newRules))
-		}
-		sort.Slice(g.Rules, func(i, j int) bool {
-			return g.Rules[i].ID() < g.Rules[j].ID()
-		})
-		sort.Slice(ng.Rules, func(i, j int) bool {
-			return ng.Rules[i].ID() < ng.Rules[j].ID()
-		})
-		for i, r := range g.Rules {
-			got, want := r, ng.Rules[i]
-			if got.ID() != want.ID() {
-				t.Fatalf("expected to have rule %q; got %q", want, got)
-			}
-			if err := CompareRules(t, got, want); err != nil {
-				t.Fatalf("comparison error: %s", err)
-			}
-		}
+	testCases := []struct {
+		name         string
+		currentRules []config.Rule
+		newRules     []config.Rule
+	}{
+		{
+			"new rule",
+			nil,
+			[]config.Rule{{Alert: "bar"}},
+		},
+		{
+			"update alerting rule",
+			[]config.Rule{
+				{
+					Alert: "foo",
+					Expr:  "up > 0",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+					Annotations: map[string]string{
+						"summary":     "{{ $value|humanize }}",
+						"description": "{{$labels}}",
+					},
+				},
+				{
+					Alert: "bar",
+					Expr:  "up > 0",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+				},
+			},
+			[]config.Rule{
+				{
+					Alert: "foo",
+					Expr:  "up > 10",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"baz": "bar",
+					},
+					Annotations: map[string]string{
+						"summary": "none",
+					},
+				},
+				{
+					Alert:         "bar",
+					Expr:          "up > 0",
+					For:           promutils.NewDuration(2 * time.Second),
+					KeepFiringFor: promutils.NewDuration(time.Minute),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+				},
+			},
+		},
+		{
+			"update recording rule",
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "max(up)",
+				Labels: map[string]string{
+					"bar": "baz",
+				},
+			}},
+			[]config.Rule{{
+				Record: "foo",
+				Expr:   "min(up)",
+				Labels: map[string]string{
+					"baz": "bar",
+				},
+			}},
+		},
+		{
+			"empty rule",
+			[]config.Rule{{Alert: "foo"}, {Record: "bar"}},
+			nil,
+		},
+		{
+			"multiple rules",
+			[]config.Rule{
+				{Alert: "bar"},
+				{Alert: "baz"},
+				{Alert: "foo"},
+			},
+			[]config.Rule{
+				{Alert: "baz"},
+				{Record: "foo"},
+			},
+		},
+		{
+			"replace rule",
+			[]config.Rule{{Alert: "foo1"}},
+			[]config.Rule{{Alert: "foo2"}},
+		},
+		{
+			"replace multiple rules",
+			[]config.Rule{
+				{Alert: "foo1"},
+				{Record: "foo2"},
+				{Alert: "foo3"},
+			},
+			[]config.Rule{
+				{Alert: "foo3"},
+				{Alert: "foo4"},
+				{Record: "foo5"},
+			},
+		},
 	}
 
-	// new rule
-	f(nil, []config.Rule{
-		{Alert: "bar"},
-	})
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := &Group{Name: "test"}
+			qb := &datasource.FakeQuerier{}
+			for _, r := range tc.currentRules {
+				r.ID = config.HashRule(r)
+				g.Rules = append(g.Rules, g.newRule(qb, r))
+			}
 
-	// update alerting rule
-	f([]config.Rule{
-		{
-			Alert: "foo",
-			Expr:  "up > 0",
-			For:   promutils.NewDuration(time.Second),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-			Annotations: map[string]string{
-				"summary":     "{{ $value|humanize }}",
-				"description": "{{$labels}}",
-			},
-		},
-		{
-			Alert: "bar",
-			Expr:  "up > 0",
-			For:   promutils.NewDuration(time.Second),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-		},
-	}, []config.Rule{
-		{
-			Alert: "foo",
-			Expr:  "up > 10",
-			For:   promutils.NewDuration(time.Second),
-			Labels: map[string]string{
-				"baz": "bar",
-			},
-			Annotations: map[string]string{
-				"summary": "none",
-			},
-		},
-		{
-			Alert:         "bar",
-			Expr:          "up > 0",
-			For:           promutils.NewDuration(2 * time.Second),
-			KeepFiringFor: promutils.NewDuration(time.Minute),
-			Labels: map[string]string{
-				"bar": "baz",
-			},
-		},
-	})
+			ng := &Group{Name: "test"}
+			for _, r := range tc.newRules {
+				r.ID = config.HashRule(r)
+				ng.Rules = append(ng.Rules, ng.newRule(qb, r))
+			}
 
-	// update recording rule
-	f([]config.Rule{{
-		Record: "foo",
-		Expr:   "max(up)",
-		Labels: map[string]string{
-			"bar": "baz",
-		},
-	}}, []config.Rule{{
-		Record: "foo",
-		Expr:   "min(up)",
-		Labels: map[string]string{
-			"baz": "bar",
-		},
-	}})
+			err := g.updateWith(ng)
+			if err != nil {
+				t.Fatal(err)
+			}
 
-	// empty rule
-	f([]config.Rule{{Alert: "foo"}, {Record: "bar"}}, nil)
-
-	// multiple rules
-	f([]config.Rule{
-		{Alert: "bar"},
-		{Alert: "baz"},
-		{Alert: "foo"},
-	}, []config.Rule{
-		{Alert: "baz"},
-		{Record: "foo"},
-	})
-
-	// replace rule
-	f([]config.Rule{{Alert: "foo1"}}, []config.Rule{{Alert: "foo2"}})
-
-	// replace multiple rules
-	f([]config.Rule{
-		{Alert: "foo1"},
-		{Record: "foo2"},
-		{Alert: "foo3"},
-	}, []config.Rule{
-		{Alert: "foo3"},
-		{Alert: "foo4"},
-		{Record: "foo5"},
-	})
+			if len(g.Rules) != len(tc.newRules) {
+				t.Fatalf("expected to have %d rules; got: %d",
+					len(g.Rules), len(tc.newRules))
+			}
+			sort.Slice(g.Rules, func(i, j int) bool {
+				return g.Rules[i].ID() < g.Rules[j].ID()
+			})
+			sort.Slice(ng.Rules, func(i, j int) bool {
+				return ng.Rules[i].ID() < ng.Rules[j].ID()
+			})
+			for i, r := range g.Rules {
+				got, want := r, ng.Rules[i]
+				if got.ID() != want.ID() {
+					t.Fatalf("expected to have rule %q; got %q", want, got)
+				}
+				if err := CompareRules(t, got, want); err != nil {
+					t.Fatalf("comparison error: %s", err)
+				}
+			}
+		})
+	}
 }
 
 func TestGroupStart(t *testing.T) {
@@ -296,23 +312,30 @@ func TestGroupStart(t *testing.T) {
 	<-finished
 }
 
-func TestGetResolveDuration(t *testing.T) {
-	f := func(groupInterval, maxDuration, resendDelay, resultExpected time.Duration) {
-		t.Helper()
-
-		result := getResolveDuration(groupInterval, resendDelay, maxDuration)
-		if result != resultExpected {
-			t.Fatalf("unexpected result; got %s; want %s", result, resultExpected)
-		}
+func TestResolveDuration(t *testing.T) {
+	testCases := []struct {
+		groupInterval time.Duration
+		maxDuration   time.Duration
+		resendDelay   time.Duration
+		expected      time.Duration
+	}{
+		{time.Minute, 0, 0, 4 * time.Minute},
+		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
+		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
+		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
+		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
+		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
+		{0, 0, 0, 0},
 	}
 
-	f(0, 0, 0, 0)
-	f(time.Minute, 0, 0, 4*time.Minute)
-	f(time.Minute, 0, 2*time.Minute, 8*time.Minute)
-	f(time.Minute, 4*time.Minute, 4*time.Minute, 4*time.Minute)
-	f(2*time.Minute, time.Minute, 2*time.Minute, time.Minute)
-	f(time.Minute, 2*time.Minute, 1*time.Minute, 2*time.Minute)
-	f(2*time.Minute, 0, 1*time.Minute, 8*time.Minute)
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
+			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
+			if got != tc.expected {
+				t.Errorf("expected to have %v; got %v", tc.expected, got)
+			}
+		})
+	}
 }
 
 func TestGetStaleSeries(t *testing.T) {
@@ -322,7 +345,6 @@ func TestGetStaleSeries(t *testing.T) {
 	}
 	f := func(r Rule, labels, expLabels [][]prompbmarshal.Label) {
 		t.Helper()
-
 		var tss []prompbmarshal.TimeSeries
 		for _, l := range labels {
 			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
@@ -584,7 +606,7 @@ func TestGroupStartDelay(t *testing.T) {
 		delay := delayBeforeStart(at, key, g.Interval, g.EvalOffset)
 		gotStart := at.Add(delay)
 		if expTS != gotStart {
-			t.Fatalf("expected to get %v; got %v instead", expTS, gotStart)
+			t.Errorf("expected to get %v; got %v instead", expTS, gotStart)
 		}
 	}
 
@@ -625,118 +647,157 @@ func TestGroupStartDelay(t *testing.T) {
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {
-	f := func(g *Group, tsOrigin, tsExpected string) {
-		t.Helper()
-
-		originT, _ := time.Parse(time.RFC3339, tsOrigin)
-		expT, _ := time.Parse(time.RFC3339, tsExpected)
-		gotTS := g.adjustReqTimestamp(originT)
-		if !gotTS.Equal(expT) {
-			t.Fatalf("get wrong prometheus request timestamp: %s; want %s", gotTS, expT)
-		}
-	}
-
 	offset := 30 * time.Minute
 	evalDelay := 1 * time.Minute
 	disableAlign := false
-
-	// with query align + default evalDelay
-	f(&Group{
-		Interval: time.Hour,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:00:00+00:00")
-
-	// without query align + default evalDelay
-	f(&Group{
-		Interval:      time.Hour,
-		evalAlignment: &disableAlign,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:10:30+00:00")
-
-	// with eval_offset, find previous offset point + default evalDelay
-	f(&Group{
-		EvalOffset: &offset,
-		Interval:   time.Hour,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T10:30:00+00:00")
-
-	// with eval_offset + default evalDelay
-	f(&Group{
-		EvalOffset: &offset,
-		Interval:   time.Hour,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:30:00+00:00")
-
-	// 1h interval with eval_delay
-	f(&Group{
-		EvalDelay: &evalDelay,
-		Interval:  time.Hour,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:00:00+00:00")
-
-	// 1m interval with eval_delay
-	f(&Group{
-		EvalDelay: &evalDelay,
-		Interval:  time.Minute,
-	}, "2023-08-28T11:41:13+00:00", "2023-08-28T11:40:00+00:00")
-
-	// disable alignment with eval_delay
-	f(&Group{
-		EvalDelay:     &evalDelay,
-		Interval:      time.Hour,
-		evalAlignment: &disableAlign,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:40:00+00:00")
+	testCases := []struct {
+		name            string
+		g               *Group
+		originTS, expTS string
+	}{
+		{
+			"with query align + default evalDelay",
+			&Group{
+				Interval: time.Hour,
+			},
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T11:00:00+00:00",
+		},
+		{
+			"without query align + default evalDelay",
+			&Group{
+				Interval:      time.Hour,
+				evalAlignment: &disableAlign,
+			},
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T11:10:30+00:00",
+		},
+		{
+			"with eval_offset, find previous offset point + default evalDelay",
+			&Group{
+				EvalOffset: &offset,
+				Interval:   time.Hour,
+			},
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T10:30:00+00:00",
+		},
+		{
+			"with eval_offset + default evalDelay",
+			&Group{
+				EvalOffset: &offset,
+				Interval:   time.Hour,
+			},
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:30:00+00:00",
+		},
+		{
+			"1h interval with eval_delay",
+			&Group{
+				EvalDelay: &evalDelay,
+				Interval:  time.Hour,
+			},
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:00:00+00:00",
+		},
+		{
+			"1m interval with eval_delay",
+			&Group{
+				EvalDelay: &evalDelay,
+				Interval:  time.Minute,
+			},
+			"2023-08-28T11:41:13+00:00",
+			"2023-08-28T11:40:00+00:00",
+		},
+		{
+			"disable alignment with eval_delay",
+			&Group{
+				EvalDelay:     &evalDelay,
+				Interval:      time.Hour,
+				evalAlignment: &disableAlign,
+			},
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:40:00+00:00",
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			originT, _ := time.Parse(time.RFC3339, tc.originTS)
+			expT, _ := time.Parse(time.RFC3339, tc.expTS)
+			gotTS := tc.g.adjustReqTimestamp(originT)
+			if !gotTS.Equal(expT) {
+				t.Fatalf("get wrong prometheus request timestamp, expect %s, got %s", expT, gotTS)
+			}
+		})
+	}
 }
 
 func TestRangeIterator(t *testing.T) {
-	f := func(ri rangeIterator, resultExpected [][2]time.Time) {
-		t.Helper()
-
-		var j int
-		for ri.next() {
-			if len(resultExpected) < j+1 {
-				t.Fatalf("unexpected result for iterator on step %d: %v - %v", j, ri.s, ri.e)
-			}
-			s, e := ri.s, ri.e
-			expS, expE := resultExpected[j][0], resultExpected[j][1]
-			if s != expS {
-				t.Fatalf("expected to get start=%v; got %v", expS, s)
-			}
-			if e != expE {
-				t.Fatalf("expected to get end=%v; got %v", expE, e)
-			}
-			j++
-		}
+	testCases := []struct {
+		ri     rangeIterator
+		result [][2]time.Time
+	}{
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+				step:  5 * time.Minute,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
+				{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
+				{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
+				{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
+				{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
+				{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+			},
+		},
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+				step:  45 * time.Minute,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+				{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+			},
+		},
+		{
+			ri: rangeIterator{
+				start: parseTime(t, "2021-01-01T12:00:12.000Z"),
+				end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
+				step:  time.Second,
+			},
+			result: [][2]time.Time{
+				{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
+				{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
+				{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
+				{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
+				{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
+			},
+		},
 	}
 
-	f(rangeIterator{
-		start: parseTime(t, "2021-01-01T12:00:00.000Z"),
-		end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
-		step:  5 * time.Minute,
-	}, [][2]time.Time{
-		{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
-		{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
-		{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
-		{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
-		{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
-		{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-	})
-
-	f(rangeIterator{
-		start: parseTime(t, "2021-01-01T12:00:00.000Z"),
-		end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
-		step:  45 * time.Minute,
-	}, [][2]time.Time{
-		{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-		{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-	})
-
-	f(rangeIterator{
-		start: parseTime(t, "2021-01-01T12:00:12.000Z"),
-		end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
-		step:  time.Second,
-	}, [][2]time.Time{
-		{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
-		{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
-		{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
-		{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
-		{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
-	})
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			var j int
+			for tc.ri.next() {
+				if len(tc.result) < j+1 {
+					t.Fatalf("unexpected result for iterator on step %d: %v - %v",
+						j, tc.ri.s, tc.ri.e)
+				}
+				s, e := tc.ri.s, tc.ri.e
+				expS, expE := tc.result[j][0], tc.result[j][1]
+				if s != expS {
+					t.Fatalf("expected to get start=%v; got %v", expS, s)
+				}
+				if e != expE {
+					t.Fatalf("expected to get end=%v; got %v", expE, e)
+				}
+				j++
+			}
+		})
+	}
 }
 
 func parseTime(t *testing.T, s string) time.Time {

app/vmalert/rule/recording_test.go

@@ -13,225 +13,218 @@ import (
 )
 
 func TestRecordingRule_Exec(t *testing.T) {
-	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
-		t.Helper()
-
-		fq := &datasource.FakeQuerier{}
-		fq.Add(metrics...)
-		rule.q = fq
-		rule.state = &ruleState{
-			entries: make([]StateEntry, 10),
-		}
-		tss, err := rule.exec(context.TODO(), time.Now(), 0)
-		if err != nil {
-			t.Fatalf("unexpected RecordingRule.exec error: %s", err)
-		}
-		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
-		}
-	}
-
 	timestamp := time.Now()
-
-	f(&RecordingRule{
-		Name: "foo",
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 10, "__name__", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foo",
-		}),
-	})
-
-	f(&RecordingRule{
-		Name: "foobarbaz",
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
-		metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "foo",
-		}),
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "bar",
-		}),
-		newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "baz",
-		}),
-	})
-
-	f(&RecordingRule{
-		Name: "job:foo",
-		Labels: map[string]string{
-			"source": "test",
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValueAndLabels(t, 10,
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foo",
+				}),
+			},
 		},
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "foo",
-			"source":   "test",
-		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__":        "job:foo",
-			"job":             "bar",
-			"source":          "test",
-			"exported_source": "origin",
-		}),
-	})
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+				metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}),
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}),
+				newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "baz",
+				}),
+			},
+		},
+		{
+			&RecordingRule{
+				Name: "job:foo",
+				Labels: map[string]string{
+					"source": "test",
+				},
+			},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}),
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__":        "job:foo",
+					"job":             "bar",
+					"source":          "test",
+					"exported_source": "origin",
+				}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &datasource.FakeQuerier{}
+			fq.Add(tc.metrics...)
+			tc.rule.q = fq
+			tc.rule.state = &ruleState{entries: make([]StateEntry, 10)}
+			tss, err := tc.rule.exec(context.TODO(), time.Now(), 0)
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
 }
 
 func TestRecordingRule_ExecRange(t *testing.T) {
-	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
-		t.Helper()
-
-		fq := &datasource.FakeQuerier{}
-		fq.Add(metrics...)
-		rule.q = fq
-		tss, err := rule.execRange(context.TODO(), time.Now(), time.Now())
-		if err != nil {
-			t.Fatalf("unexpected RecordingRule.execRange error: %s", err)
-		}
-		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries missmatch: %s", err)
-		}
-	}
-
 	timestamp := time.Now()
+	testCases := []struct {
+		rule    *RecordingRule
+		metrics []datasource.Metric
+		expTS   []prompbmarshal.TimeSeries
+	}{
+		{
+			&RecordingRule{Name: "foo"},
+			[]datasource.Metric{metricWithValuesAndLabels(t, []float64{10, 20, 30},
+				"__name__", "bar",
+			)},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{10, 20, 30},
+					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+					map[string]string{
+						"__name__": "foo",
+					}),
+			},
+		},
+		{
+			&RecordingRule{Name: "foobarbaz"},
+			[]datasource.Metric{
+				metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+				metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+				metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "foo",
+				}),
+				newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+					"__name__": "foobarbaz",
+					"job":      "bar",
+				}),
+				newTimeSeries([]float64{4, 5, 6},
+					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
+					map[string]string{
+						"__name__": "foobarbaz",
+						"job":      "baz",
+					}),
+			},
+		},
+		{
+			&RecordingRule{Name: "job:foo", Labels: map[string]string{
+				"source": "test",
+			}},
+			[]datasource.Metric{
+				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
+			},
+			[]prompbmarshal.TimeSeries{
+				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "foo",
+					"source":   "test",
+				}),
+				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+					"__name__": "job:foo",
+					"job":      "bar",
+					"source":   "test",
+				}),
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.rule.Name, func(t *testing.T) {
+			fq := &datasource.FakeQuerier{}
+			fq.Add(tc.metrics...)
+			tc.rule.q = fq
+			tss, err := tc.rule.execRange(context.TODO(), time.Now(), time.Now())
+			if err != nil {
+				t.Fatalf("unexpected Exec err: %s", err)
+			}
+			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
+				t.Fatalf("timeseries missmatch: %s", err)
+			}
+		})
+	}
+}
 
-	f(&RecordingRule{
-		Name: "foo",
-	}, []datasource.Metric{
-		metricWithValuesAndLabels(t, []float64{10, 20, 30}, "__name__", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-			"__name__": "foo",
-		}),
-	})
-
-	f(&RecordingRule{
-		Name: "foobarbaz",
-	}, []datasource.Metric{
+func TestRecordingRuleLimit(t *testing.T) {
+	timestamp := time.Now()
+	testCases := []struct {
+		limit int
+		err   string
+	}{
+		{
+			limit: 0,
+		},
+		{
+			limit: -1,
+		},
+		{
+			limit: 1,
+			err:   "exec exceeded limit of 1 with 3 series",
+		},
+		{
+			limit: 2,
+			err:   "exec exceeded limit of 2 with 3 series",
+		},
+	}
+	testMetrics := []datasource.Metric{
 		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "foo",
-		}),
-		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-			"__name__": "foobarbaz",
-			"job":      "bar",
-		}),
-		newTimeSeries([]float64{4, 5, 6},
-			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-				"__name__": "foobarbaz",
-				"job":      "baz",
-			}),
-	})
-
-	f(&RecordingRule{
-		Name: "job:foo",
+	}
+	rule := &RecordingRule{Name: "job:foo",
+		state: &ruleState{entries: make([]StateEntry, 10)},
 		Labels: map[string]string{
-			"source": "test",
+			"source": "test_limit",
 		},
-	}, []datasource.Metric{
-		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
-	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "foo",
-			"source":   "test",
-		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-			"__name__": "job:foo",
-			"job":      "bar",
-			"source":   "test",
-		}),
-	})
-}
-
-func TestRecordingRuleLimit_Failure(t *testing.T) {
-	f := func(limit int, errStrExpected string) {
-		t.Helper()
-
-		testMetrics := []datasource.Metric{
-			metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
-			metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
-			metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-		}
-
+		metrics: &recordingRuleMetrics{
+			errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
+		},
+	}
+	var err error
+	for _, testCase := range testCases {
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-
-		rule := &RecordingRule{Name: "job:foo",
-			state: &ruleState{entries: make([]StateEntry, 10)},
-			Labels: map[string]string{
-				"source": "test_limit",
-			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
-		}
 		rule.q = fq
-
-		_, err := rule.exec(context.TODO(), time.Now(), limit)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		errStr := err.Error()
-		if !strings.Contains(errStr, errStrExpected) {
-			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
+		_, err = rule.exec(context.TODO(), timestamp, testCase.limit)
+		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
+			t.Fatal(err)
 		}
 	}
-
-	f(1, "exec exceeded limit of 1 with 3 series")
-	f(2, "exec exceeded limit of 2 with 3 series")
 }
 
-func TestRecordingRuleLimit_Success(t *testing.T) {
-	f := func(limit int) {
-		t.Helper()
-
-		testMetrics := []datasource.Metric{
-			metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
-			metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
-			metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-		}
-
-		fq := &datasource.FakeQuerier{}
-		fq.Add(testMetrics...)
-
-		rule := &RecordingRule{Name: "job:foo",
-			state: &ruleState{entries: make([]StateEntry, 10)},
-			Labels: map[string]string{
-				"source": "test_limit",
-			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
-		}
-		rule.q = fq
-
-		_, err := rule.exec(context.TODO(), time.Now(), limit)
-		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
-	}
-
-	f(0)
-	f(-1)
-}
-
-func TestRecordingRuleExec_Negative(t *testing.T) {
+func TestRecordingRule_ExecNegative(t *testing.T) {
 	rr := &RecordingRule{
 		Name: "job:foo",
 		Labels: map[string]string{
@@ -263,6 +256,6 @@ func TestRecordingRuleExec_Negative(t *testing.T) {
 
 	_, err = rr.exec(context.TODO(), time.Now(), 0)
 	if err != nil {
-		t.Fatalf("cannot execute recroding rule: %s", err)
+		t.Fatal(err)
 	}
 }

app/vmalert/templates/template.go

@@ -316,7 +316,7 @@ func templateFuncs() textTpl.FuncMap {
 
 		// humanize converts given number to a human readable format
 		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(i any) (string, error) {
+		"humanize": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -347,7 +347,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(i any) (string, error) {
+		"humanize1024": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -359,7 +359,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizeDuration converts given seconds to a human-readable duration
-		"humanizeDuration": func(i any) (string, error) {
+		"humanizeDuration": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -405,7 +405,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(i any) (string, error) {
+		"humanizePercentage": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -414,7 +414,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(i any) (string, error) {
+		"humanizeTimestamp": func(i interface{}) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -427,7 +427,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// toTime converts given timestamp to a time.Time.
-		"toTime": func(i any) (time.Time, error) {
+		"toTime": func(i interface{}) (time.Time, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return time.Time{}, err
@@ -524,8 +524,8 @@ func templateFuncs() textTpl.FuncMap {
 
 		// Converts a list of objects to a map with keys arg0, arg1 etc.
 		// This is intended to allow multiple arguments to be passed to templates.
-		"args": func(args ...any) map[string]any {
-			result := make(map[string]any)
+		"args": func(args ...interface{}) map[string]interface{} {
+			result := make(map[string]interface{})
 			for i, a := range args {
 				result[fmt.Sprintf("arg%d", i)] = a
 			}
@@ -565,7 +565,7 @@ func (t Time) Time() time.Time {
 	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
 
-func toFloat64(v any) (float64, error) {
+func toFloat64(v interface{}) (float64, error) {
 	switch i := v.(type) {
 	case float64:
 		return i, nil

app/vmalert/templates/template_test.go

@@ -7,11 +7,10 @@ import (
 	textTpl "text/template"
 )
 
-func TestTemplateFuncs_StringConversion(t *testing.T) {
+func TestTemplateFuncs(t *testing.T) {
+	funcs := templateFuncs()
 	f := func(funcName, s, resultExpected string) {
 		t.Helper()
-
-		funcs := templateFuncs()
 		v := funcs[funcName]
 		fLocal := v.(func(s string) string)
 		result := fLocal(s)
@@ -19,7 +18,6 @@ func TestTemplateFuncs_StringConversion(t *testing.T) {
 			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
 		}
 	}
-
 	f("title", "foo bar", "Foo Bar")
 	f("toUpper", "foo", "FOO")
 	f("toLower", "FOO", "foo")
@@ -33,10 +31,7 @@ func TestTemplateFuncs_StringConversion(t *testing.T) {
 	f("stripPort", "foo:1234", "foo")
 	f("stripDomain", "foo.bar.baz", "foo")
 	f("stripDomain", "foo.bar:123", "foo:123")
-}
 
-func TestTemplateFuncs_Match(t *testing.T) {
-	funcs := templateFuncs()
 	// check "match" func
 	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
 	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
@@ -56,15 +51,11 @@ func TestTemplateFuncs_Match(t *testing.T) {
 	if !ok {
 		t.Fatalf("unexpected mismatch")
 	}
-}
 
-func TestTemplateFuncs_Formatting(t *testing.T) {
-	f := func(funcName string, p any, resultExpected string) {
+	formatting := func(funcName string, p interface{}, resultExpected string) {
 		t.Helper()
-
-		funcs := templateFuncs()
 		v := funcs[funcName]
-		fLocal := v.(func(s any) (string, error))
+		fLocal := v.(func(s interface{}) (string, error))
 		result, err := fLocal(p)
 		if err != nil {
 			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
@@ -73,36 +64,35 @@ func TestTemplateFuncs_Formatting(t *testing.T) {
 			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
 		}
 	}
+	formatting("humanize1024", float64(0), "0")
+	formatting("humanize1024", math.Inf(0), "+Inf")
+	formatting("humanize1024", math.NaN(), "NaN")
+	formatting("humanize1024", float64(127087), "124.1ki")
+	formatting("humanize1024", float64(130137088), "124.1Mi")
+	formatting("humanize1024", float64(133260378112), "124.1Gi")
+	formatting("humanize1024", float64(136458627186688), "124.1Ti")
+	formatting("humanize1024", float64(139733634239168512), "124.1Pi")
+	formatting("humanize1024", float64(143087241460908556288), "124.1Ei")
+	formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 
-	f("humanize1024", float64(0), "0")
-	f("humanize1024", math.Inf(0), "+Inf")
-	f("humanize1024", math.NaN(), "NaN")
-	f("humanize1024", float64(127087), "124.1ki")
-	f("humanize1024", float64(130137088), "124.1Mi")
-	f("humanize1024", float64(133260378112), "124.1Gi")
-	f("humanize1024", float64(136458627186688), "124.1Ti")
-	f("humanize1024", float64(139733634239168512), "124.1Pi")
-	f("humanize1024", float64(143087241460908556288), "124.1Ei")
-	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
-	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
-	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
+	formatting("humanize", float64(127087), "127.1k")
+	formatting("humanize", float64(136458627186688), "136.5T")
 
-	f("humanize", float64(127087), "127.1k")
-	f("humanize", float64(136458627186688), "136.5T")
+	formatting("humanizeDuration", 1, "1s")
+	formatting("humanizeDuration", 0.2, "200ms")
+	formatting("humanizeDuration", 42000, "11h 40m 0s")
+	formatting("humanizeDuration", 16790555, "194d 8h 2m 35s")
 
-	f("humanizeDuration", 1, "1s")
-	f("humanizeDuration", 0.2, "200ms")
-	f("humanizeDuration", 42000, "11h 40m 0s")
-	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
+	formatting("humanizePercentage", 1, "100%")
+	formatting("humanizePercentage", 0.8, "80%")
+	formatting("humanizePercentage", 0.015, "1.5%")
 
-	f("humanizePercentage", 1, "100%")
-	f("humanizePercentage", 0.8, "80%")
-	f("humanizePercentage", 0.015, "1.5%")
-
-	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
+	formatting("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
 }
 
-func mkTemplate(current, replacement any) textTemplate {
+func mkTemplate(current, replacement interface{}) textTemplate {
 	tmpl := textTemplate{}
 	if current != nil {
 		switch val := current.(type) {
@@ -148,201 +138,224 @@ func equalTemplates(tmpls ...*textTpl.Template) bool {
 	return true
 }
 
-func TestTemplatesLoad_Failure(t *testing.T) {
-	f := func(pathPatterns []string, expectedErrStr string) {
-		t.Helper()
-
-		err := Load(pathPatterns, false)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-
-		errStr := err.Error()
-		if !strings.Contains(errStr, expectedErrStr) {
-			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
-		}
+func TestTemplates_Load(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		pathPatterns     []string
+		overwrite        bool
+		expectedTemplate textTemplate
+		expErr           string
+	}{
+		{
+			"non existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"non existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/non-existing/good-*.tpl",
+				"templates/absent/good-*.tpl",
+			},
+			true,
+			mkTemplate(``, nil),
+			"",
+		},
+		{
+			"existing path undefined template override",
+			mkTemplate(nil, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`, nil),
+			"",
+		},
+		{
+			"existing path defined template override",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/good0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				`, `
+				{{- define "good0-test.tpl" -}}{{- end -}}
+				{{- define "test.0" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.2" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+				{{- define "test.3" -}}
+					{{ printf "Hello %s!" externalURL }}
+				{{- end -}}
+			`),
+			"",
+		},
+		{
+			"load template with syntax error",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			[]string{
+				"templates/other/nested/bad0-*.tpl",
+				"templates/test/good0-*.tpl",
+			},
+			false,
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{ printf "Hello %s!" "world" }}
+				{{- end -}}
+			`, nil),
+			"failed to parse template glob",
+		},
 	}
 
-	// load template with syntax error
-	f([]string{
-		"templates/other/nested/bad0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}, "failed to parse template glob")
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			err := Load(tc.pathPatterns, tc.overwrite)
+			if tc.expErr == "" && err != nil {
+				t.Error("happened error that wasn't expected: %w", err)
+			}
+			if tc.expErr != "" && err == nil {
+				t.Error("%+w", err)
+				t.Error("expected error that didn't happened")
+			}
+			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
+				t.Error("%+w", err)
+				t.Error("expected string doesn't exist in error message")
+			}
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
+	}
 }
 
-func TestTemplatesLoad_Success(t *testing.T) {
-	f := func(initialTmpl textTemplate, pathPatterns []string, overwrite bool, expectedTmpl textTemplate) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		masterTmpl = initialTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-
-		if err := Load(pathPatterns, overwrite); err != nil {
-			t.Fatalf("cannot load templates: %s", err)
-		}
-
-		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
-			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
-		}
-		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
-			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
-		}
+func TestTemplates_Reload(t *testing.T) {
+	testCases := []struct {
+		name             string
+		initialTemplate  textTemplate
+		expectedTemplate textTemplate
+	}{
+		{
+			"empty current and replacement templates",
+			mkTemplate(nil, nil),
+			mkTemplate(nil, nil),
+		},
+		{
+			"empty current template only",
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"empty replacement template only",
+			mkTemplate(nil, `
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+			`, nil),
+		},
+		{
+			"defined both templates",
+			mkTemplate(`
+				{{- define "test.0" -}}
+					{{- printf "value" -}}
+				{{- end -}}
+				{{- define "test.1" -}}
+					{{- printf "before" -}}
+				{{- end -}}
+			`, `
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`),
+			mkTemplate(`
+				{{- define "test.1" -}}
+					{{- printf "after" -}}
+				{{- end -}}
+			`, nil),
+		},
 	}
 
-	// non existing path undefined template override
-	initialTmpl := mkTemplate(nil, nil)
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			masterTmpl = tc.initialTemplate
+			Reload()
+			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
+				t.Fatalf("replacement template is not as expected")
+			}
+			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
+				t.Fatalf("current template is not as expected")
+			}
+		})
 	}
-	overwrite := true
-	expectedTmpl := mkTemplate(``, nil)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-
-	// non existing path defined template override
-	initialTmpl = mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil)
-	pathPatterns = []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	overwrite = true
-	expectedTmpl = mkTemplate(``, nil)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-
-	// existing path undefined template override
-	initialTmpl = mkTemplate(nil, nil)
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}
-	overwrite = false
-	expectedTmpl = mkTemplate(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`, nil)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-
-	// existing path defined template override
-	initialTmpl = mkTemplate(`
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" "world" }}
-		{{- end -}}
-	`, nil)
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}
-	overwrite = false
-	expectedTmpl = mkTemplate(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" "world" }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`, `
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`)
-	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
-}
-
-func TestTemplatesReload(t *testing.T) {
-	f := func(initialTmpl, expectedTmpl textTemplate) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		masterTmpl = initialTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-
-		Reload()
-
-		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
-			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
-		}
-		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
-			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
-		}
-	}
-
-	// empty current and replacement templates
-	f(mkTemplate(nil, nil), mkTemplate(nil, nil))
-
-	// empty current template only
-	f(mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil), mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil))
-
-	// empty replacement template only
-	f(mkTemplate(nil, `
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`), mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-	`, nil))
-
-	// defined both templates
-	f(mkTemplate(`
-		{{- define "test.0" -}}
-			{{- printf "value" -}}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{- printf "before" -}}
-		{{- end -}}
-	`, `
-		{{- define "test.1" -}}
-			{{- printf "after" -}}
-		{{- end -}}
-	`), mkTemplate(`
-		{{- define "test.1" -}}
-			{{- printf "after" -}}
-		{{- end -}}
-	`, nil))
 }

app/vmalert/utils/err_group_test.go

@@ -7,31 +7,35 @@ import (
 )
 
 func TestErrGroup(t *testing.T) {
-	f := func(errs []error, resultExpected string) {
-		t.Helper()
-
-		eg := &ErrGroup{}
-		for _, err := range errs {
+	testCases := []struct {
+		errs []error
+		exp  string
+	}{
+		{nil, ""},
+		{[]error{errors.New("timeout")}, "errors(1): timeout"},
+		{
+			[]error{errors.New("timeout"), errors.New("deadline")},
+			"errors(2): timeout\ndeadline",
+		},
+	}
+	for _, tc := range testCases {
+		eg := new(ErrGroup)
+		for _, err := range tc.errs {
 			eg.Add(err)
 		}
-		if len(errs) == 0 {
+		if len(tc.errs) == 0 {
 			if eg.Err() != nil {
 				t.Fatalf("expected to get nil error")
 			}
-			return
+			continue
 		}
 		if eg.Err() == nil {
 			t.Fatalf("expected to get non-nil error")
 		}
-		result := eg.Error()
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%v\nwant\n%v", result, resultExpected)
+		if eg.Error() != tc.exp {
+			t.Fatalf("expected to have: \n%q\ngot:\n%q", tc.exp, eg.Error())
 		}
 	}
-
-	f(nil, "")
-	f([]error{errors.New("timeout")}, "errors(1): timeout")
-	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
 }
 
 // TestErrGroupConcurrent supposed to test concurrent

app/vmalert/web.go

@@ -19,7 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
-var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
+var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
 
 var (
 	apiLinks = [][2]string{
@@ -35,7 +35,7 @@ var (
 		{"-/reload", "reload configuration"},
 	}
 	navItems = []tpl.NavItem{
-		{Name: "vmalert", Url: "../vmalert"},
+		{Name: "vmalert", Url: "."},
 		{Name: "Groups", Url: "groups"},
 		{Name: "Alerts", Url: "alerts"},
 		{Name: "Notifiers", Url: "notifiers"},
@@ -167,7 +167,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		logger.Infof("api config reload was called, sending sighup")

app/vmalert/web_test.go

@@ -36,23 +36,23 @@ func TestHandler(t *testing.T) {
 	}}
 	rh := &requestHandler{m: m}
 
-	getResp := func(t *testing.T, url string, to any, code int) {
+	getResp := func(t *testing.T, url string, to interface{}, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
 			t.Fatalf("unexpected err %s", err)
 		}
 		if code != resp.StatusCode {
-			t.Fatalf("unexpected status code %d want %d", resp.StatusCode, code)
+			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
 		}
 		defer func() {
 			if err := resp.Body.Close(); err != nil {
-				t.Fatalf("err closing body %s", err)
+				t.Errorf("err closing body %s", err)
 			}
 		}()
 		if to != nil {
 			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
-				t.Fatalf("unexpected err %s", err)
+				t.Errorf("unexpected err %s", err)
 			}
 		}
 	}
@@ -92,13 +92,13 @@ func TestHandler(t *testing.T) {
 		lr := listAlertsResponse{}
 		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
-			t.Fatalf("expected 1 alert got %d", length)
+			t.Errorf("expected 1 alert got %d", length)
 		}
 
 		lr = listAlertsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
-			t.Fatalf("expected 1 alert got %d", length)
+			t.Errorf("expected 1 alert got %d", length)
 		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
@@ -106,13 +106,13 @@ func TestHandler(t *testing.T) {
 		alert := &apiAlert{}
 		getResp(t, ts.URL+"/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
-			t.Fatalf("expected %v is equal to %v", alert, expAlert)
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}
 
 		alert = &apiAlert{}
 		getResp(t, ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
-			t.Fatalf("expected %v is equal to %v", alert, expAlert)
+			t.Errorf("expected %v is equal to %v", alert, expAlert)
 		}
 	})
 
@@ -135,13 +135,13 @@ func TestHandler(t *testing.T) {
 		lr := listGroupsResponse{}
 		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
-			t.Fatalf("expected 1 group got %d", length)
+			t.Errorf("expected 1 group got %d", length)
 		}
 
 		lr = listGroupsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
-			t.Fatalf("expected 1 group got %d", length)
+			t.Errorf("expected 1 group got %d", length)
 		}
 	})
 	t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
@@ -150,14 +150,14 @@ func TestHandler(t *testing.T) {
 		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
-			t.Fatalf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
+			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRule = apiRule{}
 		getResp(t, ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
-			t.Fatalf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
+			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRuleWithUpdates := apiRuleWithUpdates{}
@@ -173,7 +173,7 @@ func TestHandler(t *testing.T) {
 			lr := listGroupsResponse{}
 			getResp(t, ts.URL+url, &lr, 200)
 			if length := len(lr.Data.Groups); length != expGroups {
-				t.Fatalf("expected %d groups got %d", expGroups, length)
+				t.Errorf("expected %d groups got %d", expGroups, length)
 			}
 			if len(lr.Data.Groups) < 1 {
 				return
@@ -183,7 +183,7 @@ func TestHandler(t *testing.T) {
 				rulesN += len(gr.Rules)
 			}
 			if rulesN != expRules {
-				t.Fatalf("expected %d rules got %d", expRules, rulesN)
+				t.Errorf("expected %d rules got %d", expRules, rulesN)
 			}
 		}
 
@@ -241,23 +241,23 @@ func TestEmptyResponse(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rhWithNoGroups.handler(w, r) }))
 	defer ts.Close()
 
-	getResp := func(t *testing.T, url string, to any, code int) {
+	getResp := func(t *testing.T, url string, to interface{}, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
 			t.Fatalf("unexpected err %s", err)
 		}
 		if code != resp.StatusCode {
-			t.Fatalf("unexpected status code %d want %d", resp.StatusCode, code)
+			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
 		}
 		defer func() {
 			if err := resp.Body.Close(); err != nil {
-				t.Fatalf("err closing body %s", err)
+				t.Errorf("err closing body %s", err)
 			}
 		}()
 		if to != nil {
 			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
-				t.Fatalf("unexpected err %s", err)
+				t.Errorf("unexpected err %s", err)
 			}
 		}
 	}
@@ -266,13 +266,13 @@ func TestEmptyResponse(t *testing.T) {
 		lr := listAlertsResponse{}
 		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
-			t.Fatalf("expected /api/v1/alerts response to have non-nil data")
+			t.Errorf("expected /api/v1/alerts response to have non-nil data")
 		}
 
 		lr = listAlertsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
-			t.Fatalf("expected /api/v1/alerts response to have non-nil data")
+			t.Errorf("expected /api/v1/alerts response to have non-nil data")
 		}
 	})
 
@@ -280,13 +280,13 @@ func TestEmptyResponse(t *testing.T) {
 		lr := listGroupsResponse{}
 		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
-			t.Fatalf("expected /api/v1/rules response to have non-nil data")
+			t.Errorf("expected /api/v1/rules response to have non-nil data")
 		}
 
 		lr = listGroupsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
-			t.Fatalf("expected /api/v1/rules response to have non-nil data")
+			t.Errorf("expected /api/v1/rules response to have non-nil data")
 		}
 	})
 

app/vmalert/web_types.go

@@ -183,7 +183,7 @@ func (ar apiRule) WebLink() string {
 		paramGroupID, ar.GroupID, paramRuleID, ar.ID)
 }
 
-func ruleToAPI(r any) apiRule {
+func ruleToAPI(r interface{}) apiRule {
 	if ar, ok := r.(*rule.AlertingRule); ok {
 		return alertingToAPI(ar)
 	}

app/vmalert/web_types_test.go

@@ -13,11 +13,11 @@ func TestUrlValuesToStrings(t *testing.T) {
 	res := urlValuesToStrings(mapQueryParams)
 
 	if len(res) != len(expectedRes) {
-		t.Fatalf("Expected length %d, but got %d", len(expectedRes), len(res))
+		t.Errorf("Expected length %d, but got %d", len(expectedRes), len(res))
 	}
 	for ind, val := range expectedRes {
 		if val != res[ind] {
-			t.Fatalf("Expected %v; but got %v", val, res[ind])
+			t.Errorf("Expected %v; but got %v", val, res[ind])
 		}
 	}
 }

app/vmauth/auth_config.go

@@ -83,6 +83,7 @@ type UserInfo struct {
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
+	overrideHostHeader      bool
 
 	rt http.RoundTripper
 
@@ -93,9 +94,8 @@ type UserInfo struct {
 
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
-	RequestHeaders   []*Header `yaml:"headers,omitempty"`
-	ResponseHeaders  []*Header `yaml:"response_headers,omitempty"`
-	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
+	RequestHeaders  []*Header `yaml:"headers,omitempty"`
+	ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
 }
 
 func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -129,7 +129,7 @@ type Header struct {
 }
 
 // UnmarshalYAML unmarshals h from f.
-func (h *Header) UnmarshalYAML(f func(any) error) error {
+func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@@ -146,10 +146,19 @@ func (h *Header) UnmarshalYAML(f func(any) error) error {
 }
 
 // MarshalYAML marshals h to yaml.
-func (h *Header) MarshalYAML() (any, error) {
+func (h *Header) MarshalYAML() (interface{}, error) {
 	return h.sOriginal, nil
 }
 
+func overrideHostHeader(headers []*Header) bool {
+	for _, h := range headers {
+		if h.Name == "Host" && h.Value == "" {
+			return true
+		}
+	}
+	return false
+}
+
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
 	// SrcPaths is an optional list of regular expressions, which must match the request path.
@@ -192,7 +201,7 @@ type QueryArg struct {
 }
 
 // UnmarshalYAML unmarshals qa from yaml.
-func (qa *QueryArg) UnmarshalYAML(f func(any) error) error {
+func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@@ -221,7 +230,7 @@ func (qa *QueryArg) UnmarshalYAML(f func(any) error) error {
 }
 
 // MarshalYAML marshals qa to yaml.
-func (qa *QueryArg) MarshalYAML() (any, error) {
+func (qa *QueryArg) MarshalYAML() (interface{}, error) {
 	return qa.sOriginal, nil
 }
 
@@ -254,7 +263,7 @@ type URLPrefix struct {
 	nextDiscoveryDeadline atomic.Uint64
 
 	// vOriginal contains the original yaml value for URLPrefix.
-	vOriginal any
+	vOriginal interface{}
 }
 
 func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
@@ -301,18 +310,12 @@ func (up *URLPrefix) getBackendsCount() int {
 
 // getBackendURL returns the backendURL depending on the load balance policy.
 //
-// It can return nil if there are no backend urls available at the moment.
-//
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
 	pbus := up.bus.Load()
 	bus := *pbus
-	if len(bus) == 0 {
-		return nil
-	}
-
 	if up.loadBalancingPolicy == "first_available" {
 		return getFirstAvailableBackendURL(bus)
 	}
@@ -351,12 +354,11 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 			// ips for the given host have been already discovered
 			continue
 		}
-
 		var resolvedAddrs []string
 		if strings.HasPrefix(host, "srv+") {
 			// The host has the format 'srv+realhost'. Strip 'srv+' prefix before performing the lookup.
-			srvHost := strings.TrimPrefix(host, "srv+")
-			_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", srvHost)
+			host = strings.TrimPrefix(host, "srv+")
+			_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", host)
 			if err != nil {
 				logger.Warnf("cannot discover backend SRV records for %s: %s; use it literally", bu, err)
 				resolvedAddrs = []string{host}
@@ -388,13 +390,14 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	var busNew []*backendURL
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
+		host = strings.TrimPrefix(host, "srv+")
 		port := bu.Port()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
 			if port != "" {
 				if n := strings.IndexByte(buCopy.Host, ':'); n >= 0 {
-					// Drop the discovered port and substitute it the port specified in bu.
+					// Drop the discovered port and substitute it the the port specified in bu.
 					buCopy.Host = buCopy.Host[:n]
 				}
 				buCopy.Host += ":" + port
@@ -461,7 +464,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	}
 
 	// Slow path - select other backend urls.
-	n := atomicCounter.Add(1) - 1
+	n := atomicCounter.Add(1)
 
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
@@ -494,8 +497,8 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 }
 
 // UnmarshalYAML unmarshals up from yaml.
-func (up *URLPrefix) UnmarshalYAML(f func(any) error) error {
-	var v any
+func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
+	var v interface{}
 	if err := f(&v); err != nil {
 		return err
 	}
@@ -505,7 +508,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(any) error) error {
 	switch x := v.(type) {
 	case string:
 		urls = []string{x}
-	case []any:
+	case []interface{}:
 		if len(x) == 0 {
 			return fmt.Errorf("`url_prefix` must contain at least a single url")
 		}
@@ -535,7 +538,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(any) error) error {
 }
 
 // MarshalYAML marshals up to yaml.
-func (up *URLPrefix) MarshalYAML() (any, error) {
+func (up *URLPrefix) MarshalYAML() (interface{}, error) {
 	return up.vOriginal, nil
 }
 
@@ -559,7 +562,7 @@ func (r *Regex) match(s string) bool {
 }
 
 // UnmarshalYAML implements yaml.Unmarshaler
-func (r *Regex) UnmarshalYAML(f func(any) error) error {
+func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@@ -576,7 +579,7 @@ func (r *Regex) UnmarshalYAML(f func(any) error) error {
 }
 
 // MarshalYAML implements yaml.Marshaler.
-func (r *Regex) MarshalYAML() (any, error) {
+func (r *Regex) MarshalYAML() (interface{}, error) {
 	return r.sOriginal, nil
 }
 
@@ -597,7 +600,7 @@ func initAuthConfig() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	_, err := reloadAuthConfig()
+	_, err := loadAuthConfig()
 	if err != nil {
 		logger.Fatalf("cannot load auth config: %s", err)
 	}
@@ -629,7 +632,7 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 
 	updateFn := func() {
 		configReloads.Inc()
-		updated, err := reloadAuthConfig()
+		updated, err := loadAuthConfig()
 		if err != nil {
 			logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
 			configSuccess.Set(0)
@@ -655,45 +658,27 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 	}
 }
 
+// authConfigData stores the yaml definition for this config.
+// authConfigData needs to be updated each time authConfig is updated.
+var authConfigData atomic.Pointer[[]byte]
+
 var (
-	// authConfigData stores the yaml definition for this config.
-	// authConfigData needs to be updated each time authConfig is updated.
-	authConfigData atomic.Pointer[[]byte]
-
-	// authConfig contains the currently loaded auth config
-	authConfig atomic.Pointer[AuthConfig]
-
-	// authUsers contains the currently loaded auth users
-	authUsers atomic.Pointer[map[string]*UserInfo]
-
+	authConfig   atomic.Pointer[AuthConfig]
+	authUsers    atomic.Pointer[map[string]*UserInfo]
 	authConfigWG sync.WaitGroup
 	stopCh       chan struct{}
 )
 
-// reloadAuthConfig loads and applies the config from *authConfigPath.
+// loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
-func reloadAuthConfig() (bool, error) {
+func loadAuthConfig() (bool, error) {
 	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
 
-	ok, err := reloadAuthConfigData(data)
-	if err != nil {
-		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
-	}
-	if !ok {
-		return false, nil
-	}
-
-	mp := authUsers.Load()
-	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
-	return true, nil
-}
-
-func reloadAuthConfigData(data []byte) (bool, error) {
 	oldData := authConfigData.Load()
 	if oldData != nil && bytes.Equal(data, *oldData) {
 		// there are no updates in the config - skip reloading.
@@ -702,23 +687,31 @@ func reloadAuthConfigData(data []byte) (bool, error) {
 
 	ac, err := parseAuthConfig(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse auth config: %w", err)
+		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
 	}
 
 	m, err := parseAuthConfigUsers(ac)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse users from auth config: %w", err)
+		return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
 	}
+	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
-	acPrev := authConfig.Load()
-	if acPrev != nil {
-		metrics.UnregisterSet(acPrev.ms, true)
+	prevAc := authConfig.Load()
+	if prevAc != nil {
+		metrics.UnregisterSet(prevAc.ms)
 	}
 	metrics.RegisterSet(ac.ms)
-
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
+	if prevAc != nil {
+		// explicilty unregister metrics, since all summary type metrics
+		// are registered at global state of metrics package
+		// and must be removed from it to release memory.
+		// Metrics must be unregistered only after atomic.Value.Store calls above
+		// Otherwise it may lead to metric gaps, since UnregisterAllMetrics is slow operation
+		prevAc.ms.UnregisterAllMetrics()
+	}
 
 	return true, nil
 }
@@ -755,6 +748,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
+		ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
 
 		metricLabels, err := ui.getMetricLabels()
 		if err != nil {
@@ -819,6 +813,7 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+		ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
 
 		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
@@ -1009,14 +1004,6 @@ func getAuthTokensFromRequest(r *http.Request) []string {
 		}
 	}
 
-	// Authorization via http://user:pass@hosname/path
-	if u := r.URL.User; u != nil && u.Username() != "" {
-		username := u.Username()
-		password, _ := u.Password()
-		at := getHTTPAuthBasicToken(username, password)
-		ats = append(ats, at)
-	}
-
 	return ats
 }
 
@@ -1037,11 +1024,17 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 		}
 	}
 	up.bus.Store(&bus)
+	up.nextDiscoveryDeadline.Store(0)
+	up.n.Store(0)
 
 	return nil
 }
 
 func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
+	// Remove trailing '/' from urlPrefix
+	for strings.HasSuffix(urlPrefix.Path, "/") {
+		urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
+	}
 	// Validate urlPrefix
 	if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
 		return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)

app/vmauth/auth_config_test.go

@@ -82,14 +82,6 @@ users:
   headers: foobar
 `)
 
-	// Invalid keep_original_host value
-	f(`
-users:
-- username: foo
-  url_prefix: http://foo.bar
-  keep_original_host: foobar
-`)
-
 	// empty url_prefix
 	f(`
 users:
@@ -418,7 +410,7 @@ users:
 				HeadersConf: HeadersConf{
 					RequestHeaders: []*Header{
 						mustNewHeader("'foo: bar'"),
-						mustNewHeader("'xxx:'"),
+						mustNewHeader("'xxx: y'"),
 					},
 				},
 			},
@@ -437,7 +429,7 @@ users:
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
     headers:
     - "foo: bar"
-    - "xxx:"
+    - "xxx: y"
 `, map[string]*UserInfo{
 		getHTTPAuthBearerToken("foo"):    sharedUserInfo,
 		getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
@@ -466,7 +458,6 @@ users:
 	})
 
 	// with default url
-	keepOriginalHost := true
 	f(`
 users:
 - bearer_token: foo
@@ -478,7 +469,6 @@ users:
     headers:
     - "foo: bar"
     - "xxx: y"
-    keep_original_host: true
   default_url:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
@@ -501,7 +491,6 @@ users:
 							mustNewHeader("'foo: bar'"),
 							mustNewHeader("'xxx: y'"),
 						},
-						KeepOriginalHost: &keepOriginalHost,
 					},
 				},
 			},
@@ -528,7 +517,6 @@ users:
 							mustNewHeader("'foo: bar'"),
 							mustNewHeader("'xxx: y'"),
 						},
-						KeepOriginalHost: &keepOriginalHost,
 					},
 				},
 			},
@@ -548,17 +536,12 @@ users:
   metric_labels:
     dc: eu
     team: dev
-  keep_original_host: true
 - username: foo-same
   password: bar
   url_prefix: https://bar/x
   metric_labels:
     backend_env: test
     team: accounting
-  headers:
-  - "foo: bar"
-  response_headers:
-  - "Abc: def"
 `, map[string]*UserInfo{
 		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
@@ -568,9 +551,6 @@ users:
 				"dc":   "eu",
 				"team": "dev",
 			},
-			HeadersConf: HeadersConf{
-				KeepOriginalHost: &keepOriginalHost,
-			},
 		},
 		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
@@ -580,14 +560,6 @@ users:
 				"backend_env": "test",
 				"team":        "accounting",
 			},
-			HeadersConf: HeadersConf{
-				RequestHeaders: []*Header{
-					mustNewHeader("'foo: bar'"),
-				},
-				ResponseHeaders: []*Header{
-					mustNewHeader("'Abc: def'"),
-				},
-			},
 		},
 	})
 }
@@ -723,9 +695,9 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 
 	fn := func(ns ...int) {
 		t.Helper()
-		pbus := up.bus.Load()
-		bus := *pbus
-		for i, b := range bus {
+		bus := up.bus.Load()
+		pbus := *bus
+		for i, b := range pbus {
 			got := int(b.concurrentRequests.Load())
 			exp := ns[i]
 			if got != exp {
@@ -735,39 +707,39 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	}
 
 	up.getBackendURL()
-	fn(1, 0, 0)
+	fn(0, 1, 0)
 	up.getBackendURL()
-	fn(1, 1, 0)
+	fn(0, 1, 1)
 	up.getBackendURL()
 	fn(1, 1, 1)
 
 	up.getBackendURL()
 	up.getBackendURL()
-	fn(2, 2, 1)
+	fn(1, 2, 2)
 
 	bus := up.bus.Load()
 	pbus := *bus
 	pbus[0].concurrentRequests.Add(2)
 	pbus[2].concurrentRequests.Add(5)
-	fn(4, 2, 6)
+	fn(3, 2, 7)
 
 	up.getBackendURL()
-	fn(4, 3, 6)
+	fn(3, 3, 7)
 
 	up.getBackendURL()
-	fn(4, 4, 6)
+	fn(3, 4, 7)
 
 	up.getBackendURL()
-	fn(4, 5, 6)
+	fn(4, 4, 7)
 
 	up.getBackendURL()
-	fn(5, 5, 6)
+	fn(5, 4, 7)
 
 	up.getBackendURL()
-	fn(6, 5, 6)
+	fn(5, 5, 7)
 
 	up.getBackendURL()
-	fn(6, 6, 6)
+	fn(6, 5, 7)
 
 	up.getBackendURL()
 	fn(6, 6, 7)

app/vmauth/main.go

@@ -37,20 +37,20 @@ var (
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
 		"See also -maxConcurrentRequests")
-	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
-		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
+	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist.
+    Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
 	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
 		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
 		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
 		"in per-user config")
-	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
 	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
-		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
+		"Bigger values may require more memory. Negative or zero values disable request body caching and retries.")
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -99,7 +99,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
 			return true
 		}
 		configReloadRequests.Inc()
@@ -117,7 +117,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 
-		handleMissingAuthorizationError(w)
+		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 		return true
 	}
 
@@ -188,27 +189,26 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			// to a route that is not in the configuration for unauthorized user.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5236
 			if ui.BearerToken == "" && ui.Username == "" && len(*authUsers.Load()) > 0 {
-				handleMissingAuthorizationError(w)
+				w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+				http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
 				return
 			}
 			missingRouteRequests.Inc()
-			httpserver.Errorf(w, r, "missing route for %s", u.String())
+			httpserver.Errorf(w, r, "missing route for %q", u.String())
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
 		isDefault = true
 	}
-
-	rtb := getReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	defer putReadTrackingBody(rtb)
-	r.Body = rtb
-
+	// caching makes sense only for positive non zero size
+	if maxRequestBodySizeToRetry.IntN() > 0 {
+		rtb := getReadTrackingBody(r.Body, int(r.ContentLength))
+		defer putReadTrackingBody(rtb)
+		r.Body = rtb
+	}
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
-		if bu == nil {
-			break
-		}
 		targetURL := bu.url
 		// Don't change path and add request_path query param for default route.
 		if isDefault {
@@ -218,15 +218,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-
-		wasLocalRetry := false
-	again:
-		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
-		if needLocalRetry && !wasLocalRetry {
-			wasLocalRetry = true
-			goto again
-		}
-
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		bu.put()
 		if ok {
 			return
@@ -234,28 +226,29 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		bu.setBroken()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
-		StatusCode: http.StatusBadGateway,
+		Err:        fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
+		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
 	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
+	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
-
 	req.URL = targetURL
-	req.Header.Set("User-Agent", "vmauth")
-	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	if hc.KeepOriginalHost == nil || !*hc.KeepOriginalHost {
-		if host := getHostHeader(hc.RequestHeaders); host != "" {
-			req.Host = host
-		} else {
-			req.Host = targetURL.Host
-		}
-	}
 
+	if req.URL.Scheme == "https" || ui.overrideHostHeader {
+		// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
+		// so it expects the targetURL.Host in the request.
+		// There is no need in overriding the req.Host for http requests, since it is expected that backend server
+		// may properly process queries with the original req.Host.
+		req.Host = targetURL.Host
+	}
+	updateHeadersByConfig(req.Header, hc.RequestHeaders)
+	var trivialRetries int
 	rtb, rtbOK := req.Body.(*readTrackingBody)
+again:
 	res, err := ui.rt.RoundTrip(req)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -267,7 +260,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 				// Timed out request must be counted as errors, since this usually means that the backend is slow.
 				ui.backendErrors.Inc()
 			}
-			return true, false
+			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
@@ -277,19 +270,20 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
-			return true, false
+			return true
 		}
-		if netutil.IsTrivialNetworkError(err) {
-			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
-			return false, true
+		// one time retry trivial network errors, such as proxy idle timeout misconfiguration
+		// or socket close by OS
+		if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 {
+			trivialRetries++
+			goto again
 		}
-
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
-		return false, false
+		return false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
 		_ = res.Body.Close()
@@ -303,7 +297,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
-			return true, false
+			return true
 		}
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
@@ -312,7 +306,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
 			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
-		return false, false
+		return false
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
@@ -328,9 +322,9 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
-		return true, false
+		return true
 	}
-	return true, false
+	return true
 }
 
 var copyBufPool bytesutil.ByteBufferPool
@@ -343,21 +337,12 @@ func copyHeader(dst, src http.Header) {
 	}
 }
 
-func getHostHeader(headers []*Header) string {
-	for _, h := range headers {
-		if h.Name == "Host" {
-			return h.Value
-		}
-	}
-	return ""
-}
-
-func updateHeadersByConfig(dst http.Header, src []*Header) {
-	for _, h := range src {
+func updateHeadersByConfig(headers http.Header, config []*Header) {
+	for _, h := range config {
 		if h.Value == "" {
-			dst.Del(h.Name)
+			headers.Del(h.Name)
 		} else {
-			dst.Set(h.Name, h.Value)
+			headers.Set(h.Name, h.Value)
 		}
 	}
 }
@@ -465,7 +450,7 @@ func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, i
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
-	tr.DialContext = netutil.NewStatDialFunc("vmauth_backend")
+	tr.DialContext = netutil.DialMaybeSRV
 
 	rt := cfg.NewRoundTripper(tr)
 	return rt, nil
@@ -497,11 +482,6 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
 	flagutil.Usage(s)
 }
 
-func handleMissingAuthorizationError(w http.ResponseWriter) {
-	w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-	http.Error(w, "missing 'Authorization' request header", http.StatusUnauthorized)
-}
-
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
@@ -511,124 +491,85 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-// readTrackingBody must be obtained via getReadTrackingBody()
 type readTrackingBody struct {
-	// maxBodySize is the maximum body size to cache in buf.
-	//
-	// Bigger bodies cannot be retried.
-	maxBodySize int
-
 	// r contains reader for initial data reading
 	r io.ReadCloser
 
-	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
-	// If more than maxBodySize is read from r, then cannotRetry is set to true.
+	// buf is a buffer for data read from r. Buf size is limited by maxRequestBodySizeToRetry.
+	// If more than maxRequestBodySizeToRetry is read from r, then cannotRetry is set to true.
 	buf []byte
 
-	// readBuf points to the cached data at buf, which must be read in the next call to Read().
-	readBuf []byte
-
-	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
+	// cannotRetry is set to true when more than maxRequestBodySizeToRetry are read from r.
 	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
 	cannotRetry bool
 
 	// bufComplete is set to true when buf contains complete request body read from r.
 	bufComplete bool
+
+	// offset is an offset at buf for the next data read if needReadBuf is set to true.
+	offset int
 }
 
-func (rtb *readTrackingBody) reset() {
-	rtb.maxBodySize = 0
-	rtb.r = nil
-	rtb.buf = rtb.buf[:0]
-	rtb.readBuf = nil
-	rtb.cannotRetry = false
-	rtb.bufComplete = false
-}
-
-func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	v := readTrackingBodyPool.Get()
-	if v == nil {
-		v = &readTrackingBody{}
-	}
-	rtb := v.(*readTrackingBody)
-
-	if maxBodySize < 0 {
-		maxBodySize = 0
-	}
-	rtb.maxBodySize = maxBodySize
-
-	if r == nil {
-		// This is GET request without request body
-		r = (*zeroReader)(nil)
-	}
-	rtb.r = r
-	return rtb
-}
-
-type zeroReader struct{}
-
-func (r *zeroReader) Read(_ []byte) (int, error) {
-	return 0, io.EOF
-}
-func (r *zeroReader) Close() error {
-	return nil
-}
-
-func putReadTrackingBody(rtb *readTrackingBody) {
-	rtb.reset()
-	readTrackingBodyPool.Put(rtb)
-}
-
-var readTrackingBodyPool sync.Pool
-
-// Read implements io.Reader interface.
+// Read implements io.Reader interface
+// tracks body reading requests
 func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if len(rtb.readBuf) > 0 {
-		n := copy(p, rtb.readBuf)
-		rtb.readBuf = rtb.readBuf[n:]
-		return n, nil
-	}
-
-	if rtb.r == nil {
-		if rtb.bufComplete {
-			return 0, io.EOF
+	if rtb.offset < len(rtb.buf) {
+		if rtb.cannotRetry {
+			return 0, fmt.Errorf("cannot retry reading data from buf")
 		}
-		return 0, fmt.Errorf("cannot read client request body after closing client reader")
+		nb := copy(p, rtb.buf[rtb.offset:])
+		rtb.offset += nb
+		if rtb.bufComplete {
+			if rtb.offset == len(rtb.buf) {
+				return nb, io.EOF
+			}
+			return nb, nil
+		}
+		if nb < len(p) {
+			nr, err := rtb.readFromStream(p[nb:])
+			return nb + nr, err
+		}
+		return nb, nil
 	}
+	if rtb.bufComplete {
+		return 0, io.EOF
+	}
+	return rtb.readFromStream(p)
+}
 
+func (rtb *readTrackingBody) readFromStream(p []byte) (int, error) {
+	if rtb.r == nil {
+		return 0, fmt.Errorf("cannot read data after closing the reader")
+	}
 	n, err := rtb.r.Read(p)
 	if rtb.cannotRetry {
 		return n, err
 	}
-
-	if len(rtb.buf)+n > rtb.maxBodySize {
+	if rtb.offset+n > maxRequestBodySizeToRetry.IntN() {
+		rtb.cannotRetry = true
+	}
+	if n > 0 {
+		rtb.offset += n
+		rtb.buf = append(rtb.buf, p[:n]...)
+	}
+	if err != nil {
+		if err == io.EOF {
+			rtb.bufComplete = true
+			return n, err
+		}
 		rtb.cannotRetry = true
 		return n, err
 	}
-	rtb.buf = append(rtb.buf, p[:n]...)
-	if err == io.EOF {
-		rtb.bufComplete = true
-	}
-	return n, err
+	return n, nil
 }
 
 func (rtb *readTrackingBody) canRetry() bool {
-	if rtb.cannotRetry {
-		return false
-	}
-	if rtb.bufComplete {
-		return true
-	}
-	return rtb.r != nil
+	return !rtb.cannotRetry
 }
 
 // Close implements io.Closer interface.
 func (rtb *readTrackingBody) Close() error {
-	if !rtb.cannotRetry {
-		rtb.readBuf = rtb.buf
-	} else {
-		rtb.readBuf = nil
-	}
+	rtb.offset = 0
 
 	// Close rtb.r only if the request body is completely read or if it is too big.
 	// http.Roundtrip performs body.Close call even without any Read calls,
@@ -644,3 +585,38 @@ func (rtb *readTrackingBody) Close() error {
 
 	return nil
 }
+
+var readTrackingBodyPool sync.Pool
+
+func getReadTrackingBody(origin io.ReadCloser, b int) *readTrackingBody {
+	bufSize := 1024
+	if b > 0 && b < maxRequestBodySizeToRetry.IntN() {
+		bufSize = b
+	}
+	v := readTrackingBodyPool.Get()
+	if v == nil {
+		v = &readTrackingBody{
+			buf: make([]byte, 0, bufSize),
+		}
+	}
+	rtb := v.(*readTrackingBody)
+	rtb.r = origin
+	if bufSize > cap(rtb.buf) {
+		rtb.buf = make([]byte, 0, bufSize)
+	}
+
+	return rtb
+}
+
+func putReadTrackingBody(rtb *readTrackingBody) {
+	if rtb.r != nil {
+		_ = rtb.r.Close()
+	}
+	rtb.r = nil
+	rtb.buf = rtb.buf[:0]
+	rtb.offset = 0
+	rtb.cannotRetry = false
+	rtb.bufComplete = false
+
+	readTrackingBodyPool.Put(rtb)
+}

app/vmauth/main_test.go

@@ -2,520 +2,18 @@ package main
 
 import (
 	"bytes"
-	"fmt"
 	"io"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"sync/atomic"
 	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
-func TestRequestHandler(t *testing.T) {
-	f := func(cfgStr, requestURL string, backendHandler http.HandlerFunc, responseExpected string) {
+func TestReadTrackingBodyRetrySuccess(t *testing.T) {
+	f := func(s string) {
 		t.Helper()
-
-		ts := httptest.NewServer(backendHandler)
-		defer ts.Close()
-
-		cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
-		responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
-
-		cfgOrigP := authConfigData.Load()
-		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
-			t.Fatalf("cannot load config data: %s", err)
+		rtb := &readTrackingBody{
+			r: io.NopCloser(bytes.NewBufferString(s)),
 		}
-		defer func() {
-			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
-			if cfgOrigP != nil {
-				cfgOrig = *cfgOrigP
-			}
-			_, err := reloadAuthConfigData(cfgOrig)
-			if err != nil {
-				t.Fatalf("cannot load the original config: %s", err)
-			}
-		}()
-
-		r, err := http.NewRequest(http.MethodGet, requestURL, nil)
-		if err != nil {
-			t.Fatalf("cannot initialize http request: %s", err)
-		}
-
-		r.RequestURI = r.URL.RequestURI()
-		r.RemoteAddr = "42.2.3.84:6789"
-		r.Header.Set("X-Forwarded-For", "12.34.56.78")
-		r.Header.Set("Connection", "Some-Header,Other-Header")
-		r.Header.Set("Some-Header", "foobar")
-		r.Header.Set("Pass-Header", "abc")
-
-		w := &fakeResponseWriter{}
-		if !requestHandler(w, r) {
-			t.Fatalf("unexpected false is returned from requestHandler")
-		}
-
-		response := w.getResponse()
-		response = strings.ReplaceAll(response, "\r\n", "\n")
-		response = strings.TrimSpace(response)
-		responseExpected = strings.TrimSpace(responseExpected)
-		if response != responseExpected {
-			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
-		}
-	}
-
-	// regular url_prefix
-	cfgStr := `
-unauthorized_user:
-  url_prefix: {BACKEND}/foo?bar=baz`
-	requestURL := "http://some-host.com/abc/def?some_arg=some_value"
-	backendHandler := func(w http.ResponseWriter, r *http.Request) {
-		h := w.Header()
-		h.Set("Connection", "close")
-		h.Set("Foo", "bar")
-
-		var bb bytes.Buffer
-		if err := r.Header.Write(&bb); err != nil {
-			panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
-		}
-		fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
-	}
-	responseExpected := `
-statusCode=200
-Foo: bar
-requested_url={BACKEND}/foo/abc/def?bar=baz&some_arg=some_value
-Pass-Header: abc
-User-Agent: vmauth
-X-Forwarded-For: 12.34.56.78, 42.2.3.84`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// keep_original_host
-	cfgStr = `
-unauthorized_user:
-  url_prefix: "{BACKEND}/foo?bar=baz"
-  keep_original_host: true`
-	requestURL = "http://some-host.com/abc/def"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url=http://some-host.com/foo/abc/def?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// override user-agent header
-	cfgStr = `
-unauthorized_user:
-  url_prefix: "{BACKEND}/foo?bar=baz"
-  headers:
-  - "User-Agent: foobar"`
-	requestURL = "http://some-host.com/abc/def"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/foo/abc/def?bar=baz
-User-Agent=foobar`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// delete user-agent header
-	cfgStr = `
-unauthorized_user:
-  url_prefix: "{BACKEND}/foo?bar=baz"
-  headers:
-  - "User-Agent:"`
-	requestURL = "http://some-host.com/abc/def"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/foo/abc/def?bar=baz
-User-Agent=Go-http-client/1.1`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// override request host with non-empty host
-	cfgStr = `
-unauthorized_user:
-  url_prefix: "{BACKEND}/foo?bar=baz"
-  headers:
-  - "Host: other-host:12345"
-  - "abc:"`
-	requestURL = "http://some-host.com/abc/def"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url=http://other-host:12345/foo/abc/def?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// override request host with empty host
-	cfgStr = `
-unauthorized_user:
-  url_prefix: "{BACKEND}/foo?bar=baz"
-  headers:
-  - "Host:"`
-	requestURL = "http://some-host.com/abc/def"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/foo/abc/def?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// /-/reload handler failure
-	origAuthKey := reloadAuthKey.Get()
-	if err := reloadAuthKey.Set("secret"); err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	cfgStr = `
-unauthorized_user:
-  url_prefix: "{BACKEND}/foo"`
-	requestURL = "http://some-host.com/-/reload"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=401
-The provided authKey doesn't match -reloadAuthKey`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-	if err := reloadAuthKey.Set(origAuthKey); err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-
-	// missing authorization
-	cfgStr = `
-users:
-- username: foo
-  url_prefix: "{BACKEND}/bar"`
-	requestURL = "http://some-host.com/a/b"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=401
-Www-Authenticate: Basic realm="Restricted"
-missing 'Authorization' request header`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// incorrect authorization
-	cfgStr = `
-users:
-- username: foo
-  password: secret
-  url_prefix: "{BACKEND}/bar"`
-	requestURL = "http://foo:invalid-secret@some-host.com/a/b"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=401
-Unauthorized`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// incorrect authorization with logging invalid auth tokens
-	origLogInvalidAuthTokens := *logInvalidAuthTokens
-	*logInvalidAuthTokens = true
-	cfgStr = `
-users:
-- username: foo
-  password: secret
-  url_prefix: "{BACKEND}/bar"`
-	requestURL = "http://foo:invalid-secret@some-host.com/a/b?c=d"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=401
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /a/b?c=d; cannot authorize request with auth tokens ["http_auth:Basic Zm9vOmludmFsaWQtc2VjcmV0"]`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-	*logInvalidAuthTokens = origLogInvalidAuthTokens
-
-	// correct authorization
-	cfgStr = `
-users:
-- username: foo
-  password: secret
-  url_prefix: "{BACKEND}/bar"`
-	requestURL = "http://foo:secret@some-host.com/a/b"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/bar/a/b`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// verify how path cleanup works
-	cfgStr = `
-unauthorized_user:
-  url_prefix: {BACKEND}/foo?bar=baz`
-	requestURL = "http://some-host.com/../../a//.///bar/"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/foo/a/bar/?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// verify how path cleanup works for url without path
-	cfgStr = `
-unauthorized_user:
-  url_prefix: {BACKEND}/foo?bar=baz`
-	requestURL = "http://some-host.com/"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/foo?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// verify how path cleanup works for url without path if url_prefix path ends with /
-	cfgStr = `
-unauthorized_user:
-  url_prefix: {BACKEND}/foo/?bar=baz`
-	requestURL = "http://some-host.com/"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/foo/?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// verify how path cleanup works for url without path and the url_prefix without path prefix
-	cfgStr = `
-unauthorized_user:
-  url_prefix: {BACKEND}/?bar=baz`
-	requestURL = "http://some-host.com/"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/?bar=baz`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// verify routing to default_url
-	cfgStr = `
-unauthorized_user:
-  url_map:
-  - src_paths: ["/foo/.+"]
-    url_prefix: {BACKEND}/x-foo/
-  default_url: {BACKEND}/404.html`
-	requestURL = "http://some-host.com/abc?de=fg"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/404.html?request_path=http%3A%2F%2Fsome-host.com%2Fabc%3Fde%3Dfg`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// verify routing to default url_prefix
-	cfgStr = `
-unauthorized_user:
-  url_map:
-  - src_paths: ["/foo/.+"]
-    url_prefix: {BACKEND}/x-foo/
-  url_prefix: {BACKEND}/default`
-	requestURL = "http://some-host.com/abc?de=fg"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/default/abc?de=fg`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// missing default_url and default url_prefix for unauthorized user
-	cfgStr = `
-unauthorized_user:
-  url_map:
-  - src_paths: ["/foo/.+"]
-    url_prefix: {BACKEND}/x-foo/`
-	requestURL = "http://some-host.com/abc?de=fg"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=400
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users
-	cfgStr = `
-users:
-- username: some-user
-  url_map:
-  - src_paths: ["/foo/.+"]
-    url_prefix: {BACKEND}/x-foo/
-unauthorized_user:
-  url_map:
-  - src_paths: ["/abc/.*"]
-    url_prefix: {BACKEND}/x-bar`
-	requestURL = "http://some-host.com/abc?de=fg"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=401
-Www-Authenticate: Basic realm="Restricted"
-missing 'Authorization' request header`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// all the backend_urls are unavailable for unauthorized user
-	cfgStr = `
-unauthorized_user:
-  url_map:
-  - src_paths: ["/foo/.*"]
-    url_prefix:
-    - http://127.0.0.1:1/
-    - http://127.0.0.1:2/`
-	requestURL = "http://some-host.com/foo/?de=fg"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=502
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// all the backend_urls are unavailable for authorized user
-	cfgStr = `
-users:
-- username: some-user
-  url_map:
-  - src_paths: ["/foo/.*"]
-    url_prefix:
-    - http://127.0.0.1:1/
-    - http://127.0.0.1:2/`
-	requestURL = "http://some-user@some-host.com/foo/?de=fg"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=502
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "some-user" are unavailable`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-
-	// zero discovered backend IPs
-	customResolver := &fakeResolver{
-		Resolver: &net.Resolver{},
-		lookupIPAddrResults: map[string][]net.IPAddr{
-			"some-addr": {},
-		},
-	}
-	origResolver := netutil.Resolver
-	netutil.Resolver = customResolver
-	cfgStr = `
-unauthorized_user:
-  url_prefix: ['http://some-addr:1234/foo/bar']
-  discover_backend_ips: true`
-	requestURL = "http://abc.com/def/?de=fg"
-	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
-		panic(fmt.Errorf("backend handler shouldn't be called"))
-	}
-	responseExpected = `
-statusCode=502
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /def/?de=fg; all the 0 backends for the user "" are unavailable`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-	netutil.Resolver = origResolver
-
-	// retry_status_codes failure
-	var retries atomic.Int64
-	cfgStr = `
-unauthorized_user:
-  url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
-  retry_status_codes: [500, 502]`
-	requestURL = "http://some-host.com/foo/?de=fg"
-	backendHandler = func(w http.ResponseWriter, _ *http.Request) {
-		retries.Add(1)
-		w.WriteHeader(500)
-	}
-	responseExpected = `
-statusCode=502
-remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-	if n := retries.Load(); n != 2 {
-		t.Fatalf("unexpected number of retries; got %d; want 2", n)
-	}
-
-	// retry_status_codes success
-	retries.Store(0)
-	cfgStr = `
-unauthorized_user:
-  url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
-  retry_status_codes: [500, 502]`
-	requestURL = "http://some-host.com/foo/?de=fg"
-	backendHandler = func(w http.ResponseWriter, r *http.Request) {
-		if n := retries.Add(1); n < 2 {
-			w.WriteHeader(500)
-			return
-		}
-		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
-	}
-	responseExpected = `
-statusCode=200
-requested_url={BACKEND}/path2/foo/?de=fg`
-	f(cfgStr, requestURL, backendHandler, responseExpected)
-	if n := retries.Load(); n != 2 {
-		t.Fatalf("unexpected number of retries; got %d; want 2", n)
-	}
-}
-
-type fakeResponseWriter struct {
-	h http.Header
-
-	bb bytes.Buffer
-}
-
-func (w *fakeResponseWriter) getResponse() string {
-	return w.bb.String()
-}
-
-func (w *fakeResponseWriter) Header() http.Header {
-	if w.h == nil {
-		w.h = http.Header{}
-	}
-	return w.h
-}
-
-func (w *fakeResponseWriter) Write(p []byte) (int, error) {
-	return w.bb.Write(p)
-}
-
-func (w *fakeResponseWriter) WriteHeader(statusCode int) {
-	fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
-	if w.h == nil {
-		return
-	}
-	err := w.h.WriteSubset(&w.bb, map[string]bool{
-		"Content-Length":         true,
-		"Content-Type":           true,
-		"Date":                   true,
-		"X-Content-Type-Options": true,
-	})
-	if err != nil {
-		panic(fmt.Errorf("cannot marshal headers: %s", err))
-	}
-}
-
-func TestReadTrackingBody_RetrySuccess(t *testing.T) {
-	f := func(s string, maxBodySize int) {
-		t.Helper()
-
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
-
 		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
+			t.Fatalf("canRetry() must return true")
 		}
 		for i := 0; i < 5; i++ {
 			data, err := io.ReadAll(rtb)
@@ -534,99 +32,41 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 		}
 	}
 
-	f("", 0)
-	f("", -1)
-	f("", 100)
-	f("foo", 100)
-	f("foobar", 100)
-	f(newTestString(1000), 1000)
+	f("")
+	f("foo")
+	f("foobar")
+	f(newTestString(maxRequestBodySizeToRetry.IntN()))
 }
 
-func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
-	f := func(s string, maxBodySize int) {
+func TestReadTrackingBodyRetryFailure(t *testing.T) {
+	f := func(s string) {
 		t.Helper()
-
-		// Check the case with partial read
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
-
-		for i := 0; i < len(s); i++ {
-			buf := make([]byte, i)
-			n, err := io.ReadFull(rtb, buf)
-			if err != nil {
-				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
-			}
-			if n != i {
-				t.Fatalf("unexpected number of bytes read; got %d; want %d", n, i)
-			}
-			if string(buf) != s[:i] {
-				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
-			}
-			if err := rtb.Close(); err != nil {
-				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
-			}
-			if !rtb.canRetry() {
-				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
-			}
-		}
-
-		data, err := io.ReadAll(rtb)
-		if err != nil {
-			t.Fatalf("unexpected error when reading all the data: %s", err)
-		}
-		if string(data) != s {
-			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
-		}
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		rtb := &readTrackingBody{
+			r: io.NopCloser(bytes.NewBufferString(s)),
 		}
 		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
-		}
-	}
-
-	f("", 0)
-	f("", -1)
-	f("", 100)
-	f("foo", 100)
-	f("foobar", 100)
-	f(newTestString(1000), 1000)
-}
-
-func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
-	f := func(s string, maxBodySize int) {
-		t.Helper()
-
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
-
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
+			t.Fatalf("canRetry() must return true")
 		}
 		buf := make([]byte, 1)
-		n, err := io.ReadFull(rtb, buf)
+		n, err := rtb.Read(buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true after reading one byte")
-		}
 		data, err := io.ReadAll(rtb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
-		dataRead := string(buf) + string(data)
-		if dataRead != s {
-			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
+		if string(buf)+string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", string(buf)+string(data), s)
 		}
 		if err := rtb.Close(); err != nil {
 			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
 		}
 		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+			t.Fatalf("canRetry() must return false")
 		}
 
 		data, err = io.ReadAll(rtb)
@@ -638,55 +78,132 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 		}
 	}
 
-	const maxBodySize = 1000
-	f(newTestString(maxBodySize+1), maxBodySize)
-	f(newTestString(2*maxBodySize), maxBodySize)
+	f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
+	f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
 }
 
-func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
-	f := func(s string, maxBodySize int) {
-		t.Helper()
+// request body not over maxRequestBodySizeToRetry
+// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
+// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
+// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
+func TestRetryReadSuccessAfterPartialRead(t *testing.T) {
+	f := func(s string) {
+		rtb := &readTrackingBody{
+			r:   io.NopCloser(bytes.NewBufferString(s)),
+			buf: make([]byte, 0, len(s)),
+		}
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		var data []byte
+		var err error
+		halfSize := len(s) / 2
+		if halfSize == 0 {
+			halfSize = 100
+		}
+		buf := make([]byte, halfSize)
+		var n int
 
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true before reading anything")
-		}
-		data, err := io.ReadAll(rtb)
-		if err != nil {
-			t.Fatalf("unexpected error when reading all the data: %s", err)
-		}
-		if string(data) != s {
-			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
+		// read part of the data
+		n, err = rtb.Read(buf[:])
+		data = append(data, buf[:n]...)
+		if err != nil && err != io.EOF {
+			t.Fatalf("unexpected error: %s", err)
 		}
+
+		// request failed when output stream is closed (eg: server connection reset)
+		// would close the reader
 		if err := rtb.Close(); err != nil {
 			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
 		}
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true")
+		}
 
-		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false after closing the reader")
+		// retry read (read buf + remaining data)
+		data = data[:0]
+		err = nil
+		for err == nil {
+			n, err = rtb.Read(buf[:])
+			data = append(data, buf[:n]...)
 		}
-		data, err = io.ReadAll(rtb)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
+		if err != io.EOF {
+			t.Fatalf("unexpected error: %s", err)
 		}
-		if len(data) != 0 {
-			t.Fatalf("unexpected non-empty data read: %q", data)
+		if string(data) != s {
+			t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
+		}
+		// cannotRetry return false
+		// because the request data is not over maxRequestBodySizeToRetry limit
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true")
 		}
 	}
 
-	f("foobar", 0)
-	f(newTestString(1000), 0)
+	f("")
+	f("foo")
+	f("foobar")
+	f(newTestString(maxRequestBodySizeToRetry.IntN()))
+}
 
-	f("foobar", -1)
-	f(newTestString(1000), -1)
+// request body over maxRequestBodySizeToRetry
+// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
+// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
+// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
+func TestRetryReadSuccessAfterPartialReadAndCannotRetryAgain(t *testing.T) {
+	f := func(s string) {
+		rtb := &readTrackingBody{
+			r:   io.NopCloser(bytes.NewBufferString(s)),
+			buf: make([]byte, 0, len(s)),
+		}
+
+		var data []byte
+		var err error
+		halfSize := len(s) / 2
+		if halfSize == 0 {
+			halfSize = 100
+		}
+		buf := make([]byte, halfSize)
+		var n int
+
+		// read part of the data
+		n, err = rtb.Read(buf[:])
+		data = append(data, buf[:n]...)
+		if err != nil && err != io.EOF {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		// request failed when output stream is closed (eg: server connection reset)
+		if err := rtb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		}
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true")
+		}
+
+		// retry read (read buf + remaining data)
+		data = data[:0]
+		err = nil
+		for err == nil {
+			n, err = rtb.Read(buf[:])
+			data = append(data, buf[:n]...)
+		}
+		if err != io.EOF {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if string(data) != s {
+			t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
+		}
+
+		// cannotRetry returns true
+		// because the request data is over maxRequestBodySizeToRetry limit
+		if rtb.canRetry() {
+			t.Fatalf("canRetry() must return false")
+		}
+	}
+
+	f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
+	f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
 }
 
 func newTestString(sLen int) string {
-	data := make([]byte, sLen)
-	for i := range data {
-		data[i] = byte(i)
-	}
-	return string(data)
+	return string(make([]byte, sLen))
 }

app/vmauth/target_url_test.go

@@ -86,7 +86,6 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	f := func(ui *UserInfo, requestURI, expectedTarget, expectedRequestHeaders, expectedResponseHeaders string,
 		expectedRetryStatusCodes []int, expectedLoadBalancingPolicy string, expectedDropSrcPathPrefixParts int) {
 		t.Helper()
-
 		if err := ui.initURLs(); err != nil {
 			t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
 		}
@@ -123,7 +122,6 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			t.Fatalf("unexpected dropSrcPathPrefixParts; got %d; want %d", up.dropSrcPathPrefixParts, expectedDropSrcPathPrefixParts)
 		}
 	}
-
 	// Simple routing with `url_prefix`
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
@@ -262,32 +260,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	f(ui, `/api/v1/query?query=up{env="prod"}`, `http://vmselect/1/prometheus/api/v1/query?query=up%7Benv%3D%22prod%22%7D`, "", "", nil, "least_loaded", 0)
 	f(ui, `/api/v1/query?query=up{foo="bar",env="dev",pod!=""}`, `http://vmselect/0/prometheus/api/v1/query?query=up%7Bfoo%3D%22bar%22%2Cenv%3D%22dev%22%2Cpod%21%3D%22%22%7D`, "", "", nil, "least_loaded", 0)
 	f(ui, `/api/v1/query?query=up{foo="bar"}`, `http://default-server/api/v1/query?query=up%7Bfoo%3D%22bar%22%7D`, "", "", nil, "least_loaded", 0)
-}
 
-func TestUserInfoGetBackendURL_SRV(t *testing.T) {
-	f := func(ui *UserInfo, requestURI, expectedTarget string) {
-		t.Helper()
-
-		u, err := url.Parse(requestURI)
-		if err != nil {
-			t.Fatalf("cannot parse %q: %s", requestURI, err)
-		}
-		u = normalizeURL(u)
-		up, _ := ui.getURLPrefixAndHeaders(u, nil)
-		if up == nil {
-			t.Fatalf("cannot match available backend: %s", err)
-		}
-		bu := up.getBackendURL()
-		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
-		bu.put()
-
-		gotTarget := target.String()
-		if gotTarget != expectedTarget {
-			t.Fatalf("unexpected target\ngot:\n%q\nwant\n%q", gotTarget, expectedTarget)
-		}
-	}
-
-	// Discover backendURL with SRV hostnames
 	customResolver := &fakeResolver{
 		Resolver: &net.Resolver{},
 		lookupSRVResults: map[string][]*net.SRV{
@@ -310,14 +283,11 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 			},
 		},
 	}
-	origResolver := netutil.Resolver
 	netutil.Resolver = customResolver
-	defer func() {
-		netutil.Resolver = origResolver
-	}()
 
+	// Discover backendURL
 	allowed := true
-	ui := &UserInfo{
+	ui = &UserInfo{
 		URLMaps: []URLMap{
 			{
 				SrcPaths:  getRegexs([]string{"/select/.+"}),
@@ -331,64 +301,12 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 		DiscoverBackendIPs: &allowed,
 		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
 	}
-	if err := ui.initURLs(); err != nil {
-		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
-	}
-
-	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
-	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
-	f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write")
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
+	// url_prefix counter will be reset, still go to 10.6.142.51
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
+	f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write", "", "", nil, "least_loaded", 0)
 	// unsuccessful dns resolve
-	f(ui, `/test`, "http://non-exist-dns-addr/test")
-}
-
-func TestUserInfoGetBackendURL_SRVZeroBackends(t *testing.T) {
-	f := func(ui *UserInfo, requestURI string) {
-		t.Helper()
-
-		u, err := url.Parse(requestURI)
-		if err != nil {
-			t.Fatalf("cannot parse %q: %s", requestURI, err)
-		}
-		u = normalizeURL(u)
-		up, _ := ui.getURLPrefixAndHeaders(u, nil)
-		if up == nil {
-			t.Fatalf("cannot match available backend: %s", err)
-		}
-		bu := up.getBackendURL()
-		if bu != nil {
-			t.Fatalf("expecting nil backendURL; got %v", bu)
-		}
-	}
-
-	customResolver := &fakeResolver{
-		Resolver: &net.Resolver{},
-		lookupSRVResults: map[string][]*net.SRV{
-			"vmselect": {},
-		},
-	}
-	origResolver := netutil.Resolver
-	netutil.Resolver = customResolver
-	defer func() {
-		netutil.Resolver = origResolver
-	}()
-
-	allowed := true
-	ui := &UserInfo{
-		URLMaps: []URLMap{
-			{
-				SrcPaths:  getRegexs([]string{"/select/.+"}),
-				URLPrefix: mustParseURL("http://srv+vmselect"),
-			},
-		},
-		DiscoverBackendIPs: &allowed,
-		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
-	}
-	if err := ui.initURLs(); err != nil {
-		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
-	}
-
-	f(ui, `/select/0/prometheus/api/v1/query?query=up`)
+	f(ui, `/test`, "http://non-exist-dns-addr/test", "", "", nil, "least_loaded", 0)
 }
 
 func TestCreateTargetURLFailure(t *testing.T) {

app/vmbackup/main_test.go

@@ -8,18 +8,15 @@ import (
 func TestHasFilepathPrefix(t *testing.T) {
 	f := func(dst, storageDataPath string, resultExpected bool) {
 		t.Helper()
-
 		result := hasFilepathPrefix(dst, storageDataPath)
 		if result != resultExpected {
-			t.Fatalf("unexpected hasFilepathPrefix(%q, %q); got: %v; want: %v", dst, storageDataPath, result, resultExpected)
+			t.Errorf("unexpected hasFilepathPrefix(%q, %q); got: %v; want: %v", dst, storageDataPath, result, resultExpected)
 		}
 	}
-
 	pwd, err := filepath.Abs("")
 	if err != nil {
 		t.Fatalf("cannot determine working directory: %s", err)
 	}
-
 	f("s3://foo/bar", "foo", false)
 	f("fs://"+pwd+"/foo", "foo", true)
 	f("fs://"+pwd+"/foo", "foo/bar", false)

app/vmctl/backoff/backoff.go

@@ -10,6 +10,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
+const (
+	backoffRetries     = 10
+	backoffFactor      = 1.8
+	backoffMinDuration = time.Second * 2
+)
+
 // retryableFunc describes call back which will repeat on errors
 type retryableFunc func() error
 
@@ -24,22 +30,12 @@ type Backoff struct {
 }
 
 // New initialize backoff object
-func New(retries int, factor float64, minDuration time.Duration) (*Backoff, error) {
-	if retries <= 0 {
-		return nil, fmt.Errorf("number of backoff retries must be greater than 0")
-	}
-	if factor <= 1 {
-		return nil, fmt.Errorf("backoff retry factor must be greater than 1")
-	}
-	if minDuration <= 0 {
-		return nil, fmt.Errorf("backoff retry minimum duration must be greater than 0")
-	}
-
+func New() *Backoff {
 	return &Backoff{
-		retries:     retries,
-		factor:      factor,
-		minDuration: minDuration,
-	}, nil
+		retries:     backoffRetries,
+		factor:      backoffFactor,
+		minDuration: backoffMinDuration,
+	}
 }
 
 // Retry process retries until all attempts are completed

app/vmctl/backoff/backoff_test.go

@@ -3,140 +3,107 @@ package backoff
 import (
 	"context"
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 )
 
-func TestBackoffRetry_Failure(t *testing.T) {
-	f := func(backoffFactor float64, backoffRetries int, cancelTimeout time.Duration, retryFunc func() error, resultExpected int) {
-		t.Helper()
-
-		r := &Backoff{
-			retries:     backoffRetries,
-			factor:      backoffFactor,
-			minDuration: time.Millisecond * 10,
-		}
-		ctx := context.Background()
-		if cancelTimeout != 0 {
-			newCtx, cancelFn := context.WithTimeout(context.Background(), cancelTimeout)
-			ctx = newCtx
-			defer cancelFn()
-		}
-
-		result, err := r.Retry(ctx, retryFunc)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-		if result != uint64(resultExpected) {
-			t.Fatalf("unexpected result: got %d; want %d", result, resultExpected)
-		}
-	}
-
-	// return bad request
-	retryFunc := func() error {
-		return ErrBadRequest
-	}
-	f(0, 0, 0, retryFunc, 0)
-
-	// empty retries values
-	retryFunc = func() error {
-		time.Sleep(time.Millisecond * 100)
-		return nil
-	}
-	f(0, 0, 0, retryFunc, 0)
-
-	// all retries failed test
-	backoffFactor := 0.1
-	backoffRetries := 5
-	cancelTimeout := time.Second * 0
-	retryFunc = func() error {
-		t := time.NewTicker(time.Millisecond * 5)
-		defer t.Stop()
-		for range t.C {
-			return fmt.Errorf("got some error")
-		}
-		return nil
-	}
-	resultExpected := 5
-	f(backoffFactor, backoffRetries, cancelTimeout, retryFunc, resultExpected)
-
-	// cancel context
-	backoffFactor = 1.7
-	backoffRetries = 5
-	cancelTimeout = time.Millisecond * 40
-	retryFunc = func() error {
-		return fmt.Errorf("got some error")
-	}
-	resultExpected = 3
-	f(backoffFactor, backoffRetries, cancelTimeout, retryFunc, resultExpected)
-}
-
-func TestBackoffRetry_Success(t *testing.T) {
-	f := func(retryFunc func() error, resultExpected int) {
-		t.Helper()
-
-		r := &Backoff{
-			retries:     5,
-			factor:      1.7,
-			minDuration: time.Millisecond * 10,
-		}
-		ctx := context.Background()
-
-		result, err := r.Retry(ctx, retryFunc)
-		if err != nil {
-			t.Fatalf("Retry() error: %s", err)
-		}
-		if result != uint64(resultExpected) {
-			t.Fatalf("unexpected result: got %d; want %d", result, resultExpected)
-		}
-	}
-
-	// only one retry test
+func TestRetry_Do(t *testing.T) {
 	counter := 0
-	retryFunc := func() error {
-		t := time.NewTicker(time.Millisecond * 5)
-		defer t.Stop()
-		for range t.C {
-			counter++
-			if counter%2 == 0 {
-				return fmt.Errorf("got some error")
-			}
-			if counter%3 == 0 {
+	tests := []struct {
+		name               string
+		backoffRetries     int
+		backoffFactor      float64
+		backoffMinDuration time.Duration
+		retryableFunc      retryableFunc
+		cancelTimeout      time.Duration
+		want               uint64
+		wantErr            bool
+	}{
+		{
+			name: "return bad request",
+			retryableFunc: func() error {
+				return ErrBadRequest
+			},
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name: "empty retries values",
+			retryableFunc: func() error {
+				time.Sleep(time.Millisecond * 100)
 				return nil
-			}
-		}
-		return nil
+			},
+			want:    0,
+			wantErr: true,
+		},
+		{
+			name:               "only one retry test",
+			backoffRetries:     5,
+			backoffFactor:      1.7,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					counter++
+					if counter%2 == 0 {
+						return fmt.Errorf("got some error")
+					}
+					if counter%3 == 0 {
+						return nil
+					}
+				}
+				return nil
+			},
+			want:    1,
+			wantErr: false,
+		},
+		{
+			name:               "all retries failed test",
+			backoffRetries:     5,
+			backoffFactor:      0.1,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				t := time.NewTicker(time.Millisecond * 5)
+				defer t.Stop()
+				for range t.C {
+					return fmt.Errorf("got some error")
+				}
+				return nil
+			},
+			want:    5,
+			wantErr: true,
+		},
+		{
+			name:               "cancel context",
+			backoffRetries:     5,
+			backoffFactor:      1.7,
+			backoffMinDuration: time.Millisecond * 10,
+			retryableFunc: func() error {
+				return fmt.Errorf("got some error")
+			},
+			cancelTimeout: time.Millisecond * 40,
+			want:          3,
+			wantErr:       true,
+		},
 	}
-	resultExpected := 1
-	f(retryFunc, resultExpected)
-}
-
-func TestBackoff_New(t *testing.T) {
-	f := func(retries int, factor float64, minDuration time.Duration, errExpected string) {
-		t.Helper()
-
-		_, err := New(retries, factor, minDuration)
-		if err == nil {
-			if errExpected != "" {
-				t.Fatalf("expecting non-nil error")
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &Backoff{retries: tt.backoffRetries, factor: tt.backoffFactor, minDuration: tt.backoffMinDuration}
+			ctx := context.Background()
+			if tt.cancelTimeout != 0 {
+				newCtx, cancelFn := context.WithTimeout(context.Background(), tt.cancelTimeout)
+				ctx = newCtx
+				defer cancelFn()
 			}
-			return
-		}
-		if !strings.Contains(err.Error(), errExpected) {
-			t.Fatalf("unexpected error: got %q; want %q", err.Error(), errExpected)
-		}
+			got, err := r.Retry(ctx, tt.retryableFunc)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Retry() got = %v, want %v", got, tt.want)
+			}
+		})
 	}
-
-	// empty retries
-	f(0, 1.1, time.Millisecond*10, "retries must be greater than 0")
-
-	// empty factor
-	f(1, 0, time.Millisecond*10, "factor must be greater than 1")
-
-	// empty minDuration
-	f(1, 1.1, 0, "minimum duration must be greater than 0")
-
-	// no errors
-	f(1, 1.1, time.Millisecond*10, "")
 }

app/vmctl/flags.go

@@ -56,10 +56,6 @@ const (
 	vmRateLimit  = "vm-rate-limit"
 
 	vmInterCluster = "vm-intercluster"
-
-	vmBackoffRetries     = "vm-backoff-retries"
-	vmBackoffFactor      = "vm-backoff-factor"
-	vmBackoffMinDuration = "vm-backoff-min-duration"
 )
 
 var (
@@ -150,21 +146,6 @@ var (
 			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
 			Value: false,
 		},
-		&cli.IntFlag{
-			Name:  vmBackoffRetries,
-			Value: 10,
-			Usage: "How many import retries to perform before giving up.",
-		},
-		&cli.Float64Flag{
-			Name:  vmBackoffFactor,
-			Value: 1.8,
-			Usage: "Factor to multiply the base duration after each failed import retry. Must be greater than 1.0",
-		},
-		&cli.DurationFlag{
-			Name:  vmBackoffMinDuration,
-			Value: time.Second * 2,
-			Usage: "Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'.",
-		},
 	}
 )
 
@@ -449,10 +430,6 @@ const (
 	vmNativeDstCAFile             = "vm-native-dst-ca-file"
 	vmNativeDstServerName         = "vm-native-dst-server-name"
 	vmNativeDstInsecureSkipVerify = "vm-native-dst-insecure-skip-verify"
-
-	vmNativeBackoffRetries     = "vm-native-backoff-retries"
-	vmNativeBackoffFactor      = "vm-native-backoff-factor"
-	vmNativeBackoffMinDuration = "vm-native-backoff-min-duration"
 )
 
 var (
@@ -622,21 +599,6 @@ var (
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},
-		&cli.IntFlag{
-			Name:  vmNativeBackoffRetries,
-			Value: 10,
-			Usage: "How many export/import retries to perform before giving up.",
-		},
-		&cli.Float64Flag{
-			Name:  vmNativeBackoffFactor,
-			Value: 1.8,
-			Usage: "Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0",
-		},
-		&cli.DurationFlag{
-			Name:  vmNativeBackoffMinDuration,
-			Value: time.Second * 2,
-			Usage: "Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'.",
-		},
 	}
 )
 

app/vmctl/influx/influx_test.go

@@ -3,97 +3,125 @@ package influx
 import "testing"
 
 func TestFetchQuery(t *testing.T) {
-	f := func(s *Series, timeFilter, resultExpected string) {
-		t.Helper()
-
-		result := s.fetchQuery(timeFilter)
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
-		}
+	testCases := []struct {
+		s          Series
+		timeFilter string
+		expected   string
+	}{
+		{
+			s: Series{
+				Measurement: "cpu",
+				Field:       "value",
+				LabelPairs: []LabelPair{
+					{
+						Name:  "foo",
+						Value: "bar",
+					},
+				},
+			},
+			expected: `select "value" from "cpu" where "foo"::tag='bar'`,
+		},
+		{
+			s: Series{
+				Measurement: "cpu",
+				Field:       "value",
+				LabelPairs: []LabelPair{
+					{
+						Name:  "foo",
+						Value: "bar",
+					},
+					{
+						Name:  "baz",
+						Value: "qux",
+					},
+				},
+			},
+			expected: `select "value" from "cpu" where "foo"::tag='bar' and "baz"::tag='qux'`,
+		},
+		{
+			s: Series{
+				Measurement: "cpu",
+				Field:       "value",
+				LabelPairs: []LabelPair{
+					{
+						Name:  "foo",
+						Value: "b'ar",
+					},
+				},
+			},
+			timeFilter: "time >= now()",
+			expected:   `select "value" from "cpu" where "foo"::tag='b\'ar' and time >= now()`,
+		},
+		{
+			s: Series{
+				Measurement: "cpu",
+				Field:       "value",
+				LabelPairs: []LabelPair{
+					{
+						Name:  "name",
+						Value: `dev-mapper-centos\x2dswap.swap`,
+					},
+					{
+						Name:  "state",
+						Value: "dev-mapp'er-c'en'tos",
+					},
+				},
+			},
+			timeFilter: "time >= now()",
+			expected:   `select "value" from "cpu" where "name"::tag='dev-mapper-centos\\x2dswap.swap' and "state"::tag='dev-mapp\'er-c\'en\'tos' and time >= now()`,
+		},
+		{
+			s: Series{
+				Measurement: "cpu",
+				Field:       "value",
+			},
+			timeFilter: "time >= now()",
+			expected:   `select "value" from "cpu" where time >= now()`,
+		},
+		{
+			s: Series{
+				Measurement: "cpu",
+				Field:       "value",
+			},
+			expected: `select "value" from "cpu"`,
+		},
 	}
 
-	f(&Series{
-		Measurement: "cpu",
-		Field:       "value",
-		LabelPairs: []LabelPair{
-			{
-				Name:  "foo",
-				Value: "bar",
-			},
-		},
-	}, "", `select "value" from "cpu" where "foo"::tag='bar'`)
-
-	f(&Series{
-		Measurement: "cpu",
-		Field:       "value",
-		LabelPairs: []LabelPair{
-			{
-				Name:  "foo",
-				Value: "bar",
-			},
-			{
-				Name:  "baz",
-				Value: "qux",
-			},
-		},
-	}, "", `select "value" from "cpu" where "foo"::tag='bar' and "baz"::tag='qux'`)
-
-	f(&Series{
-		Measurement: "cpu",
-		Field:       "value",
-		LabelPairs: []LabelPair{
-			{
-				Name:  "foo",
-				Value: "b'ar",
-			},
-		},
-	}, "time >= now()", `select "value" from "cpu" where "foo"::tag='b\'ar' and time >= now()`)
-
-	f(&Series{
-		Measurement: "cpu",
-		Field:       "value",
-		LabelPairs: []LabelPair{
-			{
-				Name:  "name",
-				Value: `dev-mapper-centos\x2dswap.swap`,
-			},
-			{
-				Name:  "state",
-				Value: "dev-mapp'er-c'en'tos",
-			},
-		},
-	}, "time >= now()", `select "value" from "cpu" where "name"::tag='dev-mapper-centos\\x2dswap.swap' and "state"::tag='dev-mapp\'er-c\'en\'tos' and time >= now()`)
-
-	f(&Series{
-		Measurement: "cpu",
-		Field:       "value",
-	}, "time >= now()", `select "value" from "cpu" where time >= now()`)
-
-	f(&Series{
-		Measurement: "cpu",
-		Field:       "value",
-	}, "", `select "value" from "cpu"`)
+	for _, tc := range testCases {
+		query := tc.s.fetchQuery(tc.timeFilter)
+		if query != tc.expected {
+			t.Fatalf("got: \n%s;\nexpected: \n%s", query, tc.expected)
+		}
+	}
 }
 
 func TestTimeFilter(t *testing.T) {
-	f := func(start, end, resultExpected string) {
-		t.Helper()
-
-		result := timeFilter(start, end)
-		if result != resultExpected {
-			t.Fatalf("unexpected result\ngot\n%v\nwant\n%s", result, resultExpected)
+	testCases := []struct {
+		start    string
+		end      string
+		expected string
+	}{
+		{
+			start:    "2020-01-01T20:07:00Z",
+			end:      "2020-01-01T21:07:00Z",
+			expected: "time >= '2020-01-01T20:07:00Z' and time <= '2020-01-01T21:07:00Z'",
+		},
+		{
+			expected: "",
+		},
+		{
+			start:    "2020-01-01T20:07:00Z",
+			expected: "time >= '2020-01-01T20:07:00Z'",
+		},
+		{
+			end:      "2020-01-01T21:07:00Z",
+			expected: "time <= '2020-01-01T21:07:00Z'",
+		},
+	}
+	for _, tc := range testCases {
+		f := timeFilter(tc.start, tc.end)
+		if f != tc.expected {
+			t.Fatalf("got: \n%q;\nexpected: \n%q", f, tc.expected)
 		}
 	}
-
-	// no start and end filters
-	f("", "", "")
-
-	// missing end filter
-	f("2020-01-01T20:07:00Z", "", "time >= '2020-01-01T20:07:00Z'")
-
-	// missing start filter
-	f("", "2020-01-01T21:07:00Z", "time <= '2020-01-01T21:07:00Z'")
-
-	// both start and end filters
-	f("2020-01-01T20:07:00Z", "2020-01-01T21:07:00Z", "time >= '2020-01-01T20:07:00Z' and time <= '2020-01-01T21:07:00Z'")
 }

app/vmctl/influx/parser.go

@@ -12,7 +12,7 @@ import (
 
 type queryValues struct {
 	name   string
-	values map[string][]any
+	values map[string][]interface{}
 }
 
 func parseResult(r influx.Result) ([]queryValues, error) {
@@ -21,7 +21,7 @@ func parseResult(r influx.Result) ([]queryValues, error) {
 	}
 	qValues := make([]queryValues, len(r.Series))
 	for i, row := range r.Series {
-		values := make(map[string][]any, len(row.Values))
+		values := make(map[string][]interface{}, len(row.Values))
 		for _, value := range row.Values {
 			for idx, v := range value {
 				key := row.Columns[idx]
@@ -36,7 +36,7 @@ func parseResult(r influx.Result) ([]queryValues, error) {
 	return qValues, nil
 }
 
-func toFloat64(v any) (float64, error) {
+func toFloat64(v interface{}) (float64, error) {
 	switch i := v.(type) {
 	case json.Number:
 		return i.Float64()

app/vmctl/influx/parser_test.go

@@ -6,71 +6,71 @@ import (
 	"testing"
 )
 
-func TestSeriesUnmarshal(t *testing.T) {
-	f := func(s string, resultExpected *Series) {
-		t.Helper()
-
-		result := &Series{}
-		if err := result.unmarshal(s); err != nil {
-			t.Fatalf("cannot unmarshal series from %q: %s", s, err)
-		}
-		if !reflect.DeepEqual(result, resultExpected) {
-			t.Fatalf("unexpected result\ngot\n%v\nwant\n%v", result, resultExpected)
-		}
-	}
-
+func TestSeries_Unmarshal(t *testing.T) {
 	tag := func(name, value string) LabelPair {
 		return LabelPair{
 			Name:  name,
 			Value: value,
 		}
 	}
-	series := func(measurement string, lp ...LabelPair) *Series {
-		return &Series{
+	series := func(measurement string, lp ...LabelPair) Series {
+		return Series{
 			Measurement: measurement,
 			LabelPairs:  lp,
 		}
 	}
-
-	f("cpu", series("cpu"))
-
-	f("cpu,host=localhost", series("cpu", tag("host", "localhost")))
-
-	f("cpu,host=localhost,instance=instance", series("cpu", tag("host", "localhost"), tag("instance", "instance")))
-
-	f(`fo\,bar\=baz,x\=\b=\\a\,\=\q\ `, series("fo,bar=baz", tag(`x=\b`, `\a,=\q `)))
-
-	f("cpu,host=192.168.0.1,instance=fe80::fdc8:5e36:c2c6:baac%utun1", series("cpu", tag("host", "192.168.0.1"), tag("instance", "fe80::fdc8:5e36:c2c6:baac%utun1")))
-
-	f(`cpu,db=db1,host=localhost,server=host\=localhost\ user\=user\ `, series("cpu", tag("db", "db1"), tag("host", "localhost"), tag("server", "host=localhost user=user ")))
-}
-
-func TestToFloat64_Failure(t *testing.T) {
-	f := func(in any) {
-		t.Helper()
-
-		_, err := toFloat64(in)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
+	testCases := []struct {
+		got  string
+		want Series
+	}{
+		{
+			got:  "cpu",
+			want: series("cpu"),
+		},
+		{
+			got:  "cpu,host=localhost",
+			want: series("cpu", tag("host", "localhost")),
+		},
+		{
+			got:  "cpu,host=localhost,instance=instance",
+			want: series("cpu", tag("host", "localhost"), tag("instance", "instance")),
+		},
+		{
+			got:  `fo\,bar\=baz,x\=\b=\\a\,\=\q\ `,
+			want: series("fo,bar=baz", tag(`x=\b`, `\a,=\q `)),
+		},
+		{
+			got:  "cpu,host=192.168.0.1,instance=fe80::fdc8:5e36:c2c6:baac%utun1",
+			want: series("cpu", tag("host", "192.168.0.1"), tag("instance", "fe80::fdc8:5e36:c2c6:baac%utun1")),
+		},
+		{
+			got: `cpu,db=db1,host=localhost,server=host\=localhost\ user\=user\ `,
+			want: series("cpu", tag("db", "db1"),
+				tag("host", "localhost"), tag("server", "host=localhost user=user ")),
+		},
+	}
+	for _, tc := range testCases {
+		s := Series{}
+		if err := s.unmarshal(tc.got); err != nil {
+			t.Fatalf("%q: unmarshal err: %s", tc.got, err)
+		}
+		if !reflect.DeepEqual(s, tc.want) {
+			t.Fatalf("%q: expected\n%#v\nto be equal\n%#v", tc.got, s, tc.want)
 		}
 	}
-
-	f("text")
 }
 
-func TestToFloat64_Success(t *testing.T) {
-	f := func(in any, resultExpected float64) {
+func TestToFloat64(t *testing.T) {
+	f := func(in interface{}, want float64) {
 		t.Helper()
-
-		result, err := toFloat64(in)
+		got, err := toFloat64(in)
 		if err != nil {
-			t.Fatalf("unexpected error: %s", err)
+			t.Fatalf("unexpected err: %s", err)
 		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result: got %v; want %v", result, resultExpected)
+		if got != want {
+			t.Errorf("got %v; want %v", got, want)
 		}
 	}
-
 	f("123.4", 123.4)
 	f(float64(123.4), 123.4)
 	f(float32(12), 12)
@@ -78,4 +78,9 @@ func TestToFloat64_Success(t *testing.T) {
 	f(true, 1)
 	f(false, 0)
 	f(json.Number("123456.789"), 123456.789)
+
+	_, err := toFloat64("text")
+	if err == nil {
+		t.Fatalf("expected to get err; got nil instead")
+	}
 }

app/vmctl/main.go

@@ -68,7 +68,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
+						return fmt.Errorf("failed to create Transport: %s", err)
 					}
 					oCfg := opentsdb.Config{
 						Addr:       addr,
@@ -90,7 +90,6 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
-
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -144,7 +143,6 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
-
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -180,7 +178,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
+						return fmt.Errorf("failed to create transport: %s", err)
 					}
 
 					rr, err := remoteread.NewClient(remoteread.Config{
@@ -203,7 +201,6 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
-
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -236,7 +233,6 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
-
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -276,14 +272,6 @@ func main() {
 						return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
 					}
 
-					bfRetries := c.Int(vmNativeBackoffRetries)
-					bfFactor := c.Float64(vmNativeBackoffFactor)
-					bfMinDuration := c.Duration(vmNativeBackoffMinDuration)
-					bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
-					if err != nil {
-						return fmt.Errorf("failed to create backoff object: %s", err)
-					}
-
 					disableKeepAlive := c.Bool(vmNativeDisableHTTPKeepAlive)
 
 					var srcExtraLabels []string
@@ -362,7 +350,7 @@ func main() {
 							ExtraLabels: dstExtraLabels,
 							HTTPClient:  dstHTTPClient,
 						},
-						backoff:                  bf,
+						backoff:                  backoff.New(),
 						cc:                       c.Int(vmConcurrency),
 						disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
 						isNative:                 !c.Bool(vmNativeDisableBinaryProtocol),
@@ -438,15 +426,7 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 
 	tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 	if err != nil {
-		return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
-	}
-
-	bfRetries := c.Int(vmBackoffRetries)
-	bfFactor := c.Float64(vmBackoffFactor)
-	bfMinDuration := c.Duration(vmBackoffMinDuration)
-	bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
-	if err != nil {
-		return vm.Config{}, fmt.Errorf("failed to create backoff object: %s", err)
+		return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
 	}
 
 	return vm.Config{
@@ -462,6 +442,5 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
 		RateLimit:          c.Int64(vmRateLimit),
-		Backoff:            bf,
 	}, nil
 }