debug: [vmagent] add debug log for max_scrape_size

Signed-off-by: hagen1778 <roman@victoriametrics.com>

.github/workflows/benchmark.yml

@@ -1,56 +0,0 @@
-name: benchmark
-on:
-  push:
-    branches:
-      - master
-      - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - labeled
-    branches:
-      - master
-      - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-      - "dashboards/**"
-      - "deployment/**.yml"
-permissions:
-  contents: read
-  packages: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-streamaggr-benchmark-image:
-    name: build
-    runs-on: ubuntu-latest
-    if: contains(github.event.pull_request.labels.*.name, 'streamaggr-benchmark')
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Package VMAgent Docker image for benchmark
-        run: |
-          SKIP_SCRATCH_BUILD=true \
-          DOCKER_BUILD_OPTS='--cache-to type=gha,mode=max --cache-from type=gha' \
-          PKG_TAG=${{ github.event.pull_request.head.sha }} \
-          DOCKER_REGISTRY=ghcr.io \
-          TARGET_PLATFORM=linux/amd64 make publish-vmagent

.gitignore

@@ -7,6 +7,7 @@
 .vscode
 *.test
 *.swp
+/vmdocs
 /gocache-for-docker
 /victoria-logs-data
 /victoria-metrics-data
@@ -23,4 +24,5 @@ Gemfile.lock
 _site
 *.tmp
 /docs/.jekyll-metadata
-coverage.txt
+coverage.txt
+cspell.json

CODE_OF_CONDUCT_RU.md

@@ -1,120 +0,0 @@
-
-# Кодекс Поведения участника
-
-## Наши обязательства
-
-Мы, как участники, авторы и лидеры обязуемся сделать участие в сообществе
-свободным от притеснений для всех, независимо от возраста, телосложения,
-видимых или невидимых ограничений способности, этнической принадлежности,
-половых признаков, гендерной идентичности и выражения, уровня опыта,
-образования, социо-экономического статуса, национальности, внешности,
-расы, религии, или сексуальной идентичности и ориентации.
-
-Мы обещаем действовать и взаимодействовать таким образом, чтобы вносить вклад в открытое,
-дружелюбное, многообразное, инклюзивное и здоровое сообщество.
-
-## Наши стандарты
-
-Примеры поведения, создающие условия для благоприятных взаимоотношений включают в себя:
-
-* Проявление доброты и эмпатии к другим участникам проекта
-* Уважение к чужой точке зрения и опыту
-* Конструктивная критика и принятие конструктивной критики
-* Принятие ответственности, принесение извинений тем, кто пострадал от наших ошибок
-  и извлечение уроков из опыта
-* Ориентирование на то, что лучше подходит для сообщества, а не только для нас лично
-
-Примеры неприемлемого поведения участников включают в себя:
-
-* Использование выражений или изображений сексуального характера и нежелательное сексуальное внимание или домогательство в любой форме
-* Троллинг, оскорбительные или уничижительные комментарии, переход на личности или затрагивание политических убеждений
-* Публичное или приватное домогательство
-* Публикация личной информации других лиц, например, физического или электронного адреса, без явного разрешения
-* Иное поведение, которое обоснованно считать неуместным в профессиональной обстановке
-
-## Обязанности
-
-Лидеры сообщества отвечают за разъяснение и применение наших стандартов приемлемого
-поведения и будут предпринимать соответствующие и честные меры по исправлению положения
-в ответ на любое поведение, которое они сочтут неприемлемым, угрожающим, оскорбительным или вредным.
-
-Лидеры сообщества обладают правом и обязанностью удалять, редактировать или отклонять
-комментарии, коммиты, код, изменения в вики, вопросы и другой вклад, который не совпадает
-с Кодексом Поведения, и предоставят причины принятого решения, когда сочтут нужным.
-
-## Область применения
-
-Данный Кодекс Поведения применим во всех во всех публичных физических и цифровых пространства сообщества,
-а также когда человек официально представляет сообщество в публичных местах.
-Примеры представления проекта или сообщества включают использование официальной электронной почты,
-публикации в официальном аккаунте в социальных сетях,
-или упоминания как представителя в онлайн или оффлайн мероприятии.
-
-## Приведение в исполнение
-
-О случаях домогательства, а так же оскорбительного или иного другого неприемлемого
-поведения можно сообщить ответственным лидерам сообщества с помощью письма на info@victoriametrics.com
-Все жалобы будут рассмотрены и расследованы оперативно и беспристрастно.
-
-Все лидеры сообщества обязаны уважать неприкосновенность частной жизни и личную
-неприкосновенность автора сообщения.
-
-## Руководство по исполнению
-
-Лидеры сообщества будут следовать следующим Принципам Воздействия в Сообществе,
-чтобы определить последствия для тех, кого они считают виновными в нарушении данного Кодекса Поведения:
-
-### 1. Исправление
-
-**Общественное влияние**: Использование недопустимой лексики или другое поведение,
-считающиеся непрофессиональным или нежелательным в сообществе.
-
-**Последствия**: Личное, письменное предупреждение от лидеров сообщества,
-объясняющее суть нарушения и почему такое поведение
-было неуместно. Лидеры сообщества могут попросить принести публичное извинение.
-
-### 2. Предупреждение
-
-**Общественное влияние**: Нарушение в результате одного инцидента или серии действий.
-
-**Последствия**: Предупреждение о последствиях в случае продолжающегося неуместного поведения.
-На определенное время не допускается взаимодействие с людьми, вовлеченными в инцидент,
-включая незапрошенное взаимодействие
-с теми, кто обеспечивает соблюдение Кодекса. Это включает в себя избегание взаимодействия
-в публичных пространствах, а так же во внешних каналах,
-таких как социальные сети. Нарушение этих правил влечет за собой временный или вечный бан.
-
-### 3. Временный бан
-
-**Общественное влияние**: Серьёзное нарушение стандартов сообщества,
-включая продолжительное неуместное поведение.
-
-**Последствия**: Временный запрет (бан) на любое взаимодействие
-или публичное общение с сообществом на определенный период времени.
-На этот период не допускается публичное или личное взаимодействие с людьми,
-вовлеченными в инцидент, включая незапрошенное взаимодействие
-с теми, кто обеспечивает соблюдение Кодекса.
-Нарушение этих правил влечет за собой вечный бан.
-
-### 4. Вечный бан
-
-**Общественное влияние**: Демонстрация систематических нарушений стандартов сообщества,
-включая продолжающееся неуместное поведение, домогательство до отдельных лиц,
-или проявление агрессии либо пренебрежительного отношения к категориям лиц.
-
-**Последствия**: Вечный запрет на любое публичное взаимодействие с сообществом.
-
-## Атрибуция
-
-Данный Кодекс Поведения основан на [Кодекс Поведения участника][homepage],
-версии 2.0, доступной по адресу
-<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
-
-Принципы Воздействия в Сообществе были вдохновлены [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
-
-[homepage]: https://www.contributor-covenant.org
-
-Ответы на общие вопросы о данном кодексе поведения ищите на странице FAQ:
-<https://www.contributor-covenant.org/faq>. Переводы доступны по адресу
-<https://www.contributor-covenant.org/translations>.

Makefile

@@ -13,10 +13,12 @@ PKG_TAG := $(BUILDINFO_TAG)
 endif
 
 GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TAG)-$(BUILDINFO_TAG)'
+TAR_OWNERSHIP ?= --owner=1000 --group=1000
 
 .PHONY: $(MAKECMDGOALS)
 
 include app/*/Makefile
+include cspell/Makefile
 include docs/Makefile
 include deployment/*/Makefile
 include dashboards/Makefile
@@ -244,7 +246,7 @@ release-victoria-metrics-windows-amd64:
 
 release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-metrics-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-metrics-$(GOOS)-$(GOARCH)-prod \
@@ -301,7 +303,7 @@ release-victoria-logs-windows-amd64:
 
 release-victoria-logs-goos-goarch: victoria-logs-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
 		&& sha256sum victoria-logs-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			victoria-logs-$(GOOS)-$(GOARCH)-prod \
@@ -365,7 +367,7 @@ release-vmutils-goos-goarch: \
 	vmrestore-$(GOOS)-$(GOARCH)-prod \
 	vmctl-$(GOOS)-$(GOARCH)-prod
 	cd bin && \
-		tar --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
+		tar $(TAR_OWNERSHIP) --transform="flags=r;s|-$(GOOS)-$(GOARCH)||" -czf vmutils-$(GOOS)-$(GOARCH)-$(PKG_TAG).tar.gz \
 			vmagent-$(GOOS)-$(GOARCH)-prod \
 			vmalert-$(GOOS)-$(GOARCH)-prod \
 			vmalert-tool-$(GOOS)-$(GOARCH)-prod \
@@ -493,7 +495,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.59.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.1
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`
@@ -512,33 +514,3 @@ install-wwhrd:
 
 check-licenses: install-wwhrd
 	wwhrd check -f .wwhrd.yml
-
-copy-docs:
-# The 'printf' function is used instead of 'echo' or 'echo -e' to handle line breaks (e.g. '\n') in the same way on different operating systems (MacOS/Ubuntu Linux/Arch Linux) and their shells (bash/sh/zsh/fish).
-# For details, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4548#issue-1782796419 and https://stackoverflow.com/questions/8467424/echo-newline-in-bash-prints-literal-n
-	echo "---" > ${DST}
-	@if [ ${ORDER} -ne 0 ]; then \
-		echo "sort: ${ORDER}" >> ${DST}; \
-		echo "weight: ${ORDER}" >> ${DST}; \
-		printf "menu:\n  docs:\n    parent: 'victoriametrics'\n    weight: ${ORDER}\n" >> ${DST}; \
-	fi
-
-	echo "title: ${TITLE}" >> ${DST}
-	@if [ ${OLD_URL} ]; then \
-		printf "aliases:\n  - ${OLD_URL}\n" >> ${DST}; \
-	fi
-	echo "---" >> ${DST}
-	cat ${SRC} >> ${DST}
-	sed -i='.tmp' 's/<img src=\"docs\//<img src=\"/' ${DST}
-	rm -rf docs/*.tmp
-
-# Copies docs for all components and adds the order/weight tag, title, menu position and alias with the backward compatible link for the old site.
-# For ORDER=0 it adds no order tag/weight tag.
-# FOR OLD_URL - relative link, used for backward compatibility with the link from documentation based on GitHub pages (old one)
-# FOR OLD_URL='' it adds no alias, it should be empty for every new page, don't change it for already existing links.
-# Images starting with <img src="docs/ are replaced with <img src="
-# Cluster docs are supposed to be ordered as 2nd.
-# The rest of docs is ordered manually.
-docs-sync:
-	SRC=README.md DST=docs/README.md OLD_URL='' ORDER=0 TITLE=VictoriaMetrics $(MAKE) copy-docs
-	SRC=README.md DST=docs/Single-server-VictoriaMetrics.md OLD_URL='/Single-server-VictoriaMetrics.html' TITLE=VictoriaMetrics ORDER=1 $(MAKE) copy-docs

SECURITY.md

@@ -7,12 +7,12 @@ The following versions of VictoriaMetrics receive regular security fixes:
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
+| v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.93.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.
 
 ## Reporting a Vulnerability
 
-Please report any security issues to security@victoriametrics.com
+Please report any security issues to <security@victoriametrics.com>

app/victoria-logs/Makefile

@@ -81,6 +81,9 @@ victoria-logs-linux-ppc64le:
 victoria-logs-linux-s390x:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
+victoria-logs-linux-loong64:
+	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+
 victoria-logs-linux-386:
 	APP_NAME=victoria-logs CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/victoria-metrics/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -117,7 +118,7 @@ func (q *Query) metrics() ([]Metric, error) {
 type QueryInstant struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TV     [2]interface{}    `json:"value"`
+		TV     [2]any            `json:"value"`
 	} `json:"result"`
 }
 
@@ -140,7 +141,7 @@ func (q QueryInstant) metrics() ([]Metric, error) {
 type QueryRange struct {
 	Result []struct {
 		Metric map[string]string `json:"metric"`
-		Values [][]interface{}   `json:"values"`
+		Values [][]any           `json:"values"`
 	} `json:"result"`
 }
 
@@ -249,24 +250,24 @@ func TestWriteRead(t *testing.T) {
 
 func testWrite(t *testing.T) {
 	t.Run("prometheus", func(t *testing.T) {
-		for _, test := range readIn("prometheus", t, insertionTime) {
+		for _, test := range readIn("prometheus", insertionTime) {
 			if test.Data == nil {
 				continue
 			}
-			s := newSuite(t)
 			r := testutil.WriteRequest{}
-			s.noError(json.Unmarshal([]byte(strings.Join(test.Data, "\n")), &r.Timeseries))
-			data, err := testutil.Compress(r)
-			s.greaterThan(len(r.Timeseries), 0)
-			if err != nil {
-				t.Errorf("error compressing %v %s", r, err)
-				t.Fail()
+			testData := strings.Join(test.Data, "\n")
+			if err := json.Unmarshal([]byte(testData), &r.Timeseries); err != nil {
+				panic(fmt.Errorf("BUG: cannot unmarshal TimeSeries: %s\ntest data\n%s", err, testData))
 			}
+			if n := len(r.Timeseries); n <= 0 {
+				panic(fmt.Errorf("BUG: expecting non-empty Timeseries in test data:\n%s", testData))
+			}
+			data := testutil.Compress(r)
 			httpWrite(t, testPromWriteHTTPPath, test.InsertQuery, bytes.NewBuffer(data))
 		}
 	})
 	t.Run("csv", func(t *testing.T) {
-		for _, test := range readIn("csv", t, insertionTime) {
+		for _, test := range readIn("csv", insertionTime) {
 			if test.Data == nil {
 				continue
 			}
@@ -275,7 +276,7 @@ func testWrite(t *testing.T) {
 	})
 
 	t.Run("influxdb", func(t *testing.T) {
-		for _, x := range readIn("influxdb", t, insertionTime) {
+		for _, x := range readIn("influxdb", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -284,7 +285,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("graphite", func(t *testing.T) {
-		for _, x := range readIn("graphite", t, insertionTime) {
+		for _, x := range readIn("graphite", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -293,7 +294,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdb", func(t *testing.T) {
-		for _, x := range readIn("opentsdb", t, insertionTime) {
+		for _, x := range readIn("opentsdb", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -302,7 +303,7 @@ func testWrite(t *testing.T) {
 		}
 	})
 	t.Run("opentsdbhttp", func(t *testing.T) {
-		for _, x := range readIn("opentsdbhttp", t, insertionTime) {
+		for _, x := range readIn("opentsdbhttp", insertionTime) {
 			test := x
 			t.Run(test.Name, func(t *testing.T) {
 				t.Parallel()
@@ -316,7 +317,7 @@ func testWrite(t *testing.T) {
 func testRead(t *testing.T) {
 	for _, engine := range []string{"csv", "prometheus", "graphite", "opentsdb", "influxdb", "opentsdbhttp"} {
 		t.Run(engine, func(t *testing.T) {
-			for _, x := range readIn(engine, t, insertionTime) {
+			for _, x := range readIn(engine, insertionTime) {
 				test := x
 				t.Run(test.Name, func(t *testing.T) {
 					t.Parallel()
@@ -365,11 +366,10 @@ func testRead(t *testing.T) {
 	}
 }
 
-func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
-	t.Helper()
-	s := newSuite(t)
+func readIn(readFor string, insertTime time.Time) []test {
+	testDir := filepath.Join(testFixturesDir, readFor)
 	var tt []test
-	s.noError(filepath.Walk(filepath.Join(testFixturesDir, readFor), func(path string, _ os.FileInfo, err error) error {
+	err := filepath.Walk(testDir, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -377,84 +377,129 @@ func readIn(readFor string, t *testing.T, insertTime time.Time) []test {
 			return nil
 		}
 		b, err := os.ReadFile(path)
-		s.noError(err)
+		if err != nil {
+			panic(fmt.Errorf("BUG: cannot read %s: %s", path, err))
+		}
 		item := test{}
-		s.noError(json.Unmarshal(b, &item))
+		if err := json.Unmarshal(b, &item); err != nil {
+			panic(fmt.Errorf("cannot parse %T from %s: %s; data:\n%s", &item, path, err, b))
+		}
 		for i := range item.Data {
 			item.Data[i] = testutil.PopulateTimeTplString(item.Data[i], insertTime)
 		}
 		tt = append(tt, item)
 		return nil
-	}))
+	})
+	if err != nil {
+		panic(fmt.Errorf("BUG: cannot read test data at %s: %w", testDir, err))
+	}
 	if len(tt) == 0 {
-		t.Fatalf("no test found in %s", filepath.Join(testFixturesDir, readFor))
+		panic(fmt.Errorf("BUG: no tests found in %s", testDir))
 	}
 	return tt
 }
 
 func httpWrite(t *testing.T, address, query string, r io.Reader) {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Post(address+query, "", r)
-	s.noError(err)
-	s.noError(resp.Body.Close())
-	s.equalInt(resp.StatusCode, 204)
+
+	requestURL := address + query
+	resp, err := http.Post(requestURL, "", r)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != 204 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 204", requestURL, resp.StatusCode)
+	}
 }
 
-func tcpWrite(t *testing.T, address string, data string) {
+func tcpWrite(t *testing.T, address, data string) {
 	t.Helper()
-	s := newSuite(t)
+
 	conn, err := net.Dial("tcp", address)
-	s.noError(err)
+	if err != nil {
+		t.Fatalf("cannot dial %s: %s", address, err)
+	}
 	defer func() {
 		_ = conn.Close()
 	}()
 	n, err := conn.Write([]byte(data))
-	s.noError(err)
-	s.equalInt(n, len(data))
+	if err != nil {
+		t.Fatalf("cannot write %d bytes to %s: %s", len(data), address, err)
+	}
+	if n != len(data) {
+		panic(fmt.Errorf("BUG: conn.Write() returned unexpected number of written bytes to %s; got %d; want %d", address, n, len(data)))
+	}
 }
 
 func httpReadMetrics(t *testing.T, address, query string) []Metric {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+
 	var rows []Metric
-	for dec := json.NewDecoder(resp.Body); dec.More(); {
+	dec := json.NewDecoder(resp.Body)
+	for {
 		var row Metric
-		s.noError(dec.Decode(&row))
+		err := dec.Decode(&row)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				return rows
+			}
+			t.Fatalf("cannot decode %T from response received from %s: %s", &row, requestURL, err)
+		}
 		rows = append(rows, row)
 	}
-	return rows
 }
 
-func httpReadStruct(t *testing.T, address, query string, dst interface{}) {
+func httpReadStruct(t *testing.T, address, query string, dst any) {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
-	s.noError(json.NewDecoder(resp.Body).Decode(dst))
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
+	err = json.NewDecoder(resp.Body).Decode(dst)
+	if err != nil {
+		t.Fatalf("cannot decode %T from response received from %s: %s", dst, requestURL, err)
+	}
 }
 
 func httpReadData(t *testing.T, address, query string) []byte {
 	t.Helper()
-	s := newSuite(t)
-	resp, err := http.Get(address + query)
-	s.noError(err)
+
+	requestURL := address + query
+	resp, err := http.Get(requestURL)
+	if err != nil {
+		t.Fatalf("cannot send request to %s: %s", requestURL, err)
+	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-	s.equalInt(resp.StatusCode, 200)
+	if resp.StatusCode != 200 {
+		t.Fatalf("unexpected status code received from %s; got %d; want 200", requestURL, resp.StatusCode)
+	}
 	data, err := io.ReadAll(resp.Body)
-	s.noError(err)
+	if err != nil {
+		t.Fatalf("cannot read response from %s: %s", requestURL, err)
+	}
 	return data
 }
 
@@ -503,34 +548,6 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma
 	return contains
 }
 
-type suite struct{ t *testing.T }
-
-func newSuite(t *testing.T) *suite { return &suite{t: t} }
-
-func (s *suite) noError(err error) {
-	s.t.Helper()
-	if err != nil {
-		s.t.Errorf("unexpected error %v", err)
-		s.t.FailNow()
-	}
-}
-
-func (s *suite) equalInt(a, b int) {
-	s.t.Helper()
-	if a != b {
-		s.t.Errorf("%d not equal %d", a, b)
-		s.t.FailNow()
-	}
-}
-
-func (s *suite) greaterThan(a, b int) {
-	s.t.Helper()
-	if a <= b {
-		s.t.Errorf("%d less or equal then %d", a, b)
-		s.t.FailNow()
-	}
-}
-
 func TestImportJSONLines(t *testing.T) {
 	f := func(labelsCount, labelLen int) {
 		t.Helper()

app/victoria-metrics/test/prom_writer.go

@@ -1,12 +1,16 @@
 package test
 
-import "github.com/golang/snappy"
+import (
+	"fmt"
+
+	"github.com/golang/snappy"
+)
 
 // Compress marshals and compresses wr.
-func Compress(wr WriteRequest) ([]byte, error) {
+func Compress(wr WriteRequest) []byte {
 	data, err := wr.Marshal()
 	if err != nil {
-		return nil, err
+		panic(fmt.Errorf("BUG: cannot compress WriteRequest: %s", err))
 	}
-	return snappy.Encode(nil, data), nil
+	return snappy.Encode(nil, data)
 }

app/vlinsert/elasticsearch/elasticsearch.go

@@ -57,6 +57,12 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, `{}`)
 		return true
 	}
+	if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
+		// Return fake response for Logstash APIs requests.
+		// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
+		fmt.Fprintf(w, `{}`)
+		return true
+	}
 	switch path {
 	case "/":
 		switch r.Method {

app/vlinsert/loki/loki_protobuf.go

@@ -79,12 +79,14 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 	req := getPushRequest()
 	defer putPushRequest(req)
 
-	err = req.Unmarshal(bb.B)
+	err = req.UnmarshalProtobuf(bb.B)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse request body: %w", err)
 	}
 
-	var commonFields []logstorage.Field
+	fields := getFields()
+	defer putFields(fields)
+
 	rowsIngested := 0
 	streams := req.Streams
 	currentTimestamp := time.Now().UnixNano()
@@ -92,30 +94,60 @@ func parseProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor) (int
 		stream := &streams[i]
 		// st.Labels contains labels for the stream.
 		// Labels are same for all entries in the stream.
-		commonFields, err = parsePromLabels(commonFields[:0], stream.Labels)
+		fields.fields, err = parsePromLabels(fields.fields[:0], stream.Labels)
 		if err != nil {
 			return rowsIngested, fmt.Errorf("cannot parse stream labels %q: %w", stream.Labels, err)
 		}
-		fields := commonFields
+		commonFieldsLen := len(fields.fields)
 
 		entries := stream.Entries
 		for j := range entries {
-			entry := &entries[j]
-			fields = append(fields[:len(commonFields)], logstorage.Field{
+			e := &entries[j]
+			fields.fields = fields.fields[:commonFieldsLen]
+
+			for _, lp := range e.StructuredMetadata {
+				fields.fields = append(fields.fields, logstorage.Field{
+					Name:  lp.Name,
+					Value: lp.Value,
+				})
+			}
+
+			fields.fields = append(fields.fields, logstorage.Field{
 				Name:  "_msg",
-				Value: entry.Line,
+				Value: e.Line,
 			})
-			ts := entry.Timestamp.UnixNano()
+
+			ts := e.Timestamp.UnixNano()
 			if ts == 0 {
 				ts = currentTimestamp
 			}
-			lmp.AddRow(ts, fields)
+
+			lmp.AddRow(ts, fields.fields)
 		}
 		rowsIngested += len(stream.Entries)
 	}
 	return rowsIngested, nil
 }
 
+func getFields() *fields {
+	v := fieldsPool.Get()
+	if v == nil {
+		return &fields{}
+	}
+	return v.(*fields)
+}
+
+func putFields(f *fields) {
+	f.fields = f.fields[:0]
+	fieldsPool.Put(f)
+}
+
+var fieldsPool sync.Pool
+
+type fields struct {
+	fields []logstorage.Field
+}
+
 // parsePromLabels parses log fields in Prometheus text exposition format from s, appends them to dst and returns the result.
 //
 // See test data of promtail for examples: https://github.com/grafana/loki/blob/a24ef7b206e0ca63ee74ca6ecb0a09b745cd2258/pkg/push/types_test.go
@@ -181,6 +213,6 @@ func getPushRequest() *PushRequest {
 }
 
 func putPushRequest(req *PushRequest) {
-	req.Reset()
+	req.reset()
 	pushReqsPool.Put(req)
 }

app/vlinsert/loki/loki_protobuf_test.go

@@ -36,7 +36,7 @@ func (tlp *testLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.
 		Entries: []Entry{
 			{
 				Timestamp: time.Unix(0, timestamp),
-				Line:      msg,
+				Line:      strings.Clone(msg),
 			},
 		},
 	})
@@ -58,10 +58,7 @@ func TestParseProtobufRequest_Success(t *testing.T) {
 			t.Fatalf("unexpected number of streams; got %d; want %d", len(tlp.pr.Streams), n)
 		}
 
-		data, err := tlp.pr.Marshal()
-		if err != nil {
-			t.Fatalf("unexpected error when marshaling PushRequest: %s", err)
-		}
+		data := tlp.pr.MarshalProtobuf(nil)
 		encodedData := snappy.Encode(nil, data)
 
 		tlp2 := &insertutils.TestLogMessageProcessor{}

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {
@@ -38,29 +39,47 @@ func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
 	})
 }
 
-func getProtobufBody(streams, rows, labels int) []byte {
-	var pr PushRequest
-
-	for i := 0; i < streams; i++ {
-		var st Stream
-
-		st.Labels = `{`
-		for j := 0; j < labels; j++ {
-			st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
-			if j < labels-1 {
-				st.Labels += `,`
+func getProtobufBody(streamsCount, rowsCount, labelsCount int) []byte {
+	var b []byte
+	var entries []Entry
+	streams := make([]Stream, streamsCount)
+	for i := range streams {
+		b = b[:0]
+		b = append(b, '{')
+		for j := 0; j < labelsCount; j++ {
+			b = append(b, "label_"...)
+			b = strconv.AppendInt(b, int64(j), 10)
+			b = append(b, `="value_`...)
+			b = strconv.AppendInt(b, int64(j), 10)
+			b = append(b, '"')
+			if j < labelsCount-1 {
+				b = append(b, ',')
 			}
 		}
-		st.Labels += `}`
+		b = append(b, '}')
+		labels := string(b)
 
-		for j := 0; j < rows; j++ {
-			st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
+		var rowsBuf []byte
+		entriesLen := len(entries)
+		for j := 0; j < rowsCount; j++ {
+			rowsBufLen := len(rowsBuf)
+			rowsBuf = append(rowsBuf, "value_"...)
+			rowsBuf = strconv.AppendInt(rowsBuf, int64(j), 10)
+			entries = append(entries, Entry{
+				Timestamp: time.Now(),
+				Line:      bytesutil.ToUnsafeString(rowsBuf[rowsBufLen:]),
+			})
 		}
 
-		pr.Streams = append(pr.Streams, st)
+		st := &streams[i]
+		st.Labels = labels
+		st.Entries = entries[entriesLen:]
+	}
+	pr := PushRequest{
+		Streams: streams,
 	}
 
-	body, _ := pr.Marshal()
+	body := pr.MarshalProtobuf(nil)
 	encodedBody := snappy.Encode(nil, body)
 
 	return encodedBody

app/vlinsert/loki/pb.go

@@ -0,0 +1,302 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: push_request.proto
+// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push_request.proto
+// Licensed under the Apache License, Version 2.0 (the "License");
+// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
+
+package loki
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/easyproto"
+)
+
+var mp easyproto.MarshalerPool
+
+// PushRequest represents Loki PushRequest
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L14C1-L14C20
+type PushRequest struct {
+	Streams []Stream
+
+	entriesBuf   []Entry
+	labelPairBuf []LabelPair
+}
+
+func (pr *PushRequest) reset() {
+	pr.Streams = pr.Streams[:0]
+
+	pr.entriesBuf = pr.entriesBuf[:0]
+	pr.labelPairBuf = pr.labelPairBuf[:0]
+}
+
+// UnmarshalProtobuf unmarshals pr from protobuf message at src.
+//
+// pr remains valid until src is modified.
+func (pr *PushRequest) UnmarshalProtobuf(src []byte) error {
+	pr.reset()
+	var err error
+	pr.entriesBuf, pr.labelPairBuf, err = pr.unmarshalProtobuf(pr.entriesBuf, pr.labelPairBuf, src)
+	return err
+}
+
+// MarshalProtobuf marshals r to protobuf message, appends it to dst and returns the result.
+func (pr *PushRequest) MarshalProtobuf(dst []byte) []byte {
+	m := mp.Get()
+	pr.marshalProtobuf(m.MessageMarshaler())
+	dst = m.Marshal(dst)
+	mp.Put(m)
+	return dst
+}
+
+func (pr *PushRequest) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	for _, s := range pr.Streams {
+		s.marshalProtobuf(mm.AppendMessage(1))
+	}
+}
+
+func (pr *PushRequest) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
+	// message PushRequest {
+	//   repeated Stream streams = 1;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in PushRequest: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			data, ok := fc.MessageData()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Stream data")
+			}
+			pr.Streams = append(pr.Streams, Stream{})
+			s := &pr.Streams[len(pr.Streams)-1]
+			entriesBuf, labelPairBuf, err = s.unmarshalProtobuf(entriesBuf, labelPairBuf, data)
+			if err != nil {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Stream: %w", err)
+			}
+		}
+	}
+	return entriesBuf, labelPairBuf, nil
+}
+
+// Stream represents Loki stream.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L23
+type Stream struct {
+	Labels  string
+	Entries []Entry
+}
+
+func (s *Stream) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendString(1, s.Labels)
+	for _, e := range s.Entries {
+		e.marshalProtobuf(mm.AppendMessage(2))
+	}
+}
+
+func (s *Stream) unmarshalProtobuf(entriesBuf []Entry, labelPairBuf []LabelPair, src []byte) ([]Entry, []LabelPair, error) {
+	// message Stream {
+	//   string labels = 1;
+	//   repeated Entry entries = 2;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	entriesBufLen := len(entriesBuf)
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return entriesBuf, labelPairBuf, fmt.Errorf("cannot read next field in Stream: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			labels, ok := fc.String()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read labels")
+			}
+			s.Labels = labels
+		case 2:
+			data, ok := fc.MessageData()
+			if !ok {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot read Entry data")
+			}
+			entriesBuf = append(entriesBuf, Entry{})
+			e := &entriesBuf[len(entriesBuf)-1]
+			labelPairBuf, err = e.unmarshalProtobuf(labelPairBuf, data)
+			if err != nil {
+				return entriesBuf, labelPairBuf, fmt.Errorf("cannot unmarshal Entry: %w", err)
+			}
+		}
+	}
+	s.Entries = entriesBuf[entriesBufLen:]
+	return entriesBuf, labelPairBuf, nil
+}
+
+// Entry represents Loki entry.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L38
+type Entry struct {
+	Timestamp          time.Time
+	Line               string
+	StructuredMetadata []LabelPair
+}
+
+func (e *Entry) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	marshalTime(mm, 1, e.Timestamp)
+	mm.AppendString(2, e.Line)
+	for _, lp := range e.StructuredMetadata {
+		lp.marshalProtobuf(mm.AppendMessage(3))
+	}
+}
+
+func (e *Entry) unmarshalProtobuf(labelPairBuf []LabelPair, src []byte) ([]LabelPair, error) {
+	// message Entry {
+	//   Timestamp timestamp = 1;
+	//   string line = 2;
+	//   repeated LabelPair structuredMetadata = 3;
+	// }
+	var err error
+	var fc easyproto.FieldContext
+	labelPairBufLen := len(labelPairBuf)
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return labelPairBuf, fmt.Errorf("cannot read next field in Entry: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			data, ok := fc.MessageData()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read Timestamp data")
+			}
+			timestamp, err := unmarshalTime(data)
+			if err != nil {
+				return labelPairBuf, fmt.Errorf("cannot unmarshal Timestamp: %w", err)
+			}
+			e.Timestamp = timestamp
+		case 2:
+			line, ok := fc.String()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read Line")
+			}
+			e.Line = line
+		case 3:
+			data, ok := fc.MessageData()
+			if !ok {
+				return labelPairBuf, fmt.Errorf("cannot read StructuredMetadata")
+			}
+			labelPairBuf = append(labelPairBuf, LabelPair{})
+			lp := &labelPairBuf[len(labelPairBuf)-1]
+			if err := lp.unmarshalProtobuf(data); err != nil {
+				return labelPairBuf, fmt.Errorf("cannot unmarshal StructuredMetadata: %w", err)
+			}
+		}
+	}
+	e.StructuredMetadata = labelPairBuf[labelPairBufLen:]
+	return labelPairBuf, nil
+}
+
+// LabelPair represents Loki label pair.
+//
+// See https://github.com/grafana/loki/blob/4220737a52da7ab6c9346b12d5a5d7bedbcd641d/pkg/push/push.proto#L33
+type LabelPair struct {
+	Name  string
+	Value string
+}
+
+func (lp *LabelPair) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendString(1, lp.Name)
+	mm.AppendString(2, lp.Value)
+}
+
+func (lp *LabelPair) unmarshalProtobuf(src []byte) (err error) {
+	// message LabelPair {
+	//   string name = 1;
+	//   string value = 2;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read next field in LabelPair: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			name, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot read name")
+			}
+			lp.Name = name
+		case 2:
+			value, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot unmarshal value")
+			}
+			lp.Value = value
+		}
+	}
+	return nil
+}
+
+func marshalTime(mm *easyproto.MessageMarshaler, fieldNum uint32, timestamp time.Time) {
+	nsecs := timestamp.UnixNano()
+	ts := Timestamp{
+		Seconds: nsecs / 1e9,
+		Nanos:   int32(nsecs % 1e9),
+	}
+	ts.marshalProtobuf(mm.AppendMessage(fieldNum))
+}
+
+func unmarshalTime(src []byte) (time.Time, error) {
+	var ts Timestamp
+	if err := ts.unmarshalProtobuf(src); err != nil {
+		return time.Time{}, err
+	}
+	timestamp := time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
+	return timestamp, nil
+}
+
+// Timestamp is protobuf well-known timestamp type.
+type Timestamp struct {
+	Seconds int64
+	Nanos   int32
+}
+
+func (ts *Timestamp) marshalProtobuf(mm *easyproto.MessageMarshaler) {
+	mm.AppendInt64(1, ts.Seconds)
+	mm.AppendInt32(2, ts.Nanos)
+}
+
+func (ts *Timestamp) unmarshalProtobuf(src []byte) (err error) {
+	// message Timestamp {
+	//   int64 seconds = 1;
+	//   int32 nanos = 2;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read next field in Timestamp: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			seconds, ok := fc.Int64()
+			if !ok {
+				return fmt.Errorf("cannot read Seconds")
+			}
+			ts.Seconds = seconds
+		case 2:
+			nanos, ok := fc.Int32()
+			if !ok {
+				return fmt.Errorf("cannot read Nanos")
+			}
+			ts.Nanos = nanos
+		}
+	}
+	return nil
+}

app/vlinsert/loki/push_request.proto

@@ -1,38 +0,0 @@
-syntax = "proto3";
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/push.proto
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-package logproto;
-
-import "gogoproto/gogo.proto";
-import "google/protobuf/timestamp.proto";
-
-option go_package = "github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/loki";
-
-message PushRequest {
-  repeated StreamAdapter streams = 1 [
-    (gogoproto.jsontag) = "streams",
-    (gogoproto.customtype) = "Stream"
-  ];
-}
-
-message StreamAdapter {
-  string labels = 1 [(gogoproto.jsontag) = "labels"];
-  repeated EntryAdapter entries = 2 [
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "entries"
-  ];
-  // hash contains the original hash of the stream.
-  uint64 hash = 3 [(gogoproto.jsontag) = "-"];
-}
-
-message EntryAdapter {
-  google.protobuf.Timestamp timestamp = 1 [
-    (gogoproto.stdtime) = true,
-    (gogoproto.nullable) = false,
-    (gogoproto.jsontag) = "ts"
-  ];
-  string line = 2 [(gogoproto.jsontag) = "line"];
-}

app/vlinsert/loki/timestamp.go

@@ -1,110 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/timestamp.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"errors"
-	"strconv"
-	"time"
-
-	"github.com/gogo/protobuf/types"
-)
-
-const (
-	// Seconds field of the earliest valid Timestamp.
-	// This is time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	minValidSeconds = -62135596800
-	// Seconds field just after the latest valid Timestamp.
-	// This is time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC).Unix().
-	maxValidSeconds = 253402300800
-)
-
-// validateTimestamp determines whether a Timestamp is valid.
-// A valid timestamp represents a time in the range
-// [0001-01-01, 10000-01-01) and has a Nanos field
-// in the range [0, 1e9).
-//
-// If the Timestamp is valid, validateTimestamp returns nil.
-// Otherwise, it returns an error that describes
-// the problem.
-//
-// Every valid Timestamp can be represented by a time.Time, but the converse is not true.
-func validateTimestamp(ts *types.Timestamp) error {
-	if ts == nil {
-		return errors.New("timestamp: nil Timestamp")
-	}
-	if ts.Seconds < minValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " before 0001-01-01")
-	}
-	if ts.Seconds >= maxValidSeconds {
-		return errors.New("timestamp: " + formatTimestamp(ts) + " after 10000-01-01")
-	}
-	if ts.Nanos < 0 || ts.Nanos >= 1e9 {
-		return errors.New("timestamp: " + formatTimestamp(ts) + ": nanos not in range [0, 1e9)")
-	}
-	return nil
-}
-
-// formatTimestamp is equivalent to fmt.Sprintf("%#v", ts)
-// but avoids the escape incurred by using fmt.Sprintf, eliminating
-// unnecessary heap allocations.
-func formatTimestamp(ts *types.Timestamp) string {
-	if ts == nil {
-		return "nil"
-	}
-
-	seconds := strconv.FormatInt(ts.Seconds, 10)
-	nanos := strconv.FormatInt(int64(ts.Nanos), 10)
-	return "&types.Timestamp{Seconds: " + seconds + ",\nNanos: " + nanos + ",\n}"
-}
-
-func sizeOfStdTime(t time.Time) int {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0
-	}
-	return ts.Size()
-}
-
-func stdTimeMarshalTo(t time.Time, data []byte) (int, error) {
-	ts, err := timestampProto(t)
-	if err != nil {
-		return 0, err
-	}
-	return ts.MarshalTo(data)
-}
-
-func stdTimeUnmarshal(t *time.Time, data []byte) error {
-	ts := &types.Timestamp{}
-	if err := ts.Unmarshal(data); err != nil {
-		return err
-	}
-	tt, err := timestampFromProto(ts)
-	if err != nil {
-		return err
-	}
-	*t = tt
-	return nil
-}
-
-func timestampFromProto(ts *types.Timestamp) (time.Time, error) {
-	// Don't return the zero value on error, because corresponds to a valid
-	// timestamp. Instead return whatever time.Unix gives us.
-	var t time.Time
-	if ts == nil {
-		t = time.Unix(0, 0).UTC() // treat nil like the empty Timestamp
-	} else {
-		t = time.Unix(ts.Seconds, int64(ts.Nanos)).UTC()
-	}
-	return t, validateTimestamp(ts)
-}
-
-func timestampProto(t time.Time) (types.Timestamp, error) {
-	ts := types.Timestamp{
-		Seconds: t.Unix(),
-		Nanos:   int32(t.Nanosecond()),
-	}
-	return ts, validateTimestamp(&ts)
-}

app/vlinsert/loki/types.go

@@ -1,481 +0,0 @@
-package loki
-
-// source: https://raw.githubusercontent.com/grafana/loki/main/pkg/push/types.go
-// Licensed under the Apache License, Version 2.0 (the "License");
-// https://github.com/grafana/loki/blob/main/pkg/push/LICENSE
-
-import (
-	"fmt"
-	"io"
-	"time"
-)
-
-// Stream contains a unique labels set as a string and a set of entries for it.
-// We are not using the proto generated version but this custom one so that we
-// can improve serialization see benchmark.
-type Stream struct {
-	Labels  string  `protobuf:"bytes,1,opt,name=labels,proto3" json:"labels"`
-	Entries []Entry `protobuf:"bytes,2,rep,name=entries,proto3,customtype=EntryAdapter" json:"entries"`
-	Hash    uint64  `protobuf:"varint,3,opt,name=hash,proto3" json:"-"`
-}
-
-// Entry is a log entry with a timestamp.
-type Entry struct {
-	Timestamp time.Time `protobuf:"bytes,1,opt,name=timestamp,proto3,stdtime" json:"ts"`
-	Line      string    `protobuf:"bytes,2,opt,name=line,proto3" json:"line"`
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Stream) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Stream) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Stream) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if m.Hash != 0 {
-		i = encodeVarintPush(dAtA, i, m.Hash)
-		i--
-		dAtA[i] = 0x18
-	}
-	if len(m.Entries) > 0 {
-		for iNdEx := len(m.Entries) - 1; iNdEx >= 0; iNdEx-- {
-			{
-				size, err := m.Entries[iNdEx].MarshalToSizedBuffer(dAtA[:i])
-				if err != nil {
-					return 0, err
-				}
-				i -= size
-				i = encodeVarintPush(dAtA, i, uint64(size))
-			}
-			i--
-			dAtA[i] = 0x12
-		}
-	}
-	if len(m.Labels) > 0 {
-		i -= len(m.Labels)
-		copy(dAtA[i:], m.Labels)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Labels)))
-		i--
-		dAtA[i] = 0xa
-	}
-	return len(dAtA) - i, nil
-}
-
-// Marshal implements the proto.Marshaler interface.
-func (m *Entry) Marshal() (dAtA []byte, err error) {
-	size := m.Size()
-	dAtA = make([]byte, size)
-	n, err := m.MarshalToSizedBuffer(dAtA[:size])
-	if err != nil {
-		return nil, err
-	}
-	return dAtA[:n], nil
-}
-
-// MarshalTo marshals m to dst.
-func (m *Entry) MarshalTo(dAtA []byte) (int, error) {
-	size := m.Size()
-	return m.MarshalToSizedBuffer(dAtA[:size])
-}
-
-// MarshalToSizedBuffer marshals m to the sized buffer.
-func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error) {
-	i := len(dAtA)
-	_ = i
-	var l int
-	_ = l
-	if len(m.Line) > 0 {
-		i -= len(m.Line)
-		copy(dAtA[i:], m.Line)
-		i = encodeVarintPush(dAtA, i, uint64(len(m.Line)))
-		i--
-		dAtA[i] = 0x12
-	}
-	n7, err7 := stdTimeMarshalTo(m.Timestamp, dAtA[i-sizeOfStdTime(m.Timestamp):])
-	if err7 != nil {
-		return 0, err7
-	}
-	i -= n7
-	i = encodeVarintPush(dAtA, i, uint64(n7))
-	i--
-	dAtA[i] = 0xa
-	return len(dAtA) - i, nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Stream) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: StreamAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: StreamAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Labels = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Entries", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Entries = append(m.Entries, Entry{})
-			if err := m.Entries[len(m.Entries)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 3:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType)
-			}
-			m.Hash = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Hash |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Unmarshal unmarshals the given data into m.
-func (m *Entry) Unmarshal(dAtA []byte) error {
-	l := len(dAtA)
-	iNdEx := 0
-	for iNdEx < l {
-		preIndex := iNdEx
-		var wire uint64
-		for shift := uint(0); ; shift += 7 {
-			if shift >= 64 {
-				return ErrIntOverflowPush
-			}
-			if iNdEx >= l {
-				return io.ErrUnexpectedEOF
-			}
-			b := dAtA[iNdEx]
-			iNdEx++
-			wire |= uint64(b&0x7F) << shift
-			if b < 0x80 {
-				break
-			}
-		}
-		fieldNum := int32(wire >> 3)
-		wireType := int(wire & 0x7)
-		if wireType == 4 {
-			return fmt.Errorf("proto: EntryAdapter: wiretype end group for non-group")
-		}
-		if fieldNum <= 0 {
-			return fmt.Errorf("proto: EntryAdapter: illegal tag %d (wire type %d)", fieldNum, wire)
-		}
-		switch fieldNum {
-		case 1:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType)
-			}
-			var msglen int
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				msglen |= int(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			if msglen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + msglen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			if err := stdTimeUnmarshal(&m.Timestamp, dAtA[iNdEx:postIndex]); err != nil {
-				return err
-			}
-			iNdEx = postIndex
-		case 2:
-			if wireType != 2 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Line", wireType)
-			}
-			var stringLen uint64
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowPush
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				stringLen |= uint64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
-			intStringLen := int(stringLen)
-			if intStringLen < 0 {
-				return ErrInvalidLengthPush
-			}
-			postIndex := iNdEx + intStringLen
-			if postIndex < 0 {
-				return ErrInvalidLengthPush
-			}
-			if postIndex > l {
-				return io.ErrUnexpectedEOF
-			}
-			m.Line = string(dAtA[iNdEx:postIndex])
-			iNdEx = postIndex
-		default:
-			iNdEx = preIndex
-			skippy, err := skipPush(dAtA[iNdEx:])
-			if err != nil {
-				return err
-			}
-			if skippy < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) < 0 {
-				return ErrInvalidLengthPush
-			}
-			if (iNdEx + skippy) > l {
-				return io.ErrUnexpectedEOF
-			}
-			iNdEx += skippy
-		}
-	}
-
-	if iNdEx > l {
-		return io.ErrUnexpectedEOF
-	}
-	return nil
-}
-
-// Size returns the size of the serialized Stream.
-func (m *Stream) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = len(m.Labels)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	if len(m.Entries) > 0 {
-		for _, e := range m.Entries {
-			l = e.Size()
-			n += 1 + l + sovPush(uint64(l))
-		}
-	}
-	if m.Hash != 0 {
-		n += 1 + sovPush(m.Hash)
-	}
-	return n
-}
-
-// Size returns the size of the serialized Entry
-func (m *Entry) Size() (n int) {
-	if m == nil {
-		return 0
-	}
-	var l int
-	_ = l
-	l = sizeOfStdTime(m.Timestamp)
-	n += 1 + l + sovPush(uint64(l))
-	l = len(m.Line)
-	if l > 0 {
-		n += 1 + l + sovPush(uint64(l))
-	}
-	return n
-}
-
-// Equal returns true if the two Streams are equal.
-func (m *Stream) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Stream)
-	if !ok {
-		that2, ok := that.(Stream)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if m.Labels != that1.Labels {
-		return false
-	}
-	if len(m.Entries) != len(that1.Entries) {
-		return false
-	}
-	for i := range m.Entries {
-		if !m.Entries[i].Equal(that1.Entries[i]) {
-			return false
-		}
-	}
-	return m.Hash == that1.Hash
-}
-
-// Equal returns true if the two Entries are equal.
-func (m *Entry) Equal(that interface{}) bool {
-	if that == nil {
-		return m == nil
-	}
-
-	that1, ok := that.(*Entry)
-	if !ok {
-		that2, ok := that.(Entry)
-		if ok {
-			that1 = &that2
-		} else {
-			return false
-		}
-	}
-	if that1 == nil {
-		return m == nil
-	} else if m == nil {
-		return false
-	}
-	if !m.Timestamp.Equal(that1.Timestamp) {
-		return false
-	}
-	if m.Line != that1.Line {
-		return false
-	}
-	return true
-}

app/vlogsgenerator/main.go

@@ -230,7 +230,7 @@ func generateLogsAtTimestamp(bw *bufio.Writer, workerID int, ts int64, firstStre
 	for i := 0; i < activeStreams; i++ {
 		ip := toIPv4(rand.Uint32())
 		uuid := toUUID(rand.Uint64(), rand.Uint64())
-		fmt.Fprintf(bw, `{"_time":%q,"_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
+		fmt.Fprintf(bw, `{"_time":"%s","_msg":"message for the stream %d and worker %d; ip=%s; uuid=%s; u64=%d","host":"host_%d","worker_id":"%d"`,
 			timeStr, streamID, workerID, ip, uuid, rand.Uint64(), streamID, workerID)
 		fmt.Fprintf(bw, `,"run_id":"%s"`, runID)
 		for j := 0; j < *constFieldsPerLog; j++ {

app/vlselect/logsql/logsql.go

@@ -70,9 +70,10 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		fieldsLimit = 0
 	}
 
-	// Prepare the query
-	q.AddCountByTimePipe(int64(step), int64(offset), fields)
+	// Prepare the query for hits count.
 	q.Optimize()
+	q.DropAllPipes()
+	q.AddCountByTimePipe(int64(step), int64(offset), fields)
 
 	var mLock sync.Mutex
 	m := make(map[string]*hitsSeries)

app/vlselect/vmui/asset-manifest.json

@@ -1,13 +1,13 @@
 {
   "files": {
-    "main.css": "./static/css/main.1041c3d4.css",
-    "main.js": "./static/js/main.8988988c.js",
+    "main.css": "./static/css/main.c9cc37dd.css",
+    "main.js": "./static/js/main.7ae2e2c4.js",
     "static/js/685.bebe1265.chunk.js": "./static/js/685.bebe1265.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.aaabf95f2c9bf356bde4.md",
+    "static/media/MetricsQL.md": "./static/media/MetricsQL.8c2e588d62b87f90dbf0.md",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.1041c3d4.css",
-    "static/js/main.8988988c.js"
+    "static/css/main.c9cc37dd.css",
+    "static/js/main.7ae2e2c4.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.8988988c.js"></script><link href="./static/css/main.1041c3d4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.7ae2e2c4.js"></script><link href="./static/css/main.c9cc37dd.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlselect/vmui/static/media/MetricsQL.8c2e588d62b87f90dbf0.md

@@ -1,5 +1,4 @@
 ---
-sort: 23
 weight: 23
 title: MetricsQL
 menu:
@@ -10,9 +9,6 @@ aliases:
 - /ExtendedPromQL.html
 - /MetricsQL.html
 ---
-
-# MetricsQL
-
 [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) implements MetricsQL -
 query language inspired by [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 MetricsQL is backwards-compatible with PromQL, so Grafana dashboards backed by Prometheus datasource should work
@@ -220,9 +216,11 @@ See also [descent_over_time](#descent_over_time).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [median_over_time](#median_over_time).
+See also [median_over_time](#median_over_time), [min_over_time](#min_over_time) and [max_over_time](#max_over_time).
 
 #### changes
 
@@ -262,6 +260,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time), [share_eq_over_time](#share_eq_over_time) and [count_values_over_time](#count_values_over_time).
 
 #### count_gt_over_time
@@ -272,6 +272,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_gt_over_time](#share_gt_over_time).
 
 #### count_le_over_time
@@ -282,6 +284,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_over_time](#count_over_time) and [share_le_over_time](#share_le_over_time).
 
 #### count_ne_over_time
@@ -292,7 +296,9 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
-See also [count_over_time](#count_over_time).
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [count_over_time](#count_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### count_over_time
 
@@ -313,6 +319,8 @@ The results are calculated independently per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time), [count_values](#count_values) and [distinct_over_time](#distinct_over_time) and [label_match](#label_match).
 
 #### decreases_over_time
@@ -432,6 +440,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### histogram_over_time
 
 `histogram_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates
@@ -443,11 +453,15 @@ For example, the following query calculates median temperature by country over t
 
 `histogram_quantile(0.5, sum(histogram_over_time(temperature[24h])) by (vmrange,country))`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### hoeffding_bound_lower
 
 `hoeffding_bound_lower(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 lower [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 
 #### hoeffding_bound_upper
@@ -455,6 +469,8 @@ See also [hoeffding_bound_upper](#hoeffding_bound_upper).
 `hoeffding_bound_upper(phi, series_selector[d])` is a [rollup function](#rollup-functions), which calculates
 upper [Hoeffding bound](https://en.wikipedia.org/wiki/Hoeffding%27s_inequality) for the given `phi` in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 
 #### holt_winters
@@ -462,8 +478,9 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
 `holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
 (aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
-Both `sf` and `tf` must be in the range `[0...1]`. It is expected that the [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
-returns time series of [gauge type](https://docs.victoriametrics.com/keyconcepts/#gauge).
+Both `sf` and `tf` must be in the range `[0...1]`.
+
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
 
 This function is supported by PromQL.
 
@@ -495,13 +512,14 @@ See also [deriv](#deriv).
 
 `increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Unlike Prometheus, it takes into account the last sample before the given lookbehind window `d` when calculating the result.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [increase_pure](#increase_pure), [increase_prometheus](#increase_prometheus) and [delta](#delta).
@@ -510,12 +528,13 @@ See also [increase_pure](#increase_pure), [increase_prometheus](#increase_promet
 
 `increase_prometheus(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the increase
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
 See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 See also [increase_pure](#increase_pure) and [increase](#increase).
 
 #### increase_pure
@@ -524,6 +543,10 @@ See also [increase_pure](#increase_pure) and [increase](#increase).
 of the following corner case - it assumes that [counters](https://docs.victoriametrics.com/keyconcepts/#counter) always start from 0,
 while [increase](#increase) ignores the first value in a series if it is too big.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [increase](#increase) and [increase_prometheus](#increase_prometheus).
+
 #### increases_over_time
 
 `increases_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the number of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -540,15 +563,18 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### irate
 
 `irate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the "instant" per-second increase rate over
 the last two [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 See also [rate](#rate) and [rollup_rate](#rollup_rate).
@@ -587,6 +613,8 @@ See also [duration_over_time](#duration_over_time) and [lag](#lag).
 over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 #### max_over_time
@@ -594,9 +622,11 @@ See also [mad](#mad), [range_mad](#range_mad) and [outlier_iqr_over_time](#outli
 `max_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the maximum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmax_over_time](#tmax_over_time).
+See also [tmax_over_time](#tmax_over_time) and [min_over_time](#min_over_time).
 
 #### median_over_time
 
@@ -604,6 +634,8 @@ See also [tmax_over_time](#tmax_over_time).
 on the given lookbehind window `d` per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [avg_over_time](#avg_over_time).
 
 #### min_over_time
@@ -611,9 +643,11 @@ See also [avg_over_time](#avg_over_time).
 `min_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the minimum value over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
-See also [tmin_over_time](#tmin_over_time).
+See also [tmin_over_time](#tmin_over_time) and [max_over_time](#max_over_time).
 
 #### mode_over_time
 
@@ -622,6 +656,8 @@ for [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering). It is expected that [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values are discrete.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### outlier_iqr_over_time
 
 `outlier_iqr_over_time(series_selector[d])` is a [rollup function](#rollup-functions), which returns the last sample on the given lookbehind window `d`
@@ -632,6 +668,8 @@ if its value is either smaller than the `q25-1.5*iqr` or bigger than `q75+1.5*iq
 The `outlier_iqr_over_time()` is useful for detecting anomalies in gauge values based on the previous history of values.
 For example, `outlier_iqr_over_time(memory_usage_bytes[1h])` triggers when `memory_usage_bytes` suddenly goes outside the usual value range for the last hour.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [outliers_iqr](#outliers_iqr).
 
 #### predict_linear
@@ -659,6 +697,8 @@ This function is supported by PromQL.
 on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The `phi` value must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [quantiles_over_time](#quantiles_over_time).
@@ -670,6 +710,8 @@ over [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples) on
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 The function returns individual series per each `phi*` with `{phiLabel="phi*"}` label. `phi*` values must be in the range `[0...1]`.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [quantile_over_time](#quantile_over_time).
 
 #### range_over_time
@@ -680,11 +722,12 @@ E.g. it calculates `max_over_time(series_selector[d]) - min_over_time(series_sel
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rate
 
 `rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the average per-second increase rate
 over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 If the lookbehind window is skipped in square brackets, then it is automatically calculated as `max(step, scrape_interval)`, where `step` is the query arg value
 passed to [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) or [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query),
@@ -703,6 +746,8 @@ See also [irate](#irate) and [rollup_rate](#rollup_rate).
 on the given lookbehind window `d`. The calculations are performed individually per each time series returned
 from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
 #### resets
@@ -710,10 +755,11 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
 `resets(series_selector[d])` is a [rollup function](#rollup-functions), which returns the number
 of [counter](https://docs.victoriametrics.com/keyconcepts/#counter) resets over the given lookbehind window `d`
 per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-It is expected that the `series_selector` returns time series of [counter type](https://docs.victoriametrics.com/keyconcepts/#counter).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
 This function is supported by PromQL.
 
 #### rollup
@@ -725,6 +771,10 @@ These values are calculated individually per each time series returned from the
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
+See also [rollup_rate](#rollup_rate).
+
 #### rollup_candlestick
 
 `rollup_candlestick(series_selector[d])` is a [rollup function](#rollup-functions), which calculates `open`, `high`, `low` and `close` values (aka OHLC)
@@ -736,6 +786,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 Optional 2nd argument `"open"`, `"high"` or `"low"` or `"close"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### rollup_delta
 
 `rollup_delta(series_selector[d])` is a [rollup function](#rollup-functions), which calculates differences between adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -762,6 +814,8 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_increase
 
 `rollup_increase(series_selector[d])` is a [rollup function](#rollup-functions), which calculates increases for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
@@ -774,12 +828,17 @@ See also [label_match](#label_match).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names. See also [rollup_delta](#rollup_delta).
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_rate](#rollup_rate).
+
 #### rollup_rate
 
 `rollup_rate(series_selector[d])` is a [rollup function](#rollup-functions), which calculates per-second change rates
 for adjacent [raw samples](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 on the given lookbehind window `d` and returns `min`, `max` and `avg` values for the calculated per-second change rates
 and returns them in time series with `rollup="min"`, `rollup="max"` and `rollup="avg"` additional labels.
+The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 See [this article](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832) in order to understand better
 when to use `rollup_rate()`.
@@ -787,10 +846,12 @@ when to use `rollup_rate()`.
 Optional 2nd argument `"min"`, `"max"` or `"avg"` can be passed to keep only one calculation result and without adding a label.
 See also [label_match](#label_match).
 
-The calculations are performed individually per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
-
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [counters](https://docs.victoriametrics.com/keyconcepts/#counter).
+
+See also [rollup](#rollup) and [rollup_increase](#rollup_increase).
+
 #### rollup_scrape_interval
 
 `rollup_scrape_interval(series_selector[d])` is a [rollup function](#rollup-functions), which calculates the interval in seconds between
@@ -824,6 +885,8 @@ This function is useful for calculating SLI and SLO. Example: `share_gt_over_tim
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_le_over_time](#share_le_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### share_le_over_time
@@ -838,6 +901,8 @@ the share of time series values for the last 24 hours when memory usage was belo
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [share_gt_over_time](#share_gt_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### share_eq_over_time
@@ -849,6 +914,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [count_eq_over_time](#count_eq_over_time).
 
 #### stale_samples_over_time
@@ -866,6 +933,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stdvar_over_time](#stdvar_over_time).
@@ -877,35 +946,43 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 See also [stddev_over_time](#stddev_over_time).
 
 #### sum_eq_over_time
 
-`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_eq_over_time(series_selector[d], eq)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values equal to `eq` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_eq_over_time](#count_eq_over_time).
 
 #### sum_gt_over_time
 
-`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_gt_over_time(series_selector[d], gt)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values bigger than `gt` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_gt_over_time](#count_gt_over_time).
 
 #### sum_le_over_time
 
-`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-function), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
+`sum_le_over_time(series_selector[d], le)` is a [rollup function](#rollup-functions), which calculates the sum of [raw sample](https://docs.victoriametrics.com/keyconcepts/#raw-samples)
 values smaller or equal to `le` on the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#filtering).
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [sum_over_time](#sum_over_time) and [count_le_over_time](#count_le_over_time).
 
 #### sum_over_time
@@ -915,6 +992,8 @@ on the given lookbehind window `d` per each time series returned from the given
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 This function is supported by PromQL.
 
 #### sum2_over_time
@@ -924,6 +1003,8 @@ values on the given lookbehind window `d` per each time series returned from the
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 #### timestamp
 
 `timestamp(series_selector[d])` is a [rollup function](#rollup-functions), which returns the timestamp in seconds with millisecond precision
@@ -1001,6 +1082,8 @@ from the given [series_selector](https://docs.victoriametrics.com/keyconcepts/#f
 
 Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
 
+This function is usually applied to [gauges](https://docs.victoriametrics.com/keyconcepts/#gauge).
+
 See also [zscore](#zscore), [range_trim_zscore](#range_trim_zscore) and [outlier_iqr_over_time](#outlier_iqr_over_time).
 
 
@@ -2230,7 +2313,8 @@ Any [rollup function](#rollup-functions) for something other than [series select
 Nested rollup functions can be implicit thanks to the [implicit query conversions](#implicit-query-conversions).
 For example, `delta(sum(m))` is implicitly converted to `delta(sum(default_rollup(m))[1i:1i])`, so it becomes a subquery,
 since it contains [default_rollup](#default_rollup) nested into [delta](#delta).
-This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
 
 VictoriaMetrics performs subqueries in the following way:
 
@@ -2263,5 +2347,6 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   For example, `avg_over_time(rate(http_requests_total[5m])[1h])` is automatically converted to `avg_over_time(rate(http_requests_total[5m])[1h:1i])`.
 * If something other than [series selector](https://docs.victoriametrics.com/keyconcepts/#filtering)
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
-  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`. 
-  This behavior can be disabled or logged via cmd-line flags `-search.disableImplicitConversion` and `-search.logImplicitConversion` since v1.101.0.
+  For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
+  This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
+  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).

app/vlselect/vmui/victorialogs.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta
+    name="description"
+    content="UI for VictoriaMetrics"
+  />
+  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
+  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <!--
+    Notice the use of %PUBLIC_URL% in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>VM UI</title>
+
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:site" content="@VictoriaMetrics">
+
+  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta property="og:type" content="website">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlselect/vmui/victoriametrics.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta
+    name="description"
+    content="UI for VictoriaMetrics"
+  />
+  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
+  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <!--
+    Notice the use of %PUBLIC_URL% in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>VM UI</title>
+  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
+
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:site" content="@VictoriaMetrics">
+
+  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta property="og:type" content="website">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlselect/vmui/vmanomaly.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta
+    name="description"
+    content="UI for VictoriaMetrics"
+  />
+  <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png"/>
+  <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <!--
+    Notice the use of %PUBLIC_URL% in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>VM UI</title>
+  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
+
+  <meta name="twitter:card" content="summary_large_image">
+  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:site" content="@VictoriaMetrics">
+
+  <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta property="og:type" content="website">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlstorage/main.go

@@ -80,7 +80,7 @@ func Init() {
 
 // Stop stops vlstorage.
 func Stop() {
-	metrics.UnregisterSet(storageMetrics)
+	metrics.UnregisterSet(storageMetrics, true)
 	storageMetrics = nil
 
 	strg.MustClose()

app/vmagent/main.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogsketches"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadogv1"
@@ -42,7 +44,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/opentelemetry/firehose"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 )
 
 var (
@@ -70,8 +72,8 @@ var (
 		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
 	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
 		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
-	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
-	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+	configAuthKey = flagutil.NewPassword("configAuthKey", "Authorization key for accessing /config page. It must be passed via authKey query arg. It overrides -httpAuth.*")
+	reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	dryRun        = flag.Bool("dryRun", false, "Whether to check config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig, -remoteWrite.streamAggr.config . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
@@ -114,7 +116,7 @@ func main() {
 			logger.Fatalf("error when checking relabel configs: %s", err)
 		}
 		if err := remotewrite.CheckStreamAggrConfigs(); err != nil {
-			logger.Fatalf("error when checking -remoteWrite.streamAggr.config: %s", err)
+			logger.Fatalf("error when checking -streamAggr.config and -remoteWrite.streamAggr.config: %s", err)
 		}
 		logger.Infof("all the configs are ok; exiting with 0 status code")
 		return
@@ -316,6 +318,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "/influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "/opentelemetry/api/v1/push", "/opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(nil, r); err != nil {
@@ -434,7 +440,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -443,17 +449,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if !httpserver.CheckAuthFlag(w, r, configAuthKey.Get(), "configAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey) {
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()
 		w.Header().Set("Content-Type", "application/json")
 		var bb bytesutil.ByteBuffer
 		promscrape.WriteConfigData(&bb)
-		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%q}}`, bb.B)
+		fmt.Fprintf(w, `{"status":"success","data":{"yaml":%s}}`, stringsutil.JSONString(string(bb.B)))
 		return true
 	case "/prometheus/-/reload", "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
 		}
 		promscrapeConfigReloadRequests.Inc()
@@ -562,6 +568,10 @@ func processMultitenantRequest(w http.ResponseWriter, r *http.Request, path stri
 		influxQueryRequests.Inc()
 		influxutils.WriteDatabaseNames(w)
 		return true
+	case "influx/health":
+		influxHealthRequests.Inc()
+		influxutils.WriteHealthCheckResponse(w)
+		return true
 	case "opentelemetry/api/v1/push", "opentelemetry/v1/metrics":
 		opentelemetryPushRequests.Inc()
 		if err := opentelemetry.InsertHandler(at, r); err != nil {
@@ -672,7 +682,8 @@ var (
 	influxWriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/write", protocol="influx"}`)
 	influxWriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/influx/write", protocol="influx"}`)
 
-	influxQueryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxQueryRequests  = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/query", protocol="influx"}`)
+	influxHealthRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/influx/health", protocol="influx"}`)
 
 	datadogv1WriteRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/datadog/api/v1/series", protocol="datadog"}`)
 	datadogv1WriteErrors   = metrics.NewCounter(`vmagent_http_request_errors_total{path="/datadog/api/v1/series", protocol="datadog"}`)

app/vmagent/prometheusimport/request_handler_test.go

@@ -26,11 +26,11 @@ func TestInsertHandler(t *testing.T) {
 	req := httptest.NewRequest(http.MethodPost, "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
 go_memstats_alloc_bytes_total 1`))
 	if err := InsertHandler(nil, req); err != nil {
-		t.Errorf("unxepected error %s", err)
+		t.Fatalf("unxepected error %s", err)
 	}
 	expectedMsg := "cannot unmarshal Prometheus line"
 	if !strings.Contains(testOutput.String(), expectedMsg) {
-		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
+		t.Fatalf("output %q should contain %q", testOutput.String(), expectedMsg)
 	}
 }
 

app/vmagent/remotewrite/client.go

@@ -15,8 +15,8 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
@@ -34,8 +34,10 @@ var (
 	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage. See also -maxIngestionRate")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
-	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
+	sendTimeout      = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
+	retryMinInterval = flagutil.NewArrayDuration("remoteWrite.retryMinInterval", time.Second, "The minimum delay between retry attempts to send a block of data to the corresponding -remoteWrite.url. Every next retry attempt will double the delay to prevent hammering of remote database. See also -remoteWrite.retryMaxInterval")
+	retryMaxTime     = flagutil.NewArrayDuration("remoteWrite.retryMaxTime", time.Minute, "The max time spent on retry attempts to send a block of data to the corresponding -remoteWrite.url. Change this value if it is expected for -remoteWrite.url to be unreachable for more than -remoteWrite.retryMaxTime. See also -remoteWrite.retryMinInterval")
+	proxyURL         = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
 	tlsHandshakeTimeout   = flagutil.NewArrayDuration("remoteWrite.tlsHandshakeTimeout", 20*time.Second, "The timeout for establishing tls connections to the corresponding -remoteWrite.url")
@@ -90,6 +92,9 @@ type client struct {
 	fq *persistentqueue.FastQueue
 	hc *http.Client
 
+	retryMinInterval time.Duration
+	retryMaxTime     time.Duration
+
 	sendBlock func(block []byte) bool
 	authCfg   *promauth.Config
 	awsCfg    *awsapi.Config
@@ -120,7 +125,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		logger.Fatalf("cannot initialize AWS Config for -remoteWrite.url=%q: %s", remoteWriteURL, err)
 	}
 	tr := &http.Transport{
-		DialContext:         httputils.GetStatDialFunc("vmagent_remotewrite"),
+		DialContext:         netutil.NewStatDialFunc("vmagent_remotewrite"),
 		TLSHandshakeTimeout: tlsHandshakeTimeout.GetOptionalArg(argIdx),
 		MaxConnsPerHost:     2 * concurrency,
 		MaxIdleConnsPerHost: 2 * concurrency,
@@ -143,13 +148,15 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
-		sanitizedURL:   sanitizedURL,
-		remoteWriteURL: remoteWriteURL,
-		authCfg:        authCfg,
-		awsCfg:         awsCfg,
-		fq:             fq,
-		hc:             hc,
-		stopCh:         make(chan struct{}),
+		sanitizedURL:     sanitizedURL,
+		remoteWriteURL:   remoteWriteURL,
+		authCfg:          authCfg,
+		awsCfg:           awsCfg,
+		fq:               fq,
+		hc:               hc,
+		retryMinInterval: retryMinInterval.GetOptionalArg(argIdx),
+		retryMaxTime:     retryMaxTime.GetOptionalArg(argIdx),
+		stopCh:           make(chan struct{}),
 	}
 	c.sendBlock = c.sendBlockHTTP
 
@@ -396,11 +403,11 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 // sendBlockHTTP sends the given block to c.remoteWriteURL.
 //
 // The function returns false only if c.stopCh is closed.
-// Otherwise it tries sending the block to remote storage indefinitely.
+// Otherwise, it tries sending the block to remote storage indefinitely.
 func (c *client) sendBlockHTTP(block []byte) bool {
 	c.rl.Register(len(block))
-	maxRetryDuration := timeutil.AddJitterToDuration(time.Minute)
-	retryDuration := timeutil.AddJitterToDuration(time.Second)
+	maxRetryDuration := timeutil.AddJitterToDuration(c.retryMaxTime)
+	retryDuration := timeutil.AddJitterToDuration(c.retryMinInterval)
 	retriesCount := 0
 
 again:

app/vmagent/remotewrite/relabel.go

@@ -181,7 +181,7 @@ func (rctx *relabelCtx) reset() {
 }
 
 var relabelCtxPool = &sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		return &relabelCtx{}
 	},
 }

app/vmagent/remotewrite/remotewrite.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -88,15 +89,15 @@ var (
 		"By default there are no limits on samples ingestion rate. See also -remoteWrite.rateLimit")
 
 	disableOnDiskQueue = flagutil.NewArrayBool("remoteWrite.disableOnDiskQueue", "Whether to disable storing pending data to -remoteWrite.tmpDataPath "+
-		"when the configured remote storage systems cannot keep up with the data ingestion rate. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence ."+
-		"See also -remoteWrite.dropSamplesOnOverload")
-	dropSamplesOnOverload = flagutil.NewArrayBool("remoteWrite.dropSamplesOnOverload", "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
-		"cannot be pushed into the configured remote storage systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
+		"when the remote storage system at the corresponding -remoteWrite.url cannot keep up with the data ingestion rate. "+
+		"See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence . See also -remoteWrite.dropSamplesOnOverload")
+	dropSamplesOnOverload = flag.Bool("remoteWrite.dropSamplesOnOverload", false, "Whether to drop samples when -remoteWrite.disableOnDiskQueue is set and if the samples "+
+		"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/vmagent#disabling-on-disk-persistence")
 )
 
 var (
-	// rwctxs contains statically populated entries when -remoteWrite.url is specified.
-	rwctxs []*remoteWriteCtx
+	// rwctxsGlobal contains statically populated entries when -remoteWrite.url is specified.
+	rwctxsGlobal []*remoteWriteCtx
 
 	// Data without tenant id is written to defaultAuthToken if -enableMultitenantHandlers is specified.
 	defaultAuthToken = &auth.Token{}
@@ -109,8 +110,11 @@ var (
 		StatusCode: http.StatusTooManyRequests,
 	}
 
-	// disableOnDiskQueueAll is set to true if all remoteWrite.urls were configured to disable persistent queue via disableOnDiskQueue
-	disableOnDiskQueueAll bool
+	// disableOnDiskQueueAny is set to true if at least a single -remoteWrite.url is configured with -remoteWrite.disableOnDiskQueue
+	disableOnDiskQueueAny bool
+
+	// dropSamplesOnFailureGlobal is set to true if -remoteWrite.dropSamplesOnOverload is set or if multiple -remoteWrite.disableOnDiskQueue options are set.
+	dropSamplesOnFailureGlobal bool
 )
 
 // MultitenancyEnabled returns true if -enableMultitenantHandlers is specified.
@@ -203,28 +207,18 @@ func Init() {
 	relabelConfigSuccess.Set(1)
 	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
 
-	sasFile, sasOpts := getStreamAggrOpts(-1)
-	if sasFile != "" {
-		sas, err := newStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
-		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -streamAggr.config=%q: %s", sasFile, err)
-		}
-		sasGlobal.Store(sas)
-	} else if sasOpts.DedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesDropFailed, sasOpts.DedupInterval, sasOpts.DropInputLabels, sasOpts.Alias)
-	}
+	initStreamAggrConfigGlobal()
 
-	if len(*remoteWriteURLs) > 0 {
-		rwctxs = newRemoteWriteCtxs(nil, *remoteWriteURLs)
-	}
+	rwctxsGlobal = newRemoteWriteCtxs(nil, *remoteWriteURLs)
 
-	disableOnDiskQueueAll = true
-	for _, v := range *disableOnDiskQueue {
-		if !v {
-			disableOnDiskQueueAll = false
-			break
-		}
-	}
+	disableOnDiskQueues := []bool(*disableOnDiskQueue)
+	disableOnDiskQueueAny = slices.Contains(disableOnDiskQueues, true)
+
+	// Samples must be dropped if multiple -remoteWrite.disableOnDiskQueue options are configured and at least a single is set to true.
+	// In this case it is impossible to prevent from sending many duplicates of samples passed to TryPush() to all the configured -remoteWrite.url
+	// if these samples couldn't be sent to the -remoteWrite.url with the disabled persistent queue. So it is better sending samples
+	// to the remaining -remoteWrite.url and dropping them on the blocked queue.
+	dropSamplesOnFailureGlobal = *dropSamplesOnOverload || disableOnDiskQueueAny && len(disableOnDiskQueues) > 1
 
 	dropDanglingQueues()
 
@@ -234,9 +228,9 @@ func Init() {
 		defer configReloaderWG.Done()
 		for {
 			select {
-			case <-sighupCh:
 			case <-configReloaderStopCh:
 				return
+			case <-sighupCh:
 			}
 			reloadRelabelConfigs()
 			reloadStreamAggrConfigs()
@@ -255,8 +249,8 @@ func dropDanglingQueues() {
 	// In case if there were many persistent queues with identical *remoteWriteURLs
 	// the queue with the last index will be dropped.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6140
-	existingQueues := make(map[string]struct{}, len(rwctxs))
-	for _, rwctx := range rwctxs {
+	existingQueues := make(map[string]struct{}, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
 		existingQueues[rwctx.fq.Dirname()] = struct{}{}
 	}
 
@@ -273,7 +267,7 @@ func dropDanglingQueues() {
 		}
 	}
 	if removed > 0 {
-		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxs))
+		logger.Infof("removed %d dangling queues from %q, active queues: %d", removed, *tmpDataPath, len(rwctxsGlobal))
 	}
 }
 
@@ -382,10 +376,10 @@ func Stop() {
 		deduplicatorGlobal = nil
 	}
 
-	for _, rwctx := range rwctxs {
+	for _, rwctx := range rwctxsGlobal {
 		rwctx.MustStop()
 	}
-	rwctxs = nil
+	rwctxsGlobal = nil
 
 	if sl := hourlySeriesLimiter; sl != nil {
 		sl.MustStop()
@@ -397,6 +391,8 @@ func Stop() {
 
 // PushDropSamplesOnFailure pushes wr to the configured remote storage systems set via -remoteWrite.url
 //
+// PushDropSamplesOnFailure drops wr samples if they cannot be sent to -remoteWrite.url by any reason.
+//
 // PushDropSamplesOnFailure can modify wr contents.
 func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	_ = tryPush(at, wr, true)
@@ -409,7 +405,7 @@ func PushDropSamplesOnFailure(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 //
 // The caller must return ErrQueueFullHTTPRetry to the client, which sends wr, if TryPush returns false.
 func TryPush(at *auth.Token, wr *prompbmarshal.WriteRequest) bool {
-	return tryPush(at, wr, false)
+	return tryPush(at, wr, dropSamplesOnFailureGlobal)
 }
 
 func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnFailure bool) bool {
@@ -426,33 +422,30 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 		tenantRctx = getRelabelCtx()
 		defer putRelabelCtx(tenantRctx)
 	}
-	rowsCount := getRowsCount(tss)
 
 	// Quick check whether writes to configured remote storage systems are blocked.
 	// This allows saving CPU time spent on relabeling and block compression
 	// if some of remote storage systems cannot keep up with the data ingestion rate.
-	// this shortcut is only applicable if all remote writes have disableOnDiskQueue = true
-	if disableOnDiskQueueAll {
-		for _, rwctx := range rwctxs {
-			if rwctx.fq.IsWriteBlocked() {
-				rwctx.pushFailures.Inc()
-				if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-					// Just drop samples
-					rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-					continue
-				}
-				return false
-			}
-		}
+	rwctxs, ok := getEligibleRemoteWriteCtxs(tss, forceDropSamplesOnFailure)
+	if !ok {
+		// At least a single remote write queue is blocked and dropSamplesOnFailure isn't set.
+		// Return false to the caller, so it could re-send samples again.
+		return false
+	}
+	if len(rwctxs) == 0 {
+		// All the remote write queues are skipped because they are blocked and dropSamplesOnFailure is set to true.
+		// Return true to the caller, so it doesn't re-send the samples again.
+		return true
 	}
 
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 {
+	if pcsGlobal.Len() > 0 || *usePromCompatibleNaming {
 		rctx = getRelabelCtx()
 		defer putRelabelCtx(rctx)
 	}
+	rowsCount := getRowsCount(tss)
 	globalRowsPushedBeforeRelabel.Add(rowsCount)
 	maxSamplesPerBlock := *maxRowsPerBlock
 	// Allow up to 10x of labels per each block on average.
@@ -505,20 +498,46 @@ func tryPush(at *auth.Token, wr *prompbmarshal.WriteRequest, forceDropSamplesOnF
 			deduplicatorGlobal.Push(tssBlock)
 			tssBlock = tssBlock[:0]
 		}
-		if !tryPushBlockToRemoteStorages(tssBlock, forceDropSamplesOnFailure) {
+		if !tryPushBlockToRemoteStorages(rwctxs, tssBlock, forceDropSamplesOnFailure) {
 			return false
 		}
 	}
 	return true
 }
 
-func pushToRemoteStoragesDropFailed(tss []prompbmarshal.TimeSeries) {
-	if tryPushBlockToRemoteStorages(tss, true) {
+func getEligibleRemoteWriteCtxs(tss []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
+	if !disableOnDiskQueueAny {
+		return rwctxsGlobal, true
+	}
+
+	// This code is applicable if at least a single remote storage has -disableOnDiskQueue
+	rwctxs := make([]*remoteWriteCtx, 0, len(rwctxsGlobal))
+	for _, rwctx := range rwctxsGlobal {
+		if !rwctx.fq.IsWriteBlocked() {
+			rwctxs = append(rwctxs, rwctx)
+		} else {
+			rwctx.pushFailures.Inc()
+			if !forceDropSamplesOnFailure {
+				return nil, false
+			}
+			rowsCount := getRowsCount(tss)
+			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
+		}
+	}
+	return rwctxs, true
+}
+
+func pushToRemoteStoragesTrackDropped(tss []prompbmarshal.TimeSeries) {
+	rwctxs, _ := getEligibleRemoteWriteCtxs(tss, true)
+	if len(rwctxs) == 0 {
 		return
 	}
+	if !tryPushBlockToRemoteStorages(rwctxs, tss, true) {
+		logger.Panicf("BUG: tryPushBlockToRemoteStorages() must return true when forceDropSamplesOnFailure=true")
+	}
 }
 
-func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
+func tryPushBlockToRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, forceDropSamplesOnFailure bool) bool {
 	if len(tssBlock) == 0 {
 		// Nothing to push
 		return true
@@ -537,7 +556,7 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 		if replicas <= 0 {
 			replicas = 1
 		}
-		return tryShardingBlockAmongRemoteStorages(tssBlock, replicas, forceDropSamplesOnFailure)
+		return tryShardingBlockAmongRemoteStorages(rwctxs, tssBlock, replicas, forceDropSamplesOnFailure)
 	}
 
 	// Replicate tssBlock samples among rwctxs.
@@ -558,7 +577,7 @@ func tryPushBlockToRemoteStorages(tssBlock []prompbmarshal.TimeSeries, forceDrop
 	return !anyPushFailed.Load()
 }
 
-func tryShardingBlockAmongRemoteStorages(tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
+func tryShardingBlockAmongRemoteStorages(rwctxs []*remoteWriteCtx, tssBlock []prompbmarshal.TimeSeries, replicas int, forceDropSamplesOnFailure bool) bool {
 	x := getTSSShards(len(rwctxs))
 	defer putTSSShards(x)
 
@@ -745,10 +764,8 @@ type remoteWriteCtx struct {
 	sas          atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
 
-	streamAggrKeepInput   bool
-	streamAggrDropInput   bool
-	disableOnDiskQueue    bool
-	dropSamplesOnOverload bool
+	streamAggrKeepInput bool
+	streamAggrDropInput bool
 
 	pss        []*pendingSeries
 	pssNextIdx atomic.Uint64
@@ -773,6 +790,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
 		maxPendingBytes = persistentqueue.DefaultChunkFileSize
 	}
+
 	isPQDisabled := disableOnDiskQueue.GetOptionalArg(argIdx)
 	fq := persistentqueue.MustOpenFastQueue(queuePath, sanitizedURL, maxInmemoryBlocks, maxPendingBytes, isPQDisabled)
 	_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_pending_data_bytes{path=%q, url=%q}`, queuePath, sanitizedURL), func() float64 {
@@ -817,31 +835,13 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks in
 		c:   c,
 		pss: pss,
 
-		dropSamplesOnOverload: dropSamplesOnOverload.GetOptionalArg(argIdx),
-		disableOnDiskQueue:    isPQDisabled,
+		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 
-		rowsPushedAfterRelabel: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_rows_pushed_after_relabel_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedByRelabel:   metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_relabel_metrics_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-
-		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q, url=%q}`, queuePath, sanitizedURL)),
-	}
-
-	// Initialize sas
-	sasFile, sasOpts := getStreamAggrOpts(argIdx)
-	if sasFile != "" {
-		sas, err := newStreamAggrConfig(argIdx, rwctx.pushInternalTrackDropped)
-		if err != nil {
-			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
-		}
-		rwctx.sas.Store(sas)
-		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(argIdx)
-		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(argIdx)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, sasFile)).Set(1)
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, sasFile)).Set(fasttime.UnixTimestamp())
-	} else if sasOpts.DedupInterval > 0 {
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, sasOpts.DedupInterval, sasOpts.DropInputLabels, sasOpts.Alias)
+		pushFailures:             metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_push_failures_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
+		rowsDroppedOnPushFailure: metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_samples_dropped_total{path=%q,url=%q}`, queuePath, sanitizedURL)),
 	}
+	rwctx.initStreamAggrConfig()
 
 	return rwctx
 }
@@ -934,8 +934,9 @@ func (rwctx *remoteWriteCtx) TryPush(tss []prompbmarshal.TimeSeries, forceDropSa
 
 	// Couldn't push tss to remote storage
 	rwctx.pushFailures.Inc()
-	if forceDropSamplesOnFailure || rwctx.dropSamplesOnOverload {
-		rwctx.rowsDroppedOnPushFailure.Add(len(tss))
+	if forceDropSamplesOnFailure {
+		rowsCount := getRowsCount(tss)
+		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		return true
 	}
 	return false
@@ -962,14 +963,12 @@ func (rwctx *remoteWriteCtx) pushInternalTrackDropped(tss []prompbmarshal.TimeSe
 	if rwctx.tryPushInternal(tss) {
 		return
 	}
-	if !rwctx.disableOnDiskQueue {
+	if !rwctx.fq.IsPersistentQueueDisabled() {
 		logger.Panicf("BUG: tryPushInternal must return true if -remoteWrite.disableOnDiskQueue isn't set")
 	}
 	rwctx.pushFailures.Inc()
-	if dropSamplesOnOverload.GetOptionalArg(rwctx.idx) {
-		rowsCount := getRowsCount(tss)
-		rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
-	}
+	rowsCount := getRowsCount(tss)
+	rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 }
 
 func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) bool {
@@ -1000,7 +999,7 @@ func (rwctx *remoteWriteCtx) tryPushInternal(tss []prompbmarshal.TimeSeries) boo
 }
 
 var tssPool = &sync.Pool{
-	New: func() interface{} {
+	New: func() any {
 		a := []prompbmarshal.TimeSeries{}
 		return &a
 	},

app/vmagent/remotewrite/remotewrite_test.go

@@ -77,14 +77,16 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
 		}
 
-		if len(streamAggrConfig) > 0 {
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), nil, streamaggr.Options{})
+		if streamAggrConfig != "" {
+			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
+			defer sas.MustStop()
 			rwctx.sas.Store(sas)
 		}
 
@@ -94,7 +96,9 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 
 		// copy inputTss to make sure it is not mutated during TryPush call
 		copy(expectedTss, inputTss)
-		rwctx.TryPush(inputTss, false)
+		if !rwctx.TryPush(inputTss, false) {
+			t.Fatalf("cannot push samples to rwctx")
+		}
 
 		if !reflect.DeepEqual(expectedTss, inputTss) {
 			t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", inputTss, expectedTss)

app/vmagent/remotewrite/streamaggr.go

@@ -50,7 +50,7 @@ var (
 	streamAggrIgnoreOldSamples = flagutil.NewArrayBool("remoteWrite.streamAggr.ignoreOldSamples", "Whether to ignore input samples with old timestamps outside the current "+
 		"aggregation interval for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
-	streamAggrIgnoreFirstIntervals = flag.Int("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
+	streamAggrIgnoreFirstIntervals = flagutil.NewArrayInt("remoteWrite.streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start "+
 		"for the corresponding -remoteWrite.streamAggr.config at the corresponding -remoteWrite.url. Increase this value if "+
 		"you observe incorrect aggregation results after vmagent restarts. It could be caused by receiving bufferred delayed data from clients pushing data into the vmagent. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
@@ -61,104 +61,182 @@ var (
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
 func CheckStreamAggrConfigs() error {
-	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
+	// Check global config
+	sas, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		return err
+	}
+	sas.MustStop()
 
-	if _, err := newStreamAggrConfig(-1, pushNoop); err != nil {
-		return fmt.Errorf("could not load -streamAggr.config stream aggregation config: %w", err)
-	}
 	if len(*streamAggrConfig) > len(*remoteWriteURLs) {
-		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d",
-			len(*streamAggrConfig), len(*remoteWriteURLs))
+		return fmt.Errorf("too many -remoteWrite.streamAggr.config args: %d; it mustn't exceed the number of -remoteWrite.url args: %d", len(*streamAggrConfig), len(*remoteWriteURLs))
 	}
+
+	pushNoop := func(_ []prompbmarshal.TimeSeries) {}
 	for idx := range *streamAggrConfig {
-		if _, err := newStreamAggrConfig(idx, pushNoop); err != nil {
+		sas, err := newStreamAggrConfigPerURL(idx, pushNoop)
+		if err != nil {
 			return err
 		}
+		sas.MustStop()
 	}
 	return nil
 }
 
 func reloadStreamAggrConfigs() {
-	reloadStreamAggrConfig(-1, pushToRemoteStoragesDropFailed)
-	for idx, rwctx := range rwctxs {
-		reloadStreamAggrConfig(idx, rwctx.pushInternalTrackDropped)
+	reloadStreamAggrConfigGlobal()
+	for _, rwctx := range rwctxsGlobal {
+		rwctx.reloadStreamAggrConfig()
 	}
 }
 
-func reloadStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) {
-	path, opts := getStreamAggrOpts(idx)
-	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
-	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
-
-	sasNew, err := newStreamAggrConfigWithOpts(pushFunc, path, opts)
-	if err != nil {
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
-		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
-		logger.Errorf("cannot reload stream aggregation config at %q; continue using the previously loaded config; error: %s", path, err)
+func reloadStreamAggrConfigGlobal() {
+	path := *streamAggrGlobalConfig
+	if path == "" {
 		return
 	}
 
-	var sas *streamaggr.Aggregators
-	if idx < 0 {
-		sas = sasGlobal.Load()
-	} else {
-		sas = rwctxs[idx].sas.Load()
+	logger.Infof("reloading stream aggregation configs pointed by -streamAggr.config=%q", path)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
+
+	sasNew, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
+		logger.Errorf("cannot reload -streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		return
 	}
 
+	sas := sasGlobal.Load()
 	if !sasNew.Equal(sas) {
-		var sasOld *streamaggr.Aggregators
-		if idx < 0 {
-			sasOld = sasGlobal.Swap(sasNew)
-		} else {
-			sasOld = rwctxs[idx].sas.Swap(sasNew)
-		}
+		sasOld := sasGlobal.Swap(sasNew)
 		sasOld.MustStop()
-		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
+		logger.Infof("successfully reloaded -streamAggr.config=%q", path)
 	} else {
 		sasNew.MustStop()
-		logger.Infof("successfully reloaded stream aggregation configs at %q", path)
+		logger.Infof("-streamAggr.config=%q wasn't changed since the last reload", path)
 	}
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
 }
 
-func getStreamAggrOpts(idx int) (string, streamaggr.Options) {
-	if idx < 0 {
-		return *streamAggrGlobalConfig, streamaggr.Options{
-			DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
-			DropInputLabels:      *streamAggrGlobalDropInputLabels,
-			IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
-			IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
-			Alias:                "global",
+func initStreamAggrConfigGlobal() {
+	sas, err := newStreamAggrConfigGlobal()
+	if err != nil {
+		logger.Fatalf("cannot initialize gloabl stream aggregators: %s", err)
+	}
+	if sas != nil {
+		filePath := sas.FilePath()
+		sasGlobal.Store(sas)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
+	} else {
+		dedupInterval := streamAggrGlobalDedupInterval.Duration()
+		if dedupInterval > 0 {
+			deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 		}
 	}
-	url := fmt.Sprintf("%d:secret-url", idx+1)
-	if *showRemoteWriteURL {
-		url = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+}
+
+func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
+	idx := rwctx.idx
+
+	sas, err := rwctx.newStreamAggrConfig()
+	if err != nil {
+		logger.Fatalf("cannot initialize stream aggregators: %s", err)
 	}
-	opts := streamaggr.Options{
+	if sas != nil {
+		filePath := sas.FilePath()
+		rwctx.sas.Store(sas)
+		rwctx.streamAggrKeepInput = streamAggrKeepInput.GetOptionalArg(idx)
+		rwctx.streamAggrDropInput = streamAggrDropInput.GetOptionalArg(idx)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, filePath)).Set(1)
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, filePath)).Set(fasttime.UnixTimestamp())
+	} else {
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
+		if dedupInterval > 0 {
+			alias := fmt.Sprintf("dedup-%d", idx+1)
+			rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, *streamAggrDropInputLabels, alias)
+		}
+	}
+}
+
+func (rwctx *remoteWriteCtx) reloadStreamAggrConfig() {
+	path := streamAggrConfig.GetOptionalArg(rwctx.idx)
+	if path == "" {
+		return
+	}
+
+	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", path)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, path)).Inc()
+
+	sasNew, err := rwctx.newStreamAggrConfig()
+	if err != nil {
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, path)).Inc()
+		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(0)
+		logger.Errorf("cannot reload -remoteWrite.streamAggr.config=%q; continue using the previously loaded config; error: %s", path, err)
+		return
+	}
+
+	sas := rwctx.sas.Load()
+	if !sasNew.Equal(sas) {
+		sasOld := rwctx.sas.Swap(sasNew)
+		sasOld.MustStop()
+		logger.Infof("successfully reloaded -remoteWrite.streamAggr.config=%q", path)
+	} else {
+		sasNew.MustStop()
+		logger.Infof("-remoteWrite.streamAggr.config=%q wasn't changed since the last reload", path)
+	}
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_successful{path=%q}`, path)).Set(1)
+	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reload_success_timestamp_seconds{path=%q}`, path)).Set(fasttime.UnixTimestamp())
+}
+
+func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
+	path := *streamAggrGlobalConfig
+	if path == "" {
+		return nil, nil
+	}
+
+	opts := &streamaggr.Options{
+		DedupInterval:        streamAggrGlobalDedupInterval.Duration(),
+		DropInputLabels:      *streamAggrGlobalDropInputLabels,
+		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
+		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
+		KeepInput:            *streamAggrGlobalKeepInput,
+	}
+
+	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
+	if err != nil {
+		return nil, fmt.Errorf("cannot load -streamAggr.config=%q: %w", *streamAggrGlobalConfig, err)
+	}
+	return sas, nil
+}
+
+func (rwctx *remoteWriteCtx) newStreamAggrConfig() (*streamaggr.Aggregators, error) {
+	return newStreamAggrConfigPerURL(rwctx.idx, rwctx.pushInternalTrackDropped)
+}
+
+func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
+	path := streamAggrConfig.GetOptionalArg(idx)
+	if path == "" {
+		return nil, nil
+	}
+
+	alias := fmt.Sprintf("%d:secret-url", idx+1)
+	if *showRemoteWriteURL {
+		alias = fmt.Sprintf("%d:%s", idx+1, remoteWriteURLs.GetOptionalArg(idx))
+	}
+	opts := &streamaggr.Options{
 		DedupInterval:        streamAggrDedupInterval.GetOptionalArg(idx),
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
-		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
-		Alias:                url,
+		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
+		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
 	}
 
-	if len(*streamAggrConfig) == 0 {
-		return "", opts
+	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)
+	if err != nil {
+		return nil, fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", path, err)
 	}
-	return streamAggrConfig.GetOptionalArg(idx), opts
-}
-
-func newStreamAggrConfigWithOpts(pushFunc streamaggr.PushFunc, path string, opts streamaggr.Options) (*streamaggr.Aggregators, error) {
-	if len(path) == 0 {
-		// Skip empty stream aggregation config.
-		return nil, nil
-	}
-	return streamaggr.LoadFromFile(path, pushFunc, opts)
-}
-
-func newStreamAggrConfig(idx int, pushFunc streamaggr.PushFunc) (*streamaggr.Aggregators, error) {
-	path, opts := getStreamAggrOpts(idx)
-	return newStreamAggrConfigWithOpts(pushFunc, path, opts)
+	return sas, nil
 }

app/vmalert-tool/Makefile

@@ -81,6 +81,9 @@ vmalert-tool-linux-ppc64le:
 vmalert-tool-linux-s390x:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=s390x $(MAKE) app-local-goos-goarch
 
+vmalert-tool-linux-loong64:
+	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=loong64 $(MAKE) app-local-goos-goarch
+
 vmalert-tool-linux-386:
 	APP_NAME=vmalert-tool CGO_ENABLED=0 GOOS=linux GOARCH=386 $(MAKE) app-local-goos-goarch
 

app/vmalert-tool/main.go

@@ -41,9 +41,19 @@ Examples:
 						Usage:    "disable adding group's Name as label to generated alerts and time series.",
 						Required: false,
 					},
+					&cli.StringSliceFlag{
+						Name:     "external.label",
+						Usage:    `Optional label in the form 'name=value' to add to all generated recording rules and alerts. Supports an array of values separated by comma or specified via multiple flags.`,
+						Required: false,
+					},
+					&cli.StringFlag{
+						Name:     "external.url",
+						Usage:    `Optional external URL to template in rule's labels or annotations.`,
+						Required: false,
+					},
 				},
 				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel")); failed {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url")); failed {
 						return fmt.Errorf("unittest failed")
 					}
 					return nil

app/vmalert-tool/unittest/input.go

@@ -74,10 +74,7 @@ func writeInputSeries(input []series, interval *promutils.Duration, startStamp t
 		r.Timeseries = append(r.Timeseries, testutil.TimeSeries{Labels: ls, Samples: samples})
 	}
 
-	data, err := testutil.Compress(r)
-	if err != nil {
-		return fmt.Errorf("failed to compress data: %v", err)
-	}
+	data := testutil.Compress(r)
 	// write input series to vm
 	httpWrite(dst, bytes.NewBuffer(data))
 	vmstorage.Storage.DebugFlush()

app/vmalert-tool/unittest/input_test.go

@@ -6,88 +6,61 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 )
 
-func TestParseInputValue(t *testing.T) {
-	testCases := []struct {
-		input  string
-		exp    []sequenceValue
-		failed bool
-	}{
-		{
-			"",
-			nil,
-			true,
-		},
-		{
-			"testfailed",
-			nil,
-			true,
-		},
-		// stale doesn't support operations
-		{
-			"stalex3",
-			nil,
-			true,
-		},
-		{
-			"-4",
-			[]sequenceValue{{Value: -4}},
-			false,
-		},
-		{
-			"_",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"stale",
-			[]sequenceValue{{Value: decimal.StaleNaN}},
-			false,
-		},
-		{
-			"-4x1",
-			[]sequenceValue{{Value: -4}, {Value: -4}},
-			false,
-		},
-		{
-			"_x1",
-			[]sequenceValue{{Omitted: true}},
-			false,
-		},
-		{
-			"1+1x2 0.1 0.1+0.3x2 3.14",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}},
-			false,
-		},
-		{
-			"2-1x4",
-			[]sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}},
-			false,
-		},
-		{
-			"1+1x1 _ -4 stale 3+20x1",
-			[]sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}},
-			false,
-		},
+func TestParseInputValue_Failure(t *testing.T) {
+	f := func(input string) {
+		t.Helper()
+
+		_, err := parseInputValue(input, true)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
 	}
 
-	for _, tc := range testCases {
-		output, err := parseInputValue(tc.input, true)
-		if err != nil != tc.failed {
-			t.Fatalf("failed to parse %s, expect %t, got %t", tc.input, tc.failed, err != nil)
+	f("")
+	f("testfailed")
+
+	// stale doesn't support operations
+	f("stalex3")
+}
+
+func TestParseInputValue_Success(t *testing.T) {
+	f := func(input string, outputExpected []sequenceValue) {
+		t.Helper()
+
+		output, err := parseInputValue(input, true)
+		if err != nil {
+			t.Fatalf("unexpected error in parseInputValue: %s", err)
 		}
-		if len(tc.exp) != len(output) {
-			t.Fatalf("expect %v, got %v", tc.exp, output)
+
+		if len(outputExpected) != len(output) {
+			t.Fatalf("unexpected output length; got %d; want %d", len(outputExpected), len(output))
 		}
-		for i := 0; i < len(tc.exp); i++ {
-			if tc.exp[i].Omitted != output[i].Omitted {
-				t.Fatalf("expect %v, got %v", tc.exp, output)
+		for i := 0; i < len(outputExpected); i++ {
+			if outputExpected[i].Omitted != output[i].Omitted {
+				t.Fatalf("unexpected Omitted field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
-			if tc.exp[i].Value != output[i].Value {
-				if decimal.IsStaleNaN(tc.exp[i].Value) && decimal.IsStaleNaN(output[i].Value) {
+			if outputExpected[i].Value != output[i].Value {
+				if decimal.IsStaleNaN(outputExpected[i].Value) && decimal.IsStaleNaN(output[i].Value) {
 					continue
 				}
-				t.Fatalf("expect %v, got %v", tc.exp, output)
+				t.Fatalf("unexpeccted Value field in the output\ngot\n%v\nwant\n%v", output, outputExpected)
 			}
 		}
 	}
+
+	f("-4", []sequenceValue{{Value: -4}})
+
+	f("_", []sequenceValue{{Omitted: true}})
+
+	f("stale", []sequenceValue{{Value: decimal.StaleNaN}})
+
+	f("-4x1", []sequenceValue{{Value: -4}, {Value: -4}})
+
+	f("_x1", []sequenceValue{{Omitted: true}})
+
+	f("1+1x2 0.1 0.1+0.3x2 3.14", []sequenceValue{{Value: 1}, {Value: 2}, {Value: 3}, {Value: 0.1}, {Value: 0.1}, {Value: 0.4}, {Value: 0.7}, {Value: 3.14}})
+
+	f("2-1x4", []sequenceValue{{Value: 2}, {Value: 1}, {Value: 0}, {Value: -1}, {Value: -2}})
+
+	f("1+1x1 _ -4 stale 3+20x1", []sequenceValue{{Value: 1}, {Value: 2}, {Omitted: true}, {Value: -4}, {Value: decimal.StaleNaN}, {Value: 3}, {Value: 23}})
 }

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -24,20 +24,16 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
-              datacenter: dc-123
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
+              description: "localhost:9090 of job vmagent2 in cluster  has been down for more than 5 minutes."
+              dashboard: "/d/dashboard?orgId=1"
 
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - exp_labels:
-              datacenter: dc-123
+          - {}
 
       - eval_time: 0
         alertname: InstanceDown
         exp_alerts: []
-
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -8,7 +8,8 @@ groups:
           severity: page
         annotations:
           summary: "Instance {{ $labels.instance }} down"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+          description: "{{ $labels.instance }} of job {{ $labels.job }} in cluster {{ $externalLabels.cluster }} has been down for more than 5 minutes."
+          dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
       - alert: AlwaysFiring
         expr: 1
       - alert: SameAlertNameWithDifferentGroup

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -16,7 +16,8 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - {}
+          - exp_labels:
+              cluster: prod
       - eval_time: 1m
         groupname: group2
         alertname: SameAlertNameWithDifferentGroup
@@ -25,7 +26,8 @@ tests:
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
         exp_alerts:
-          - {}
+          - exp_labels:
+              cluster: prod
       - eval_time: 6m
         groupname: group1
         alertname: SameAlertNameWithDifferentGroup
@@ -61,18 +63,18 @@ tests:
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t1", datacenter="dc-123"}'
+            labels: '{__name__="t1", cluster="prod"}'
       - expr: t2
         eval_time: 4m
         exp_samples:
           - value: 4
-            labels: '{__name__="t2", datacenter="dc-123"}'
+            labels: '{__name__="t2", cluster="prod"}'
       - expr: t3
         eval_time: 4m
         exp_samples:
           # t3 is 3 instead of 4 cause it's rules3 is evaluated before rules1
           - value: 3
-            labels: '{__name__="t3", datacenter="dc-123"}'
+            labels: '{__name__="t3", cluster="prod"}'
 
     alert_rule_test:
       - eval_time: 10m
@@ -83,22 +85,21 @@ tests:
               job: vmagent1
               severity: page
               instance: localhost:9090
-              datacenter: dc-123
+              cluster: prod
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent1 has been down for more than 5 minutes."
+              description: "localhost:9090 of job vmagent1 in cluster prod has been down for more than 5 minutes."
+              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              datacenter: dc-123
+              cluster: prod
 
       - eval_time: 0
         groupname: alerts
         alertname: InstanceDown
         exp_alerts: []
 
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -13,7 +13,7 @@ tests:
       - expr: suquery_interval_test
         eval_time: 4m
         exp_samples:
-          - labels: '{__name__="suquery_interval_test",datacenter="dc-123", instance="localhost:9090", job="vmagent2"}'
+          - labels: '{__name__="suquery_interval_test", cluster="prod", instance="localhost:9090", job="vmagent2"}'
             value: 1
 
     alert_rule_test:
@@ -25,22 +25,21 @@ tests:
               job: vmagent2
               severity: page
               instance: localhost:9090
-              datacenter: dc-123
+              cluster: prod
             exp_annotations:
               summary: "Instance localhost:9090 down"
-              description: "localhost:9090 of job vmagent2 has been down for more than 5 minutes."
+              description: "localhost:9090 of job vmagent2 in cluster prod has been down for more than 5 minutes."
+              dashboard: "http://grafana:3000/d/dashboard?orgId=1"
 
       - eval_time: 0
         groupname: group1
         alertname: AlwaysFiring
         exp_alerts:
           - exp_labels:
-              datacenter: dc-123
+              cluster: prod
 
       - eval_time: 0
         groupname: group1
         alertname: InstanceDown
         exp_alerts: []
 
-    external_labels:
-      datacenter: dc-123

app/vmalert-tool/unittest/unittest.go

@@ -55,7 +55,7 @@ const (
 )
 
 // UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool) bool {
+func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL string) bool {
 	if err := templates.Load([]string{}, true); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
@@ -71,14 +71,34 @@ func UnitTest(files []string, disableGroupLabel bool) bool {
 
 	testfiles, err := config.ReadFromFS(files)
 	if err != nil {
-		fmt.Println("  FAILED")
-		fmt.Printf("\nfailed to read test files: \n%v", err)
+		logger.Fatalf("failed to load test files %q: %v", files, err)
 	}
+	if len(testfiles) == 0 {
+		fmt.Println("no test file found")
+		return false
+	}
+
+	labels := make(map[string]string)
+	for _, s := range externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+	_, err = notifier.Init(nil, labels, externalURL)
+	if err != nil {
+		logger.Fatalf("failed to init notifier: %v", err)
+	}
+
 	var failed bool
 	for fileName, file := range testfiles {
-		if err := ruleUnitTest(fileName, file); err != nil {
+		if err := ruleUnitTest(fileName, file, labels); err != nil {
 			fmt.Println("  FAILED")
-			fmt.Printf("\nfailed to run unit test for file %q: \n%v", file, err)
+			fmt.Printf("\nfailed to run unit test for file %q: \n%v", fileName, err)
 			failed = true
 		} else {
 			fmt.Println("  SUCCESS")
@@ -88,7 +108,7 @@ func UnitTest(files []string, disableGroupLabel bool) bool {
 	return failed
 }
 
-func ruleUnitTest(filename string, content []byte) []error {
+func ruleUnitTest(filename string, content []byte, externalLabels map[string]string) []error {
 	fmt.Println("\nUnit Testing: ", filename)
 	var unitTestInp unitTestFile
 	if err := yaml.UnmarshalStrict(content, &unitTestInp); err != nil {
@@ -126,7 +146,7 @@ func ruleUnitTest(filename string, content []byte) []error {
 			errs = append(errs, err)
 			continue
 		}
-		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups)
+		testErrs := t.test(unitTestInp.EvaluationInterval.Duration(), groupOrderMap, testGroups, externalLabels)
 		errs = append(errs, testErrs...)
 	}
 
@@ -163,6 +183,10 @@ func verifyTestGroup(group testGroup) error {
 			return fmt.Errorf("\n%s    missing required field \"eval_time\"", testGroupName)
 		}
 	}
+	if group.ExternalLabels != nil {
+		fmt.Printf("\n%s    warning: filed `external_labels` will be deprecated soon, please use `-external.label` cmd-line flag instead. "+
+			"Check https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6735 for details.\n", testGroupName)
+	}
 	return nil
 }
 
@@ -179,6 +203,7 @@ func processFlags() {
 		{flag: "retentionPeriod", value: "100y"},
 		{flag: "datasource.url", value: testDataSourcePath},
 		{flag: "remoteWrite.url", value: testRemoteWritePath},
+		{flag: "notifier.blackhole", value: "true"},
 	} {
 		// panics if flag doesn't exist
 		if err := flag.Lookup(fv.flag).Value.Set(fv.value); err != nil {
@@ -239,7 +264,7 @@ func tearDown() {
 	fs.MustRemoveAll(storagePath)
 }
 
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group) (checkErrs []error) {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, testGroups []vmalertconfig.Group, externalLabels map[string]string) (checkErrs []error) {
 	// set up vmstorage and http server for ingest and read queries
 	setUp()
 	// tear down vmstorage and clean the data dir
@@ -288,7 +313,14 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	// create groups with given rule
 	var groups []*rule.Group
 	for _, group := range testGroups {
-		ng := rule.NewGroup(group, q, time.Minute, tg.ExternalLabels)
+		mergedExternalLabels := make(map[string]string)
+		for k, v := range tg.ExternalLabels {
+			mergedExternalLabels[k] = v
+		}
+		for k, v := range externalLabels {
+			mergedExternalLabels[k] = v
+		}
+		ng := rule.NewGroup(group, q, time.Minute, mergedExternalLabels)
 		groups = append(groups, ng)
 	}
 

app/vmalert-tool/unittest/unittest_test.go

@@ -14,34 +14,34 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestUnitRule(t *testing.T) {
-	testCases := []struct {
-		name              string
-		disableGroupLabel bool
-		files             []string
-		failed            bool
-	}{
-		{
-			name:   "run multi files",
-			files:  []string{"./testdata/test1.yaml", "./testdata/test2.yaml"},
-			failed: false,
-		},
-		{
-			name:              "disable group label",
-			disableGroupLabel: true,
-			files:             []string{"./testdata/disable-group-label.yaml"},
-			failed:            false,
-		},
-		{
-			name:   "failing test",
-			files:  []string{"./testdata/failed-test.yaml"},
-			failed: true,
-		},
-	}
-	for _, tc := range testCases {
-		fail := UnitTest(tc.files, tc.disableGroupLabel)
-		if fail != tc.failed {
-			t.Fatalf("failed to test %s, expect %t, got %t", tc.name, tc.failed, fail)
+func TestUnitTest_Failure(t *testing.T) {
+	f := func(files []string) {
+		t.Helper()
+
+		failed := UnitTest(files, false, nil, "")
+		if !failed {
+			t.Fatalf("expecting failed test")
 		}
 	}
+
+	// failing test
+	f([]string{"./testdata/failed-test.yaml"})
+}
+
+func TestUnitTest_Success(t *testing.T) {
+	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
+		t.Helper()
+
+		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL)
+		if failed {
+			t.Fatalf("unexpected failed test")
+		}
+	}
+
+	// run multi files
+	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
+
+	// disable group label
+	// template with null external values
+	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
 }

app/vmalert/config/config.go

@@ -45,11 +45,11 @@ type Group struct {
 	// EvalAlignment will make the timestamp of group query requests be aligned with interval
 	EvalAlignment *bool `yaml:"eval_alignment,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (g *Group) UnmarshalYAML(unmarshal func(any) error) error {
 	type group Group
 	if err := unmarshal((*group)(g)); err != nil {
 		return err
@@ -142,11 +142,11 @@ type Rule struct {
 	UpdateEntriesLimit *int `yaml:"update_entries_limit,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (r *Rule) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (r *Rule) UnmarshalYAML(unmarshal func(any) error) error {
 	type rule Rule
 	if err := unmarshal((*rule)(r)); err != nil {
 		return err
@@ -301,7 +301,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	g := struct {
 		Groups []Group `yaml:"groups"`
 		// Catches all undefined fields and must be empty after parsing.
-		XXX map[string]interface{} `yaml:",inline"`
+		XXX map[string]any `yaml:",inline"`
 	}{}
 	err = yaml.Unmarshal(data, &g)
 	if err != nil {
@@ -310,7 +310,7 @@ func parseConfig(data []byte) ([]Group, error) {
 	return g.Groups, checkOverflow(g.XXX, "config")
 }
 
-func checkOverflow(m map[string]interface{}, ctx string) error {
+func checkOverflow(m map[string]any, ctx string) error {
 	if len(m) > 0 {
 		var keys []string
 		for k := range m {

app/vmalert/config/config_test.go

@@ -23,12 +23,6 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func TestParseGood(t *testing.T) {
-	if _, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing files %s", err)
-	}
-}
-
 func TestParseFromURL(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/bad", func(w http.ResponseWriter, _ *http.Request) {
@@ -55,438 +49,353 @@ groups:
 	defer srv.Close()
 
 	if _, err := Parse([]string{srv.URL + "/good-alert", srv.URL + "/good-rr"}, notifier.ValidateTemplates, true); err != nil {
-		t.Errorf("error parsing URLs %s", err)
+		t.Fatalf("error parsing URLs %s", err)
 	}
 
 	if _, err := Parse([]string{srv.URL + "/bad"}, notifier.ValidateTemplates, true); err == nil {
-		t.Errorf("expected parsing error: %s", err)
+		t.Fatalf("expected parsing error: %s", err)
 	}
 }
 
-func TestParseBad(t *testing.T) {
-	testCases := []struct {
-		path   []string
-		expErr string
-	}{
-		{
-			[]string{"testdata/rules/rules_interval_bad.rules"},
-			"eval_offset should be smaller than interval",
-		},
-		{
-			[]string{"testdata/rules/rules0-bad.rules"},
-			"unexpected token",
-		},
-		{
-			[]string{"testdata/dir/rules0-bad.rules"},
-			"error parsing annotation",
-		},
-		{
-			[]string{"testdata/dir/rules1-bad.rules"},
-			"duplicate in file",
-		},
-		{
-			[]string{"testdata/dir/rules2-bad.rules"},
-			"function \"unknown\" not defined",
-		},
-		{
-			[]string{"testdata/dir/rules3-bad.rules"},
-			"either `record` or `alert` must be set",
-		},
-		{
-			[]string{"testdata/dir/rules4-bad.rules"},
-			"either `record` or `alert` must be set",
-		},
-		{
-			[]string{"testdata/rules/rules1-bad.rules"},
-			"bad graphite expr",
-		},
-		{
-			[]string{"testdata/dir/rules6-bad.rules"},
-			"missing ':' in header",
-		},
-		{
-			[]string{"http://unreachable-url"},
-			"failed to",
-		},
+func TestParse_Success(t *testing.T) {
+	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
+	if err != nil {
+		t.Fatalf("error parsing files %s", err)
 	}
-	for _, tc := range testCases {
-		_, err := Parse(tc.path, notifier.ValidateTemplates, true)
+}
+
+func TestParse_Failure(t *testing.T) {
+	f := func(paths []string, errStrExpected string) {
+		t.Helper()
+
+		_, err := Parse(paths, notifier.ValidateTemplates, true)
 		if err == nil {
-			t.Errorf("expected to get error")
-			return
+			t.Fatalf("expected to get error")
 		}
-		if !strings.Contains(err.Error(), tc.expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
+		if !strings.Contains(err.Error(), errStrExpected) {
+			t.Fatalf("expected err to contain %q; got %q instead", errStrExpected, err)
 		}
 	}
+
+	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
+	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
+	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
+	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
+	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
+	f([]string{"testdata/dir/rules4-bad.rules"}, "either `record` or `alert` must be set")
+	f([]string{"testdata/rules/rules1-bad.rules"}, "bad graphite expr")
+	f([]string{"testdata/dir/rules6-bad.rules"}, "missing ':' in header")
+	f([]string{"http://unreachable-url"}, "failed to")
 }
 
-func TestRule_Validate(t *testing.T) {
+func TestRuleValidate(t *testing.T) {
 	if err := (&Rule{}).Validate(); err == nil {
-		t.Errorf("expected empty name error")
+		t.Fatalf("expected empty name error")
 	}
 	if err := (&Rule{Alert: "alert"}).Validate(); err == nil {
-		t.Errorf("expected empty expr error")
+		t.Fatalf("expected empty expr error")
 	}
 	if err := (&Rule{Alert: "alert", Expr: "test>0"}).Validate(); err != nil {
-		t.Errorf("expected valid rule; got %s", err)
+		t.Fatalf("expected valid rule; got %s", err)
 	}
 }
 
-func TestGroup_Validate(t *testing.T) {
-	testCases := []struct {
-		group               *Group
-		rules               []Rule
-		validateAnnotations bool
-		validateExpressions bool
-		expErr              string
-	}{
-		{
-			group:  &Group{},
-			expErr: "group name must be set",
-		},
-		{
-			group: &Group{
-				Name:     "negative interval",
-				Interval: promutils.NewDuration(-1),
+func TestGroupValidate_Failure(t *testing.T) {
+	f := func(group *Group, validateExpressions bool, errStrExpected string) {
+		t.Helper()
+
+		err := group.Validate(nil, validateExpressions)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		errStr := err.Error()
+		if !strings.Contains(errStr, errStrExpected) {
+			t.Fatalf("missing %q in the returned error %q", errStrExpected, errStr)
+		}
+	}
+
+	f(&Group{}, false, "group name must be set")
+
+	f(&Group{
+		Name:     "negative interval",
+		Interval: promutils.NewDuration(-1),
+	}, false, "interval shouldn't be lower than 0")
+
+	f(&Group{
+		Name:       "wrong eval_offset",
+		Interval:   promutils.NewDuration(time.Minute),
+		EvalOffset: promutils.NewDuration(2 * time.Minute),
+	}, false, "eval_offset should be smaller than interval")
+
+	f(&Group{
+		Name:  "wrong limit",
+		Limit: -1,
+	}, false, "invalid limit")
+
+	f(&Group{
+		Name:        "wrong concurrency",
+		Concurrency: -1,
+	}, false, "invalid concurrency")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
 			},
-			expErr: "interval shouldn't be lower than 0",
-		},
-		{
-			group: &Group{
-				Name:       "wrong eval_offset",
-				Interval:   promutils.NewDuration(time.Minute),
-				EvalOffset: promutils.NewDuration(2 * time.Minute),
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
 			},
-			expErr: "eval_offset should be smaller than interval",
 		},
-		{
-			group: &Group{
-				Name:  "wrong limit",
-				Limit: -1,
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Record: "record", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Record: "record", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{Record: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"summary": "{{ value|query }}",
+			}},
+		},
+	}, false, "duplicate")
+
+	f(&Group{
+		Name: "test graphite prometheus bad expr",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Expr: "sum(up == 0 ) by (host)",
+				For:  promutils.NewDuration(10 * time.Millisecond),
 			},
-			expErr: "invalid limit",
-		},
-		{
-			group: &Group{
-				Name:        "wrong concurrency",
-				Concurrency: -1,
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
 			},
-			expErr: "invalid concurrency",
 		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Record: "record",
-						Expr:   "up | 0",
-					},
+	}, false, "invalid rule")
+
+	f(&Group{
+		Name: "test graphite inherit",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
+				For:  promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Expr: "sum(up == 0 ) by (host)",
+			},
+		},
+	}, false, "either `record` or `alert` must be set")
+
+	// validate expressions
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, true, "invalid expression")
+
+	f(&Group{
+		Name: "test thanos",
+		Type: NewRawType("thanos"),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, true, "unknown datasource type")
+
+	f(&Group{
+		Name: "test graphite",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "some-description",
+			}},
+		},
+	}, true, "bad graphite expr")
+}
+
+func TestGroupValidate_Success(t *testing.T) {
+	f := func(group *Group, validateAnnotations, validateExpressions bool) {
+		t.Helper()
+
+		var validateTplFn ValidateTplFn
+		if validateAnnotations {
+			validateTplFn = notifier.ValidateTemplates
+		}
+		err := group.Validate(validateTplFn, validateExpressions)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	}
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, false, false)
+
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
+				Labels: map[string]string{
+					"summary": "{{ value|query }}",
 				},
 			},
-			expErr: "",
 		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Record: "record",
-						Expr:   "up | 0",
-					},
-				},
-			},
-			expErr:              "invalid expression",
-			validateExpressions: true,
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-						Labels: map[string]string{
-							"summary": "{{ value|query }}",
-						},
-					},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-						Labels: map[string]string{
-							"summary": `
+	}, false, false)
+
+	// validate annotiations
+	f(&Group{
+		Name: "test",
+		Rules: []Rule{
+			{
+				Alert: "alert",
+				Expr:  "up == 1",
+				Labels: map[string]string{
+					"summary": `
 {{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
   {{ . | first | value | humanize1024 }}B
 {{ end }}`,
-						},
-					},
-				},
-			},
-			validateAnnotations: true,
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-					},
-					{
-						Alert: "alert",
-						Expr:  "up == 1",
-					},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Record: "record", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Record: "record", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "duplicate",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test",
-				Rules: []Rule{
-					{Record: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"summary": "{{ value|query }}",
-					}},
-				},
-			},
-			expErr: "",
-		},
-		{
-			group: &Group{
-				Name: "test thanos",
-				Type: NewRawType("thanos"),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "unknown datasource type",
-		},
-		{
-			group: &Group{
-				Name: "test graphite",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "some-description",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "",
-		},
-		{
-			group: &Group{
-				Name: "test prometheus",
-				Type: NewPrometheusType(),
-				Rules: []Rule{
-					{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-						"description": "{{ value|query }}",
-					}},
-				},
-			},
-			validateExpressions: true,
-			expErr:              "",
-		},
-		{
-			group: &Group{
-				Name: "test graphite inherit",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{
-						Expr: "sumSeries(time('foo.bar',10))",
-						For:  promutils.NewDuration(10 * time.Millisecond),
-					},
-					{
-						Expr: "sum(up == 0 ) by (host)",
-					},
 				},
 			},
 		},
-		{
-			group: &Group{
-				Name: "test graphite prometheus bad expr",
-				Type: NewGraphiteType(),
-				Rules: []Rule{
-					{
-						Expr: "sum(up == 0 ) by (host)",
-						For:  promutils.NewDuration(10 * time.Millisecond),
-					},
-					{
-						Expr: "sumSeries(time('foo.bar',10))",
-					},
-				},
-			},
-			expErr: "invalid rule",
-		},
-	}
+	}, true, false)
 
-	for _, tc := range testCases {
-		var validateTplFn ValidateTplFn
-		if tc.validateAnnotations {
-			validateTplFn = notifier.ValidateTemplates
-		}
-		err := tc.group.Validate(validateTplFn, tc.validateExpressions)
-		if err == nil {
-			if tc.expErr != "" {
-				t.Errorf("expected to get err %q; got nil insted", tc.expErr)
-			}
-			continue
-		}
-		if !strings.Contains(err.Error(), tc.expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", tc.expErr, err)
-		}
-	}
+	// validate expressions
+	f(&Group{
+		Name: "test prometheus",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+				"description": "{{ value|query }}",
+			}},
+		},
+	}, false, true)
 }
 
-func TestHashRule(t *testing.T) {
-	testCases := []struct {
-		a, b  Rule
-		equal bool
-	}{
-		{
-			Rule{Record: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1"},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-				"foo": "bar",
-			}},
-			true,
-		},
-		{
-			Rule{Alert: "record", Expr: "up == 1"},
-			Rule{Alert: "record", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute)},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			true,
-		},
-		{
-			Rule{Alert: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 1"},
-			false,
-		},
-		{
-			Rule{Record: "record", Expr: "up == 1"},
-			Rule{Record: "record", Expr: "up == 2"},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-				"foo": "baz",
-			}},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"baz": "foo",
-			}},
-			false,
-		},
-		{
-			Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
-				"foo": "bar",
-				"baz": "foo",
-			}},
-			Rule{Alert: "alert", Expr: "up == 1"},
-			false,
-		},
-	}
-	for i, tc := range testCases {
-		aID, bID := HashRule(tc.a), HashRule(tc.b)
-		if tc.equal != (aID == bID) {
-			t.Fatalf("missmatch for rule %d", i)
+func TestHashRule_NotEqual(t *testing.T) {
+	f := func(a, b Rule) {
+		t.Helper()
+
+		aID, bID := HashRule(a), HashRule(b)
+		if aID == bID {
+			t.Fatalf("rule hashes mustn't be equal; got %d", aID)
 		}
 	}
+
+	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
+
+	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 2"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+		"foo": "baz",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1"})
+}
+
+func TestHashRule_Equal(t *testing.T) {
+	f := func(a, b Rule) {
+		t.Helper()
+
+		aID, bID := HashRule(a), HashRule(b)
+		if aID != bID {
+			t.Fatalf("rule hashes must be equal; got %d and %d", aID, bID)
+		}
+	}
+
+	f(Rule{Record: "record", Expr: "up == 1"}, Rule{Record: "record", Expr: "up == 1"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1"}, Rule{Alert: "alert", Expr: "up == 1"})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}})
+
+	f(Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"foo": "bar",
+		"baz": "foo",
+	}}, Rule{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
+		"baz": "foo",
+		"foo": "bar",
+	}})
+
+	f(Rule{Alert: "record", Expr: "up == 1"}, Rule{Alert: "record", Expr: "up == 1"})
+
+	f(Rule{
+		Alert: "alert", Expr: "up == 1", For: promutils.NewDuration(time.Minute), KeepFiringFor: promutils.NewDuration(time.Minute),
+	}, Rule{Alert: "alert", Expr: "up == 1"})
 }
 
 func TestGroupChecksum(t *testing.T) {

app/vmalert/config/log/logger.go

@@ -29,7 +29,7 @@ func (l *Logger) isDisabled() bool {
 }
 
 // Errorf logs error message.
-func (l *Logger) Errorf(format string, args ...interface{}) {
+func (l *Logger) Errorf(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -37,7 +37,7 @@ func (l *Logger) Errorf(format string, args ...interface{}) {
 }
 
 // Warnf logs warning message.
-func (l *Logger) Warnf(format string, args ...interface{}) {
+func (l *Logger) Warnf(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -45,7 +45,7 @@ func (l *Logger) Warnf(format string, args ...interface{}) {
 }
 
 // Infof logs info message.
-func (l *Logger) Infof(format string, args ...interface{}) {
+func (l *Logger) Infof(format string, args ...any) {
 	if l.isDisabled() {
 		return
 	}
@@ -54,6 +54,6 @@ func (l *Logger) Infof(format string, args ...interface{}) {
 
 // Panicf logs panic message and panics.
 // Panicf can't be suppressed
-func (l *Logger) Panicf(format string, args ...interface{}) {
+func (l *Logger) Panicf(format string, args ...any) {
 	logger.Panicf(format, args...)
 }

app/vmalert/config/log/logger_test.go

@@ -18,14 +18,14 @@ func TestOutput(t *testing.T) {
 
 	mustMatch := func(exp string) {
 		t.Helper()
+
 		if exp == "" {
 			if testOutput.String() != "" {
-				t.Errorf("expected output to be empty; got %q", testOutput.String())
-				return
+				t.Fatalf("expected output to be empty; got %q", testOutput.String())
 			}
 		}
 		if !strings.Contains(testOutput.String(), exp) {
-			t.Errorf("output %q should contain %q", testOutput.String(), exp)
+			t.Fatalf("output %q should contain %q", testOutput.String(), exp)
 		}
 		fmt.Println(testOutput.String())
 		testOutput.Reset()

app/vmalert/config/types.go

@@ -69,7 +69,7 @@ func (t *Type) ValidateExpr(expr string) error {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err
@@ -87,7 +87,7 @@ func (t *Type) UnmarshalYAML(unmarshal func(interface{}) error) error {
 }
 
 // MarshalYAML implements the yaml.Unmarshaler interface.
-func (t Type) MarshalYAML() (interface{}, error) {
+func (t Type) MarshalYAML() (any, error) {
 	return t.Name, nil
 }
 
@@ -98,7 +98,7 @@ type Header struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (h *Header) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (h *Header) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err

app/vmalert/datasource/init.go

@@ -12,11 +12,12 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("datasource.url", "", "Datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect endpoint. Required parameter. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also -remoteRead.disablePathAppend and -datasource.showURL")
 	appendTypePrefix  = flag.Bool("datasource.appendTypePrefix", false, "Whether to add type prefix to -datasource.url based on the query type. Set to true if sending different query types to the vmselect URL.")
 	showDatasourceURL = flag.Bool("datasource.showURL", false, "Whether to avoid stripping sensitive information such as auth headers or passwords from URLs in log messages or UI and exported metrics. "+
@@ -98,9 +99,9 @@ func Init(extraParams url.Values) (QuerierBuilder, error) {
 
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -datasource.url=%q: %w", *addr, err)
 	}
-	tr.DialContext = httputils.GetStatDialFunc("vmalert_datasource")
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_datasource")
 	tr.DisableKeepAlives = *disableKeepAlive
 	tr.MaxIdleConnsPerHost = *maxIdleConnections
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {

app/vmalert/datasource/vm_prom_api.go

@@ -57,20 +57,23 @@ var jsonParserPool fastjson.ParserPool
 //	[{"metric":{"__name__":"up","job":"prometheus"},value": [ 1435781451.781,"1"]},
 //	{"metric":{"__name__":"up","job":"node"},value": [ 1435781451.781,"0"]}]
 func (pi *promInstant) Unmarshal(b []byte) error {
+	var metrics []json.RawMessage
+	// metrics slice could be large, so parsing it with fastjson could consume a lot of memory.
+	// We parse the slice with standard lib to keep mem usage low.
+	// And each metric object will be parsed with fastjson to reduce allocations.
+	if err := json.Unmarshal(b, &metrics); err != nil {
+		return fmt.Errorf("cannot unmarshal metrics: %w", err)
+	}
+
 	p := jsonParserPool.Get()
 	defer jsonParserPool.Put(p)
 
-	v, err := p.ParseBytes(b)
-	if err != nil {
-		return err
-	}
-
-	rows, err := v.Array()
-	if err != nil {
-		return fmt.Errorf("cannot find the top-level array of result objects: %w", err)
-	}
-	pi.ms = make([]Metric, len(rows))
-	for i, row := range rows {
+	pi.ms = make([]Metric, len(metrics))
+	for i, data := range metrics {
+		row, err := p.ParseBytes(data)
+		if err != nil {
+			return fmt.Errorf("cannot parse metric object: %w", err)
+		}
 		metric := row.Get("metric")
 		if metric == nil {
 			return fmt.Errorf("can't find `metric` object in %q", row)
@@ -119,7 +122,7 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 type promRange struct {
 	Result []struct {
 		Labels map[string]string `json:"metric"`
-		TVs    [][2]interface{}  `json:"values"`
+		TVs    [][2]any          `json:"values"`
 	} `json:"result"`
 }
 
@@ -147,7 +150,7 @@ func (r promRange) metrics() ([]Metric, error) {
 	return result, nil
 }
 
-type promScalar [2]interface{}
+type promScalar [2]any
 
 func (r promScalar) metrics() ([]Metric, error) {
 	var m Metric

app/vmalert/datasource/vm_prom_api_timing_test.go

@@ -1,43 +1,24 @@
 package datasource
 
 import (
-	"bytes"
-	"io"
-	"net/http"
 	"os"
 	"testing"
 )
 
-func BenchmarkMetrics(b *testing.B) {
-	payload := []byte(`[{"metric":{"__name__":"vm_rows"},"value":[1583786142,"13763"]},{"metric":{"__name__":"vm_requests", "foo":"bar", "baz": "qux"},"value":[1583786140,"2000"]}]`)
-
-	var pi promInstant
-	if err := pi.Unmarshal(payload); err != nil {
-		b.Fatalf(err.Error())
-	}
-	b.Run("Instant", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			_, _ = pi.metrics()
-		}
-	})
-}
-
-func BenchmarkParsePrometheusResponse(b *testing.B) {
-	req, _ := http.NewRequest("GET", "", nil)
-	resp := &http.Response{StatusCode: http.StatusOK}
+func BenchmarkPromInstantUnmarshal(b *testing.B) {
 	data, err := os.ReadFile("testdata/instant_response.json")
 	if err != nil {
 		b.Fatalf("error while reading file: %s", err)
 	}
-	resp.Body = io.NopCloser(bytes.NewReader(data))
 
-	b.Run("Instant", func(b *testing.B) {
+	// BenchmarkParsePrometheusResponse/Instant_std+fastjson-10                    1760            668959 ns/op          280147 B/op       5781 allocs/op
+	b.Run("Instant std+fastjson", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			_, err := parsePrometheusResponse(req, resp)
+			var pi promInstant
+			err = pi.Unmarshal(data)
 			if err != nil {
 				b.Fatalf("unexpected parse err: %s", err)
 			}
-			resp.Body = io.NopCloser(bytes.NewReader(data))
 		}
 	})
 }

app/vmalert/datasource/vm_test.go

@@ -31,26 +31,26 @@ var (
 func TestVMInstantQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Errorf("expected POST method got %s", r.Method)
+			t.Fatalf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		timeParam := r.URL.Query().Get("time")
 		if timeParam == "" {
-			t.Errorf("expected 'time' in query param, got nil instead")
+			t.Fatalf("expected 'time' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, timeParam); err != nil {
-			t.Errorf("failed to parse 'time' query param %q: %s", timeParam, err)
+			t.Fatalf("failed to parse 'time' query param %q: %s", timeParam, err)
 		}
 		switch c {
 		case 0:
@@ -197,13 +197,13 @@ func TestVMInstantQuery(t *testing.T) {
 func TestVMInstantQueryWithRetry(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		switch c {
 		case 0:
@@ -289,37 +289,37 @@ func metricsEqual(t *testing.T, gotM, expectedM []Metric) {
 func TestVMRangeQuery(t *testing.T) {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc("/api/v1/query_range", func(w http.ResponseWriter, r *http.Request) {
 		c++
 		if r.Method != http.MethodPost {
-			t.Errorf("expected POST method got %s", r.Method)
+			t.Fatalf("expected POST method got %s", r.Method)
 		}
 		if name, pass, _ := r.BasicAuth(); name != basicAuthName || pass != basicAuthPass {
-			t.Errorf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
+			t.Fatalf("expected %s:%s as basic auth got %s:%s", basicAuthName, basicAuthPass, name, pass)
 		}
 		if r.URL.Query().Get("query") != query {
-			t.Errorf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
+			t.Fatalf("expected %s in query param, got %s", query, r.URL.Query().Get("query"))
 		}
 		startTS := r.URL.Query().Get("start")
 		if startTS == "" {
-			t.Errorf("expected 'start' in query param, got nil instead")
+			t.Fatalf("expected 'start' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, startTS); err != nil {
-			t.Errorf("failed to parse 'start' query param: %s", err)
+			t.Fatalf("failed to parse 'start' query param: %s", err)
 		}
 		endTS := r.URL.Query().Get("end")
 		if endTS == "" {
-			t.Errorf("expected 'end' in query param, got nil instead")
+			t.Fatalf("expected 'end' in query param, got nil instead")
 		}
 		if _, err := time.Parse(time.RFC3339, endTS); err != nil {
-			t.Errorf("failed to parse 'end' query param: %s", err)
+			t.Fatalf("failed to parse 'end' query param: %s", err)
 		}
 		step := r.URL.Query().Get("step")
 		if step != "15s" {
-			t.Errorf("expected 'step' query param to be 15s; got %q instead", step)
+			t.Fatalf("expected 'step' query param to be 15s; got %q instead", step)
 		}
 		switch c {
 		case 0:
@@ -370,368 +370,299 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
-	authCfg, err := baCfg.NewConfig(".")
-	if err != nil {
-		t.Fatalf("unexpected: %s", err)
-	}
 	query := "up"
 	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
+
+	f := func(isQueryRange bool, vm *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
+		t.Helper()
+
+		req, err := vm.newRequest(ctx)
+		if err != nil {
+			t.Fatalf("error in newRequest: %s", err)
+		}
+
+		switch vm.dataSourceType {
+		case "", datasourcePrometheus:
+			if isQueryRange {
+				vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
+			} else {
+				vm.setPrometheusInstantReqParams(req, query, timestamp)
+			}
+		case datasourceGraphite:
+			vm.setGraphiteReqParams(req, query)
+		}
+
+		checkFn(t, req)
+	}
+
+	authCfg, err := baCfg.NewConfig(".")
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
 	storage := VMStorage{
 		extraParams: url.Values{"round_digits": {"10"}},
 	}
-	testCases := []struct {
-		name       string
-		queryRange bool
-		vm         *VMStorage
-		checkFn    func(t *testing.T, r *http.Request)
-	}{
-		{
-			"prometheus path",
-			false,
-			&VMStorage{
-				dataSourceType: datasourcePrometheus,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/api/v1/query", r.URL.Path)
-			},
-		},
-		{
-			"prometheus prefix",
-			false,
-			&VMStorage{
-				dataSourceType:   datasourcePrometheus,
-				appendTypePrefix: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
-			},
-		},
-		{
-			"prometheus range path",
-			true,
-			&VMStorage{
-				dataSourceType: datasourcePrometheus,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/api/v1/query_range", r.URL.Path)
-			},
-		},
-		{
-			"prometheus range prefix",
-			true,
-			&VMStorage{
-				dataSourceType:   datasourcePrometheus,
-				appendTypePrefix: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
-			},
-		},
-		{
-			"graphite path",
-			false,
-			&VMStorage{
-				dataSourceType: datasourceGraphite,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, graphitePath, r.URL.Path)
-			},
-		},
-		{
-			"graphite prefix",
-			false,
-			&VMStorage{
-				dataSourceType:   datasourceGraphite,
-				appendTypePrefix: true,
-			},
-			func(t *testing.T, r *http.Request) {
-				checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
-			},
-		},
-		{
-			"default params",
-			false,
-			&VMStorage{},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"default range params",
-			true,
-			&VMStorage{},
-			func(t *testing.T, r *http.Request) {
-				ts := timestamp.Format(time.RFC3339)
-				exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"basic auth",
-			false,
-			&VMStorage{authCfg: authCfg},
-			func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "foo", u)
-				checkEqualString(t, "bar", p)
-			},
-		},
-		{
-			"basic auth range",
-			true,
-			&VMStorage{authCfg: authCfg},
-			func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "foo", u)
-				checkEqualString(t, "bar", p)
-			},
-		},
-		{
-			"evaluation interval",
-			false,
-			&VMStorage{
-				evaluationInterval: 15 * time.Second,
-			},
-			func(t *testing.T, r *http.Request) {
-				evalInterval := 15 * time.Second
-				exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"step override",
-			false,
-			&VMStorage{
-				queryStep: time.Minute,
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{
-					"query": {query},
-					"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
-					"time":  {timestamp.Format(time.RFC3339)},
-				}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"step to seconds",
-			false,
-			&VMStorage{
-				evaluationInterval: 3 * time.Hour,
-			},
-			func(t *testing.T, r *http.Request) {
-				evalInterval := 3 * time.Hour
-				exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"prometheus extra params",
-			false,
-			&VMStorage{
-				extraParams: url.Values{"round_digits": {"10"}},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"prometheus extra params range",
-			true,
-			&VMStorage{
-				extraParams: url.Values{
-					"nocache":      {"1"},
-					"max_lookback": {"1h"},
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{
-					"query":        {query},
-					"end":          {timestamp.Format(time.RFC3339)},
-					"start":        {timestamp.Format(time.RFC3339)},
-					"nocache":      {"1"},
-					"max_lookback": {"1h"},
-				}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"custom params overrides the original params",
-			false,
-			storage.Clone().ApplyParams(QuerierParams{
-				QueryParams: url.Values{"round_digits": {"2"}},
-			}),
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"allow duplicates in query params",
-			false,
-			storage.Clone().ApplyParams(QuerierParams{
-				QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
-			}),
-			func(t *testing.T, r *http.Request) {
-				exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
-				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
-			},
-		},
-		{
-			"graphite extra params",
-			false,
-			&VMStorage{
-				dataSourceType: datasourceGraphite,
-				extraParams: url.Values{
-					"nocache":      {"1"},
-					"max_lookback": {"1h"},
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-		{
-			"graphite extra params allows to override from",
-			false,
-			&VMStorage{
-				dataSourceType: datasourceGraphite,
-				extraParams: url.Values{
-					"from": {"-10m"},
-				},
-			},
-			func(t *testing.T, r *http.Request) {
-				exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
-				checkEqualString(t, exp, r.URL.RawQuery)
-			},
-		},
-	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequest(ctx)
-			if err != nil {
-				t.Fatal(err)
-			}
-			switch tc.vm.dataSourceType {
-			case "", datasourcePrometheus:
-				if tc.queryRange {
-					tc.vm.setPrometheusRangeReqParams(req, query, timestamp, timestamp)
-				} else {
-					tc.vm.setPrometheusInstantReqParams(req, query, timestamp)
-				}
-			case datasourceGraphite:
-				tc.vm.setGraphiteReqParams(req, query)
-			}
-			tc.checkFn(t, req)
-		})
-	}
+	// prometheus path
+	f(false, &VMStorage{
+		dataSourceType: datasourcePrometheus,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/api/v1/query", r.URL.Path)
+	})
+
+	// prometheus prefix
+	f(false, &VMStorage{
+		dataSourceType:   datasourcePrometheus,
+		appendTypePrefix: true,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/prometheus/api/v1/query", r.URL.Path)
+	})
+
+	// prometheus range path
+	f(true, &VMStorage{
+		dataSourceType: datasourcePrometheus,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/api/v1/query_range", r.URL.Path)
+	})
+
+	// prometheus range prefix
+	f(true, &VMStorage{
+		dataSourceType:   datasourcePrometheus,
+		appendTypePrefix: true,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, "/prometheus/api/v1/query_range", r.URL.Path)
+	})
+
+	// graphite path
+	f(false, &VMStorage{
+		dataSourceType: datasourceGraphite,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, graphitePath, r.URL.Path)
+	})
+
+	// graphite prefix
+	f(false, &VMStorage{
+		dataSourceType:   datasourceGraphite,
+		appendTypePrefix: true,
+	}, func(t *testing.T, r *http.Request) {
+		checkEqualString(t, graphitePrefix+graphitePath, r.URL.Path)
+	})
+
+	// default params
+	f(false, &VMStorage{}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// default range params
+	f(true, &VMStorage{}, func(t *testing.T, r *http.Request) {
+		ts := timestamp.Format(time.RFC3339)
+		exp := url.Values{"query": {query}, "start": {ts}, "end": {ts}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// basic auth
+	f(false, &VMStorage{
+		authCfg: authCfg,
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "foo", u)
+		checkEqualString(t, "bar", p)
+	})
+
+	// basic auth range
+	f(true, &VMStorage{
+		authCfg: authCfg,
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "foo", u)
+		checkEqualString(t, "bar", p)
+	})
+
+	// evaluation interval
+	f(false, &VMStorage{
+		evaluationInterval: 15 * time.Second,
+	}, func(t *testing.T, r *http.Request) {
+		evalInterval := 15 * time.Second
+		exp := url.Values{"query": {query}, "step": {evalInterval.String()}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// step override
+	f(false, &VMStorage{
+		queryStep: time.Minute,
+	}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{
+			"query": {query},
+			"step":  {fmt.Sprintf("%ds", int(time.Minute.Seconds()))},
+			"time":  {timestamp.Format(time.RFC3339)},
+		}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	//  step to seconds
+	f(false, &VMStorage{
+		evaluationInterval: 3 * time.Hour,
+	}, func(t *testing.T, r *http.Request) {
+		evalInterval := 3 * time.Hour
+		exp := url.Values{"query": {query}, "step": {fmt.Sprintf("%ds", int(evalInterval.Seconds()))}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// prometheus extra params
+	f(false, &VMStorage{
+		extraParams: url.Values{"round_digits": {"10"}},
+	}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "round_digits": {"10"}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// prometheus extra params range
+	f(true, &VMStorage{
+		extraParams: url.Values{
+			"nocache":      {"1"},
+			"max_lookback": {"1h"},
+		},
+	}, func(t *testing.T, r *http.Request) {
+		exp := url.Values{
+			"query":        {query},
+			"end":          {timestamp.Format(time.RFC3339)},
+			"start":        {timestamp.Format(time.RFC3339)},
+			"nocache":      {"1"},
+			"max_lookback": {"1h"},
+		}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// custom params overrides the original params
+	f(false, storage.Clone().ApplyParams(QuerierParams{
+		QueryParams: url.Values{"round_digits": {"2"}},
+	}), func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "round_digits": {"2"}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// allow duplicates in query params
+	f(false, storage.Clone().ApplyParams(QuerierParams{
+		QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
+	}), func(t *testing.T, r *http.Request) {
+		exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
+		checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+	})
+
+	// graphite extra params
+	f(false, &VMStorage{
+		dataSourceType: datasourceGraphite,
+		extraParams: url.Values{
+			"nocache":      {"1"},
+			"max_lookback": {"1h"},
+		},
+	}, func(t *testing.T, r *http.Request) {
+		exp := fmt.Sprintf("format=json&from=-5min&max_lookback=1h&nocache=1&target=%s&until=now", query)
+		checkEqualString(t, exp, r.URL.RawQuery)
+	})
+
+	// graphite extra params allows to override from
+	f(false, &VMStorage{
+		dataSourceType: datasourceGraphite,
+		extraParams: url.Values{
+			"from": {"-10m"},
+		},
+	}, func(t *testing.T, r *http.Request) {
+		exp := fmt.Sprintf("format=json&from=-10m&target=%s&until=now", query)
+		checkEqualString(t, exp, r.URL.RawQuery)
+	})
 }
 
 func TestHeaders(t *testing.T) {
-	testCases := []struct {
-		name    string
-		vmFn    func() *VMStorage
-		checkFn func(t *testing.T, r *http.Request)
-	}{
-		{
-			name: "basic auth",
-			vmFn: func() *VMStorage {
-				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-				if err != nil {
-					t.Errorf("Error get auth config: %s", err)
-				}
-				return &VMStorage{authCfg: cfg}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "foo", u)
-				checkEqualString(t, "bar", p)
-			},
-		},
-		{
-			name: "bearer auth",
-			vmFn: func() *VMStorage {
-				cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
-				if err != nil {
-					t.Errorf("Error get auth config: %s", err)
-				}
-				return &VMStorage{authCfg: cfg}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				reqToken := r.Header.Get("Authorization")
-				splitToken := strings.Split(reqToken, "Bearer ")
-				if len(splitToken) != 2 {
-					t.Errorf("expected two items got %d", len(splitToken))
-				}
-				token := splitToken[1]
-				checkEqualString(t, "foo", token)
-			},
-		},
-		{
-			name: "custom extraHeaders",
-			vmFn: func() *VMStorage {
-				return &VMStorage{extraHeaders: []keyValue{
-					{key: "Foo", value: "bar"},
-					{key: "Baz", value: "qux"},
-				}}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				h1 := r.Header.Get("Foo")
-				checkEqualString(t, "bar", h1)
-				h2 := r.Header.Get("Baz")
-				checkEqualString(t, "qux", h2)
-			},
-		},
-		{
-			name: "custom header overrides basic auth",
-			vmFn: func() *VMStorage {
-				cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
-				if err != nil {
-					t.Errorf("Error get auth config: %s", err)
-				}
-				return &VMStorage{
-					authCfg: cfg,
-					extraHeaders: []keyValue{
-						{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
-					},
-				}
-			},
-			checkFn: func(t *testing.T, r *http.Request) {
-				u, p, _ := r.BasicAuth()
-				checkEqualString(t, "Aladdin", u)
-				checkEqualString(t, "open sesame", p)
-			},
-		},
-	}
-	for _, tt := range testCases {
-		t.Run(tt.name, func(t *testing.T) {
-			vm := tt.vmFn()
-			req, err := vm.newQueryRequest(ctx, "foo", time.Now())
-			if err != nil {
-				t.Fatal(err)
-			}
-			tt.checkFn(t, req)
-		})
+	f := func(vmFn func() *VMStorage, checkFn func(t *testing.T, r *http.Request)) {
+		t.Helper()
+
+		vm := vmFn()
+		req, err := vm.newQueryRequest(ctx, "foo", time.Now())
+		if err != nil {
+			t.Fatalf("error in newQueryRequest: %s", err)
+		}
+		checkFn(t, req)
 	}
+
+	// basic auth
+	f(func() *VMStorage {
+		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+		if err != nil {
+			t.Fatalf("Error get auth config: %s", err)
+		}
+		return &VMStorage{authCfg: cfg}
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "foo", u)
+		checkEqualString(t, "bar", p)
+	})
+
+	// bearer auth
+	f(func() *VMStorage {
+		cfg, err := utils.AuthConfig(utils.WithBearer("foo", ""))
+		if err != nil {
+			t.Fatalf("Error get auth config: %s", err)
+		}
+		return &VMStorage{authCfg: cfg}
+	}, func(t *testing.T, r *http.Request) {
+		reqToken := r.Header.Get("Authorization")
+		splitToken := strings.Split(reqToken, "Bearer ")
+		if len(splitToken) != 2 {
+			t.Fatalf("expected two items got %d", len(splitToken))
+		}
+		token := splitToken[1]
+		checkEqualString(t, "foo", token)
+	})
+
+	// custom extraHeaders
+	f(func() *VMStorage {
+		return &VMStorage{extraHeaders: []keyValue{
+			{key: "Foo", value: "bar"},
+			{key: "Baz", value: "qux"},
+		}}
+	}, func(t *testing.T, r *http.Request) {
+		h1 := r.Header.Get("Foo")
+		checkEqualString(t, "bar", h1)
+		h2 := r.Header.Get("Baz")
+		checkEqualString(t, "qux", h2)
+	})
+
+	// custom header overrides basic auth
+	f(func() *VMStorage {
+		cfg, err := utils.AuthConfig(utils.WithBasicAuth("foo", "bar", ""))
+		if err != nil {
+			t.Fatalf("Error get auth config: %s", err)
+		}
+		return &VMStorage{
+			authCfg: cfg,
+			extraHeaders: []keyValue{
+				{key: "Authorization", value: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="},
+			},
+		}
+	}, func(t *testing.T, r *http.Request) {
+		u, p, _ := r.BasicAuth()
+		checkEqualString(t, "Aladdin", u)
+		checkEqualString(t, "open sesame", p)
+	})
 }
 
 func checkEqualString(t *testing.T, exp, got string) {
 	t.Helper()
+
 	if got != exp {
-		t.Errorf("expected to get: \n%q; \ngot: \n%q", exp, got)
+		t.Fatalf("expected to get: \n%q; \ngot: \n%q", exp, got)
 	}
 }
 
 func expectError(t *testing.T, err error, exp string) {
 	t.Helper()
+
 	if err == nil {
-		t.Errorf("expected non-nil error")
+		t.Fatalf("expected non-nil error")
 	}
 	if !strings.Contains(err.Error(), exp) {
-		t.Errorf("expected error %q to contain %q", err, exp)
+		t.Fatalf("expected error %q to contain %q", err, exp)
 	}
 }

app/vmalert/main_test.go

@@ -25,26 +25,26 @@ func TestGetExternalURL(t *testing.T) {
 	invalidURL := "victoriametrics.com/path"
 	_, err := getExternalURL(invalidURL)
 	if err == nil {
-		t.Errorf("expected error, got nil")
+		t.Fatalf("expected error, got nil")
 	}
 
 	expURL := "https://victoriametrics.com/path"
 	u, err := getExternalURL(expURL)
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url: want %q, got %s", expURL, u.String())
+		t.Fatalf("unexpected url: want %q, got %s", expURL, u.String())
 	}
 
 	h, _ := os.Hostname()
 	expURL = fmt.Sprintf("http://%s:8880", h)
 	u, err = getExternalURL("")
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if u.String() != expURL {
-		t.Errorf("unexpected url: want %s, got %s", expURL, u.String())
+		t.Fatalf("unexpected url: want %s, got %s", expURL, u.String())
 	}
 }
 
@@ -53,22 +53,22 @@ func TestGetAlertURLGenerator(t *testing.T) {
 	u, _ := url.Parse("https://victoriametrics.com/path")
 	fn, err := getAlertURLGenerator(u, "", false)
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	exp := fmt.Sprintf("https://victoriametrics.com/path/vmalert/alert?%s=42&%s=2", paramGroupID, paramAlertID)
 	if exp != fn(testAlert) {
-		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 	_, err = getAlertURLGenerator(nil, "foo?{{invalid}}", true)
 	if err == nil {
-		t.Errorf("expected template validation error got nil")
+		t.Fatalf("expected template validation error got nil")
 	}
 	fn, err = getAlertURLGenerator(u, "foo?query={{$value}}&ds={{ $labels.tenant }}", true)
 	if err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 	if exp := "https://victoriametrics.com/path/foo?query=4&ds=baz"; exp != fn(testAlert) {
-		t.Errorf("unexpected url want %s, got %s", exp, fn(testAlert))
+		t.Fatalf("unexpected url want %s, got %s", exp, fn(testAlert))
 	}
 }
 

app/vmalert/manager_test.go

@@ -82,9 +82,8 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	wg.Wait()
 }
 
-// TestManagerUpdate tests sequential configuration
-// updates.
-func TestManagerUpdate(t *testing.T) {
+// TestManagerUpdate tests sequential configuration updates.
+func TestManagerUpdate_Success(t *testing.T) {
 	const defaultEvalInterval = time.Second * 30
 	currentEvalInterval := *evaluationInterval
 	*evaluationInterval = defaultEvalInterval
@@ -120,145 +119,127 @@ func TestManagerUpdate(t *testing.T) {
 		}
 	)
 
-	testCases := []struct {
-		name       string
-		initPath   string
-		updatePath string
-		want       []*rule.Group
-	}{
-		{
-			name:       "update good rules",
-			initPath:   "config/testdata/rules/rules0-good.rules",
-			updatePath: "config/testdata/dir/rules1-good.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/dir/rules1-good.rules",
-					Name:     "duplicatedGroupDiffFiles",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules: []rule.Rule{
-						&rule.AlertingRule{
-							Name:   "VMRows",
-							Expr:   "vm_rows > 0",
-							For:    5 * time.Minute,
-							Labels: map[string]string{"dc": "gcp", "label": "bar"},
-							Annotations: map[string]string{
-								"summary":     "{{ $value }}",
-								"description": "{{$labels}}",
-							},
-						},
-					},
-				},
-			},
-		},
-		{
-			name:       "update good rules from 1 to 2 groups",
-			initPath:   "config/testdata/dir/rules/rules1-good.rules",
-			updatePath: "config/testdata/rules/rules0-good.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Name:     "groupGorSingleAlert",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules:    []rule.Rule{VMRows},
-				},
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     config.NewPrometheusType(),
-					Name:     "TestGroup",
-					Rules: []rule.Rule{
-						Conns,
-						ExampleAlertAlwaysFiring,
-					},
-				},
-			},
-		},
-		{
-			name:       "update with one bad rule file",
-			initPath:   "config/testdata/rules/rules0-good.rules",
-			updatePath: "config/testdata/dir/rules2-bad.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Name:     "groupGorSingleAlert",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules:    []rule.Rule{VMRows},
-				},
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Interval: defaultEvalInterval,
-					Name:     "TestGroup",
-					Type:     config.NewPrometheusType(),
-					Rules: []rule.Rule{
-						Conns,
-						ExampleAlertAlwaysFiring,
-					},
-				},
-			},
-		},
-		{
-			name:       "update empty dir rules from 0 to 2 groups",
-			initPath:   "config/testdata/empty/*",
-			updatePath: "config/testdata/rules/rules0-good.rules",
-			want: []*rule.Group{
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Name:     "groupGorSingleAlert",
-					Type:     config.NewPrometheusType(),
-					Interval: defaultEvalInterval,
-					Rules:    []rule.Rule{VMRows},
-				},
-				{
-					File:     "config/testdata/rules/rules0-good.rules",
-					Interval: defaultEvalInterval,
-					Type:     config.NewPrometheusType(),
-					Name:     "TestGroup",
-					Rules: []rule.Rule{
-						Conns,
-						ExampleAlertAlwaysFiring,
-					},
-				},
-			},
-		},
+	f := func(initPath, updatePath string, groupsExpected []*rule.Group) {
+		t.Helper()
+
+		ctx, cancel := context.WithCancel(context.TODO())
+		m := &manager{
+			groups:         make(map[uint64]*rule.Group),
+			querierBuilder: &datasource.FakeQuerier{},
+			notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
+		}
+
+		cfgInit := loadCfg(t, []string{initPath}, true, true)
+		if err := m.update(ctx, cfgInit, false); err != nil {
+			t.Fatalf("failed to complete initial rules update: %s", err)
+		}
+
+		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
+		if err == nil { // update can fail and that's expected
+			_ = m.update(ctx, cfgUpdate, false)
+		}
+		if len(groupsExpected) != len(m.groups) {
+			t.Fatalf("unexpected number of groups; got %d; want %d", len(m.groups), len(groupsExpected))
+		}
+
+		for _, wantG := range groupsExpected {
+			gotG, ok := m.groups[wantG.ID()]
+			if !ok {
+				t.Fatalf("expected to have group %q", wantG.Name)
+			}
+			compareGroups(t, wantG, gotG)
+		}
+
+		cancel()
+		m.close()
 	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			ctx, cancel := context.WithCancel(context.TODO())
-			m := &manager{
-				groups:         make(map[uint64]*rule.Group),
-				querierBuilder: &datasource.FakeQuerier{},
-				notifiers:      func() []notifier.Notifier { return []notifier.Notifier{&notifier.FakeNotifier{}} },
-			}
 
-			cfgInit := loadCfg(t, []string{tc.initPath}, true, true)
-			if err := m.update(ctx, cfgInit, false); err != nil {
-				t.Fatalf("failed to complete initial rules update: %s", err)
-			}
+	// update good rules
+	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules1-good.rules", []*rule.Group{
+		{
+			File:     "config/testdata/dir/rules1-good.rules",
+			Name:     "duplicatedGroupDiffFiles",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules: []rule.Rule{
+				&rule.AlertingRule{
+					Name:   "VMRows",
+					Expr:   "vm_rows > 0",
+					For:    5 * time.Minute,
+					Labels: map[string]string{"dc": "gcp", "label": "bar"},
+					Annotations: map[string]string{
+						"summary":     "{{ $value }}",
+						"description": "{{$labels}}",
+					},
+				},
+			},
+		},
+	})
 
-			cfgUpdate, err := config.Parse([]string{tc.updatePath}, notifier.ValidateTemplates, true)
-			if err == nil { // update can fail and that's expected
-				_ = m.update(ctx, cfgUpdate, false)
-			}
-			if len(tc.want) != len(m.groups) {
-				t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
-			}
+	// update good rules from 1 to 2 groups
+	f("config/testdata/dir/rules/rules1-good.rules", "config/testdata/rules/rules0-good.rules", []*rule.Group{
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Name:     "groupGorSingleAlert",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules:    []rule.Rule{VMRows},
+		},
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Interval: defaultEvalInterval,
+			Type:     config.NewPrometheusType(),
+			Name:     "TestGroup",
+			Rules: []rule.Rule{
+				Conns,
+				ExampleAlertAlwaysFiring,
+			},
+		},
+	})
 
-			for _, wantG := range tc.want {
-				gotG, ok := m.groups[wantG.ID()]
-				if !ok {
-					t.Fatalf("expected to have group %q", wantG.Name)
-				}
-				compareGroups(t, wantG, gotG)
-			}
+	// update with one bad rule file
+	f("config/testdata/rules/rules0-good.rules", "config/testdata/dir/rules2-bad.rules", []*rule.Group{
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Name:     "groupGorSingleAlert",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules:    []rule.Rule{VMRows},
+		},
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Interval: defaultEvalInterval,
+			Name:     "TestGroup",
+			Type:     config.NewPrometheusType(),
+			Rules: []rule.Rule{
+				Conns,
+				ExampleAlertAlwaysFiring,
+			},
+		},
+	})
 
-			cancel()
-			m.close()
-		})
-	}
+	// update empty dir rules from 0 to 2 groups
+	f("config/testdata/empty/*", "config/testdata/rules/rules0-good.rules", []*rule.Group{
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Name:     "groupGorSingleAlert",
+			Type:     config.NewPrometheusType(),
+			Interval: defaultEvalInterval,
+			Rules:    []rule.Rule{VMRows},
+		},
+		{
+			File:     "config/testdata/rules/rules0-good.rules",
+			Interval: defaultEvalInterval,
+			Type:     config.NewPrometheusType(),
+			Name:     "TestGroup",
+			Rules: []rule.Rule{
+				Conns,
+				ExampleAlertAlwaysFiring,
+			},
+		},
+	})
 }
+
 func compareGroups(t *testing.T, a, b *rule.Group) {
 	t.Helper()
 	if a.Name != b.Name {
@@ -285,82 +266,59 @@ func compareGroups(t *testing.T, a, b *rule.Group) {
 	}
 }
 
-func TestManagerUpdateNegative(t *testing.T) {
-	testCases := []struct {
-		notifiers []notifier.Notifier
-		rw        remotewrite.RWClient
-		cfg       config.Group
-		expErr    string
-	}{
-		{
-			nil,
-			nil,
-			config.Group{
-				Name: "Recording rule only",
-				Rules: []config.Rule{
-					{Record: "record", Expr: "max(up)"},
-				},
-			},
-			"contains recording rules",
-		},
-		{
-			nil,
-			nil,
-			config.Group{
-				Name: "Alerting rule only",
-				Rules: []config.Rule{
-					{Alert: "alert", Expr: "up > 0"},
-				},
-			},
-			"contains alerting rules",
-		},
-		{
-			[]notifier.Notifier{&notifier.FakeNotifier{}},
-			nil,
-			config.Group{
-				Name: "Recording and alerting rules",
-				Rules: []config.Rule{
-					{Alert: "alert1", Expr: "up > 0"},
-					{Alert: "alert2", Expr: "up > 0"},
-					{Record: "record", Expr: "max(up)"},
-				},
-			},
-			"contains recording rules",
-		},
-		{
-			nil,
-			&remotewrite.Client{},
-			config.Group{
-				Name: "Recording and alerting rules",
-				Rules: []config.Rule{
-					{Record: "record1", Expr: "max(up)"},
-					{Record: "record2", Expr: "max(up)"},
-					{Alert: "alert", Expr: "up > 0"},
-				},
-			},
-			"contains alerting rules",
-		},
+func TestManagerUpdate_Failure(t *testing.T) {
+	f := func(notifiers []notifier.Notifier, rw remotewrite.RWClient, cfg config.Group, errStrExpected string) {
+		t.Helper()
+
+		m := &manager{
+			groups:         make(map[uint64]*rule.Group),
+			querierBuilder: &datasource.FakeQuerier{},
+			rw:             rw,
+		}
+		if notifiers != nil {
+			m.notifiers = func() []notifier.Notifier { return notifiers }
+		}
+		err := m.update(context.Background(), []config.Group{cfg}, false)
+		if err == nil {
+			t.Fatalf("expected to get error; got nil")
+		}
+		errStr := err.Error()
+		if !strings.Contains(errStr, errStrExpected) {
+			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
+		}
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.cfg.Name, func(t *testing.T) {
-			m := &manager{
-				groups:         make(map[uint64]*rule.Group),
-				querierBuilder: &datasource.FakeQuerier{},
-				rw:             tc.rw,
-			}
-			if tc.notifiers != nil {
-				m.notifiers = func() []notifier.Notifier { return tc.notifiers }
-			}
-			err := m.update(context.Background(), []config.Group{tc.cfg}, false)
-			if err == nil {
-				t.Fatalf("expected to get error; got nil")
-			}
-			if !strings.Contains(err.Error(), tc.expErr) {
-				t.Fatalf("expected err to contain %q; got %q", tc.expErr, err)
-			}
-		})
-	}
+	f(nil, nil, config.Group{
+		Name: "Recording rule only",
+		Rules: []config.Rule{
+			{Record: "record", Expr: "max(up)"},
+		},
+	}, "contains recording rules")
+
+	f(nil, nil, config.Group{
+		Name: "Alerting rule only",
+		Rules: []config.Rule{
+			{Alert: "alert", Expr: "up > 0"},
+		},
+	}, "contains alerting rules")
+
+	f([]notifier.Notifier{&notifier.FakeNotifier{}}, nil, config.Group{
+		Name: "Recording and alerting rules",
+		Rules: []config.Rule{
+			{Alert: "alert1", Expr: "up > 0"},
+			{Alert: "alert2", Expr: "up > 0"},
+			{Record: "record", Expr: "max(up)"},
+		},
+	}, "contains recording rules")
+
+	f(nil, &remotewrite.Client{}, config.Group{
+		Name: "Recording and alerting rules",
+		Rules: []config.Rule{
+			{Record: "record1", Expr: "max(up)"},
+			{Record: "record2", Expr: "max(up)"},
+			{Alert: "alert", Expr: "up > 0"},
+		},
+	}, "contains alerting rules")
 }
 
 func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressions bool) []config.Group {

app/vmalert/notifier/alert_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-func TestAlert_ExecTemplate(t *testing.T) {
+func TestAlertExecTemplate(t *testing.T) {
 	extLabels := make(map[string]string)
 	const (
 		extCluster = "prod"
@@ -23,201 +23,164 @@ func TestAlert_ExecTemplate(t *testing.T) {
 	_, err := Init(nil, extLabels, extURL)
 	checkErr(t, err)
 
-	testCases := []struct {
-		name        string
-		alert       *Alert
-		annotations map[string]string
-		expTpl      map[string]string
-	}{
-		{
-			name:        "empty-alert",
-			alert:       &Alert{},
-			annotations: map[string]string{},
-			expTpl:      map[string]string{},
-		},
-		{
-			name: "no-template",
-			alert: &Alert{
-				Value: 1e4,
-				Labels: map[string]string{
-					"instance": "localhost",
+	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
+		t.Helper()
+
+		if err := ValidateTemplates(annotations); err != nil {
+			t.Fatalf("cannot validate annotations: %s", err)
+		}
+
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return []datasource.Metric{
+				{
+					Labels: []datasource.Label{
+						{Name: "foo", Value: "bar"},
+						{Name: "baz", Value: "qux"},
+					},
+					Values:     []float64{1},
+					Timestamps: []int64{1},
 				},
-			},
-			annotations: map[string]string{},
-			expTpl:      map[string]string{},
-		},
-		{
-			name: "label-template",
-			alert: &Alert{
-				Value: 1e4,
-				Labels: map[string]string{
-					"job":      "staging",
-					"instance": "localhost",
+				{
+					Labels: []datasource.Label{
+						{Name: "foo", Value: "garply"},
+						{Name: "baz", Value: "fred"},
+					},
+					Values:     []float64{2},
+					Timestamps: []int64{1},
 				},
-				For: 5 * time.Minute,
-			},
-			annotations: map[string]string{
-				"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-				"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-			},
-			expTpl: map[string]string{
-				"summary":     "Too high connection number for localhost for job staging",
-				"description": "It is 10000 connections for localhost for more than 5m0s",
-			},
-		},
-		{
-			name: "expression-template",
-			alert: &Alert{
-				Expr: `vm_rows{"label"="bar"}<0`,
-			},
-			annotations: map[string]string{
-				"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-				"exprEscapedPath":   "{{ $expr|pathEscape }}",
-				"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-				"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-				"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-			},
-			expTpl: map[string]string{
-				"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-				"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-				"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-				"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-				"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-			},
-		},
-		{
-			name:  "query",
-			alert: &Alert{Expr: `vm_rows{"label"="bar"}>0`},
-			annotations: map[string]string{
-				"summary": `{{ query "foo" | first | value }}`,
-				"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-			},
-			expTpl: map[string]string{
-				"summary": "1",
-				"desc":    "bar 1;garply 2;",
-			},
-		},
-		{
-			name: "external",
-			alert: &Alert{
-				Value: 1e4,
-				Labels: map[string]string{
-					"job":      "staging",
-					"instance": "localhost",
-				},
-			},
-			annotations: map[string]string{
-				"url":         "{{ $externalURL }}",
-				"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-				"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-			},
-			expTpl: map[string]string{
-				"url":         extURL,
-				"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-				"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-			},
-		},
-		{
-			name: "alert and group IDs",
-			alert: &Alert{
-				ID:      42,
-				GroupID: 24,
-			},
-			annotations: map[string]string{
-				"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-			},
-			expTpl: map[string]string{
-				"url": "/api/v1/alert?alertID=42&groupID=24",
-			},
-		},
-		{
-			name: "ActiveAt time",
-			alert: &Alert{
-				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-			},
-			annotations: map[string]string{
-				"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-			},
-			expTpl: map[string]string{
-				"diagram": "![](http://example.com?render=1660941298",
-			},
-		},
-		{
-			name:  "ActiveAt time is nil",
-			alert: &Alert{},
-			annotations: map[string]string{
-				"default_time": "{{$activeAt}}",
-			},
-			expTpl: map[string]string{
-				"default_time": "0001-01-01 00:00:00 +0000 UTC",
-			},
-		},
-		{
-			name: "ActiveAt custom format",
-			alert: &Alert{
-				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-			},
-			annotations: map[string]string{
-				"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-			},
-			expTpl: map[string]string{
-				"fire_time": "2022/08/19 20:34:58",
-			},
-		},
-		{
-			name: "ActiveAt query range",
-			alert: &Alert{
-				ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-			},
-			annotations: map[string]string{
-				"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-			},
-			expTpl: map[string]string{
-				"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-			},
-		},
+			}, nil
+		}
+
+		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
+		if err != nil {
+			t.Fatalf("cannot execute template: %s", err)
+		}
+		if len(tpl) != len(tplExpected) {
+			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
+		}
+		for k := range tplExpected {
+			got, exp := tpl[k], tplExpected[k]
+			if got != exp {
+				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
+			}
+		}
 	}
 
-	qFn := func(_ string) ([]datasource.Metric, error) {
-		return []datasource.Metric{
-			{
-				Labels: []datasource.Label{
-					{Name: "foo", Value: "bar"},
-					{Name: "baz", Value: "qux"},
-				},
-				Values:     []float64{1},
-				Timestamps: []int64{1},
-			},
-			{
-				Labels: []datasource.Label{
-					{Name: "foo", Value: "garply"},
-					{Name: "baz", Value: "fred"},
-				},
-				Values:     []float64{2},
-				Timestamps: []int64{1},
-			},
-		}, nil
-	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			if err := ValidateTemplates(tc.annotations); err != nil {
-				t.Fatal(err)
-			}
-			tpl, err := tc.alert.ExecTemplate(qFn, tc.alert.Labels, tc.annotations)
-			if err != nil {
-				t.Fatal(err)
-			}
-			if len(tpl) != len(tc.expTpl) {
-				t.Fatalf("expected %d elements; got %d", len(tc.expTpl), len(tpl))
-			}
-			for k := range tc.expTpl {
-				got, exp := tpl[k], tc.expTpl[k]
-				if got != exp {
-					t.Fatalf("expected %q=%q; got %q=%q", k, exp, k, got)
-				}
-			}
-		})
-	}
+	// empty-alert
+	f(&Alert{}, map[string]string{}, map[string]string{})
+
+	// no-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"instance": "localhost",
+		},
+	}, map[string]string{}, map[string]string{})
+
+	// label-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+		For: 5 * time.Minute,
+	}, map[string]string{
+		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+	}, map[string]string{
+		"summary":     "Too high connection number for localhost for job staging",
+		"description": "It is 10000 connections for localhost for more than 5m0s",
+	})
+
+	// expression-template
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}<0`,
+	}, map[string]string{
+		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+		"exprEscapedPath":   "{{ $expr|pathEscape }}",
+		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+	}, map[string]string{
+		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+	})
+
+	// query
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}>0`,
+	}, map[string]string{
+		"summary": `{{ query "foo" | first | value }}`,
+		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+	}, map[string]string{
+		"summary": "1",
+		"desc":    "bar 1;garply 2;",
+	})
+
+	// external
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"url":         "{{ $externalURL }}",
+		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+	}, map[string]string{
+		"url":         extURL,
+		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+	})
+
+	// alert and group IDs
+	f(&Alert{
+		ID:      42,
+		GroupID: 24,
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID=42&groupID=24",
+	})
+
+	// ActiveAt time
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+	}, map[string]string{
+		"diagram": "![](http://example.com?render=1660941298",
+	})
+
+	// ActiveAt time is nil
+	f(&Alert{}, map[string]string{
+		"default_time": "{{$activeAt}}",
+	}, map[string]string{
+		"default_time": "0001-01-01 00:00:00 +0000 UTC",
+	})
+
+	// ActiveAt custom format
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+	}, map[string]string{
+		"fire_time": "2022/08/19 20:34:58",
+	})
+
+	// ActiveAt query range
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+	}, map[string]string{
+		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+	})
 }
 
 func TestAlert_toPromLabels(t *testing.T) {

app/vmalert/notifier/alertmanager.go

@@ -76,9 +76,6 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	for key, value := range headers {
-		req.Header.Set(key, value)
-	}
 
 	if am.timeout > 0 {
 		var cancel context.CancelFunc
@@ -94,6 +91,11 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[st
 			return err
 		}
 	}
+	// external headers have higher priority
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
 	resp, err := am.client.Do(req)
 	if err != nil {
 		return err
@@ -130,7 +132,8 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	}
 	tr, err := httputils.Transport(alertManagerURL, tls.CertFile, tls.KeyFile, tls.CAFile, tls.ServerName, tls.InsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for alertmanager URL=%q: %w", alertManagerURL, err)
+
 	}
 
 	ba := new(promauth.BasicAuthConfig)
@@ -145,7 +148,9 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 	aCfg, err := utils.AuthConfig(
 		utils.WithBasicAuth(ba.Username, ba.Password.String(), ba.PasswordFile),
 		utils.WithBearer(authCfg.BearerToken.String(), authCfg.BearerTokenFile),
-		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams))
+		utils.WithOAuth(oauth.ClientID, oauth.ClientSecret.String(), oauth.ClientSecretFile, oauth.TokenURL, strings.Join(oauth.Scopes, ";"), oauth.EndpointParams),
+		utils.WithHeaders(strings.Join(authCfg.Headers, "^^")),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure auth: %w", err)
 	}

app/vmalert/notifier/alertmanager_test.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -16,10 +17,10 @@ func TestAlertManager_Addr(t *testing.T) {
 	const addr = "http://localhost"
 	am, err := NewAlertManager(addr, nil, promauth.HTTPClientConfig{}, nil, 0)
 	if err != nil {
-		t.Errorf("unexpected error: %s", err)
+		t.Fatalf("unexpected error: %s", err)
 	}
 	if am.Addr() != addr {
-		t.Errorf("expected to have %q; got %q", addr, am.Addr())
+		t.Fatalf("expected to have %q; got %q", addr, am.Addr())
 	}
 }
 
@@ -28,27 +29,29 @@ func TestAlertManager_Send(t *testing.T) {
 	const headerKey, headerValue = "TenantID", "foo"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
-		t.Errorf("should not be called")
+		t.Fatalf("should not be called")
 	})
 	c := -1
 	mux.HandleFunc(alertManagerPath, func(w http.ResponseWriter, r *http.Request) {
 		user, pass, ok := r.BasicAuth()
 		if !ok {
-			t.Errorf("unauthorized request")
+			t.Fatalf("unauthorized request")
 		}
 		if user != baUser || pass != baPass {
-			t.Errorf("wrong creds %q:%q; expected %q:%q",
-				user, pass, baUser, baPass)
+			t.Fatalf("wrong creds %q:%q; expected %q:%q", user, pass, baUser, baPass)
 		}
 		c++
 		if r.Method != http.MethodPost {
-			t.Errorf("expected POST method got %s", r.Method)
+			t.Fatalf("expected POST method got %s", r.Method)
 		}
 		switch c {
 		case 0:
 			conn, _, _ := w.(http.Hijacker).Hijack()
 			_ = conn.Close()
 		case 1:
+			if r.Header.Get(headerKey) != headerValue {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
+			}
 			w.WriteHeader(500)
 		case 2:
 			var a []struct {
@@ -59,25 +62,26 @@ func TestAlertManager_Send(t *testing.T) {
 				GeneratorURL string            `json:"generatorURL"`
 			}
 			if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
-				t.Errorf("can not unmarshal data into alert %s", err)
-				t.FailNow()
+				t.Fatalf("can not unmarshal data into alert %s", err)
 			}
 			if len(a) != 1 {
-				t.Errorf("expected 1 alert in array got %d", len(a))
+				t.Fatalf("expected 1 alert in array got %d", len(a))
 			}
 			if a[0].GeneratorURL != "0/0" {
-				t.Errorf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
+				t.Fatalf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
 			}
 			if a[0].StartsAt.IsZero() {
-				t.Errorf("expected non-zero start time")
+				t.Fatalf("expected non-zero start time")
 			}
 			if a[0].EndAt.IsZero() {
-				t.Errorf("expected non-zero end time")
+				t.Fatalf("expected non-zero end time")
+			}
+			if r.Header.Get(headerKey) != "bar" {
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
 			}
 		case 3:
 			if r.Header.Get(headerKey) != headerValue {
-				t.Errorf("expected header %q to be set to %q; got %q instead",
-					headerKey, headerValue, r.Header.Get(headerKey))
+				t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
 			}
 		}
 	})
@@ -89,18 +93,19 @@ func TestAlertManager_Send(t *testing.T) {
 			Username: baUser,
 			Password: promauth.NewSecret(baPass),
 		},
+		Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
 	}
 	am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
 		return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
 	}, aCfg, nil, 0)
 	if err != nil {
-		t.Errorf("unexpected error: %s", err)
+		t.Fatalf("unexpected error: %s", err)
 	}
 	if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
-		t.Error("expected connection error got nil")
+		t.Fatalf("expected connection error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
-		t.Error("expected wrong http code error got nil")
+		t.Fatalf("expected wrong http code error got nil")
 	}
 	if err := am.Send(context.Background(), []Alert{{
 		GroupID:     0,
@@ -108,13 +113,13 @@ func TestAlertManager_Send(t *testing.T) {
 		Start:       time.Now().UTC(),
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
-	}}, nil); err != nil {
-		t.Errorf("unexpected error %s", err)
+	}}, map[string]string{headerKey: "bar"}); err != nil {
+		t.Fatalf("unexpected error %s", err)
 	}
 	if c != 2 {
-		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
+		t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
 	}
 	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 }

app/vmalert/notifier/config.go

@@ -51,7 +51,7 @@ type Config struct {
 	Checksum string
 
 	// Catches all undefined fields and must be empty after parsing.
-	XXX map[string]interface{} `yaml:",inline"`
+	XXX map[string]any `yaml:",inline"`
 
 	// This is set to the directory from where the config has been loaded.
 	baseDir string
@@ -73,7 +73,7 @@ type StaticConfig struct {
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (cfg *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	type config Config
 	if err := unmarshal((*config)(cfg)); err != nil {
 		return err

app/vmalert/notifier/config_test.go

@@ -5,10 +5,14 @@ import (
 	"testing"
 )
 
-func TestConfigParseGood(t *testing.T) {
+func TestParseConfig_Success(t *testing.T) {
 	f := func(path string) {
+		t.Helper()
+
 		_, err := parseConfig(path)
-		checkErr(t, err)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
 	}
 	f("testdata/mixed.good.yaml")
 	f("testdata/consul.good.yaml")
@@ -16,14 +20,16 @@ func TestConfigParseGood(t *testing.T) {
 	f("testdata/static.good.yaml")
 }
 
-func TestConfigParseBad(t *testing.T) {
+func TestParseConfig_Failure(t *testing.T) {
 	f := func(path, expErr string) {
+		t.Helper()
+
 		_, err := parseConfig(path)
 		if err == nil {
 			t.Fatalf("expected to get non-nil err for config %q", path)
 		}
 		if !strings.Contains(err.Error(), expErr) {
-			t.Errorf("expected err to contain %q; got %q instead", expErr, err)
+			t.Fatalf("expected err to contain %q; got %q instead", expErr, err)
 		}
 	}
 

app/vmalert/notifier/config_watcher_test.go

@@ -319,46 +319,41 @@ func TestMergeHTTPClientConfigs(t *testing.T) {
 	}
 }
 
-func TestParseLabels(t *testing.T) {
-	testCases := []struct {
-		name            string
-		target          string
-		cfg             *Config
-		expectedAddress string
-		expectedErr     bool
-	}{
-		{
-			"invalid address",
-			"invalid:*//url",
-			&Config{},
-			"",
-			true,
-		},
-		{
-			"use some default params",
-			"alertmanager:9093",
-			&Config{PathPrefix: "test"},
-			"http://alertmanager:9093/test/api/v2/alerts",
-			false,
-		},
-		{
-			"use target address",
-			"https://alertmanager:9093/api/v1/alerts",
-			&Config{Scheme: "http", PathPrefix: "test"},
-			"https://alertmanager:9093/api/v1/alerts",
-			false,
-		},
+func TestParseLabels_Failure(t *testing.T) {
+	f := func(target string, cfg *Config) {
+		t.Helper()
+
+		_, _, err := parseLabels(target, nil, cfg)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			address, _, err := parseLabels(tc.target, nil, tc.cfg)
-			if err == nil == tc.expectedErr {
-				t.Fatalf("unexpected error; got %t; want %t", err != nil, tc.expectedErr)
-			}
-			if address != tc.expectedAddress {
-				t.Fatalf("unexpected address; got %q; want %q", address, tc.expectedAddress)
-			}
-		})
-	}
+	// invalid address
+	f("invalid:*//url", &Config{})
+}
+
+func TestParseLabels_Success(t *testing.T) {
+	f := func(target string, cfg *Config, expectedAddress string) {
+		t.Helper()
+
+		address, _, err := parseLabels(target, nil, cfg)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if address != expectedAddress {
+			t.Fatalf("unexpected address; got %q; want %q", address, expectedAddress)
+		}
+	}
+
+	// use some default params
+	f("alertmanager:9093", &Config{
+		PathPrefix: "test",
+	}, "http://alertmanager:9093/test/api/v2/alerts")
+
+	// use target address
+	f("https://alertmanager:9093/api/v1/alerts", &Config{
+		Scheme:     "http",
+		PathPrefix: "test",
+	}, "https://alertmanager:9093/api/v1/alerts")
 }

app/vmalert/notifier/init.go

@@ -25,6 +25,9 @@ var (
 		"Enable this flag if you want vmalert to evaluate alerting rules without sending any notifications to external receivers (eg. alertmanager). "+
 		"-notifier.url, -notifier.config and -notifier.blackhole are mutually exclusive.")
 
+	headers = flagutil.NewArrayString("notifier.headers", "Optional HTTP headers to send with each request to the corresponding -notifier.url. "+
+		"For example, -remoteWrite.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -notifier.url. "+
+		"Multiple headers must be delimited by '^^': -notifier.headers='header1:value1^^header2:value2,header3:value3'")
 	basicAuthUsername     = flagutil.NewArrayString("notifier.basicAuth.username", "Optional basic auth username for -notifier.url")
 	basicAuthPassword     = flagutil.NewArrayString("notifier.basicAuth.password", "Optional basic auth password for -notifier.url")
 	basicAuthPasswordFile = flagutil.NewArrayString("notifier.basicAuth.passwordFile", "Optional path to basic auth password file for -notifier.url")
@@ -171,6 +174,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 				Scopes:           strings.Split(oauth2Scopes.GetOptionalArg(i), ";"),
 				TokenURL:         oauth2TokenURL.GetOptionalArg(i),
 			},
+			Headers: []string{headers.GetOptionalArg(i)},
 		}
 
 		addr = strings.TrimSuffix(addr, "/")

app/vmalert/notifier/notifier_blackhole_test.go

@@ -17,12 +17,12 @@ func TestBlackHoleNotifier_Send(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 
 	alertCount := bh.metrics.alertsSent.Get()
 	if alertCount != 1 {
-		t.Errorf("expect value 1; instead got %d", alertCount)
+		t.Fatalf("expect value 1; instead got %d", alertCount)
 	}
 }
 
@@ -35,7 +35,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 		End:         time.Now().UTC(),
 		Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
 	}}, nil); err != nil {
-		t.Errorf("unexpected error %s", err)
+		t.Fatalf("unexpected error %s", err)
 	}
 
 	bh.Close()
@@ -44,7 +44,7 @@ func TestBlackHoleNotifier_Close(t *testing.T) {
 	alertMetricName := "vmalert_alerts_sent_total{addr=\"blackhole\"}"
 	for _, name := range defaultMetrics.ListMetricNames() {
 		if name == alertMetricName {
-			t.Errorf("Metric name should have unregistered.But still present")
+			t.Fatalf("Metric name should have unregistered.But still present")
 		}
 	}
 }

app/vmalert/remoteread/init.go

@@ -10,13 +10,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("remoteRead.url", "", "Optional URL to datasource compatible with Prometheus HTTP API. It can be single node VictoriaMetrics or vmselect."+
 		"Remote read is used to restore alerts state."+
 		"This configuration makes sense only if `vmalert` was configured with `remoteWrite.url` before and has been successfully persisted its state. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"See also '-remoteRead.disablePathAppend', '-remoteRead.showURL'.")
 
 	showRemoteReadURL = flag.Bool("remoteRead.showURL", false, "Whether to show -remoteRead.url in the exported metrics. "+
@@ -67,10 +68,10 @@ func Init() (datasource.QuerierBuilder, error) {
 	}
 	tr, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteRead.url=%q: %w", *addr, err)
 	}
 	tr.IdleConnTimeout = *idleConnectionTimeout
-	tr.DialContext = httputils.GetStatDialFunc("vmalert_remoteread")
+	tr.DialContext = netutil.NewStatDialFunc("vmalert_remoteread")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/remotewrite/client_test.go

@@ -44,8 +44,7 @@ func TestClient_Push(t *testing.T) {
 	}
 
 	r := rand.New(rand.NewSource(1))
-	const rowsN = 1e4
-	var sent int
+	const rowsN = int(1e4)
 	for i := 0; i < rowsN; i++ {
 		s := prompbmarshal.TimeSeries{
 			Samples: []prompbmarshal.Sample{{
@@ -57,17 +56,11 @@ func TestClient_Push(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected err: %s", err)
 		}
-		if err == nil {
-			sent++
-		}
 		err = faultyClient.Push(s)
 		if err != nil {
 			t.Fatalf("unexpected err: %s", err)
 		}
 	}
-	if sent == 0 {
-		t.Fatalf("0 series sent")
-	}
 	if err := client.Close(); err != nil {
 		t.Fatalf("failed to close client: %s", err)
 	}
@@ -75,77 +68,66 @@ func TestClient_Push(t *testing.T) {
 		t.Fatalf("failed to close faulty client: %s", err)
 	}
 	got := testSrv.accepted()
-	if got != sent {
-		t.Fatalf("expected to have %d series; got %d", sent, got)
+	if got != rowsN {
+		t.Fatalf("expected to have %d series; got %d", rowsN, got)
 	}
 	got = faultySrv.accepted()
-	if got != sent {
-		t.Fatalf("expected to have %d series for faulty client; got %d", sent, got)
+	if got != rowsN {
+		t.Fatalf("expected to have %d series for faulty client; got %d", rowsN, got)
 	}
 }
 
 func TestClient_run_maxBatchSizeDuringShutdown(t *testing.T) {
-	batchSize := 20
+	const batchSize = 20
 
-	testTable := []struct {
-		name     string // name of the test case
-		pushCnt  int    // how many time series is pushed to the client
-		batchCnt int    // the expected batch count sent by the client
-	}{
-		{
-			name:     "pushCnt % batchSize == 0",
-			pushCnt:  batchSize * 40,
-			batchCnt: 40,
-		},
-		{
-			name:     "pushCnt % batchSize != 0",
-			pushCnt:  batchSize*40 + 1,
-			batchCnt: 40 + 1,
-		},
-	}
+	f := func(pushCnt, batchCntExpected int) {
+		t.Helper()
 
-	for _, tt := range testTable {
-		t.Run(tt.name, func(t *testing.T) {
-			// run new server
-			bcServer := newBatchCntRWServer()
+		// run new server
+		bcServer := newBatchCntRWServer()
 
-			// run new client
-			rwClient, err := NewClient(context.Background(), Config{
-				MaxBatchSize: batchSize,
+		// run new client
+		rwClient, err := NewClient(context.Background(), Config{
+			MaxBatchSize: batchSize,
 
-				// Set everything to 1 to simplify the calculation.
-				Concurrency:   1,
-				MaxQueueSize:  1000,
-				FlushInterval: time.Minute,
+			// Set everything to 1 to simplify the calculation.
+			Concurrency:   1,
+			MaxQueueSize:  1000,
+			FlushInterval: time.Minute,
 
-				// batch count server
-				Addr: bcServer.URL,
-			})
-			if err != nil {
-				t.Fatalf("new remote write client failed, err: %v", err)
-			}
-
-			// push time series to the client.
-			for i := 0; i < tt.pushCnt; i++ {
-				if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
-					t.Fatalf("push time series to the client failed, err: %v", err)
-				}
-			}
-
-			// close the client so the rest ts will be flushed in `shutdown`
-			if err = rwClient.Close(); err != nil {
-				t.Fatalf("shutdown client failed, err: %v", err)
-			}
-
-			// finally check how many batches is sent.
-			if tt.batchCnt != bcServer.acceptedBatches() {
-				t.Errorf("client sent batch count incorrect, want: %d, get: %d", tt.batchCnt, bcServer.acceptedBatches())
-			}
-			if tt.pushCnt != bcServer.accepted() {
-				t.Errorf("client sent time series count incorrect, want: %d, get: %d", tt.pushCnt, bcServer.accepted())
-			}
+			// batch count server
+			Addr: bcServer.URL,
 		})
+		if err != nil {
+			t.Fatalf("cannot create remote write client: %s", err)
+		}
+
+		// push time series to the client.
+		for i := 0; i < pushCnt; i++ {
+			if err = rwClient.Push(prompbmarshal.TimeSeries{}); err != nil {
+				t.Fatalf("cannot time series to the client: %s", err)
+			}
+		}
+
+		// close the client so the rest ts will be flushed in `shutdown`
+		if err = rwClient.Close(); err != nil {
+			t.Fatalf("cannot shutdown client: %s", err)
+		}
+
+		// finally check how many batches is sent.
+		if bcServer.acceptedBatches() != batchCntExpected {
+			t.Fatalf("client sent batch count incorrect; got %d; want %d", bcServer.acceptedBatches(), batchCntExpected)
+		}
+		if pushCnt != bcServer.accepted() {
+			t.Fatalf("client sent time series count incorrect; got %d; want %d", bcServer.accepted(), pushCnt)
+		}
 	}
+
+	// pushCnt % batchSize == 0
+	f(batchSize*40, 40)
+
+	//pushCnt % batchSize != 0
+	f(batchSize*40+1, 40+1)
 }
 
 func newRWServer() *rwServer {

app/vmalert/remotewrite/debug_client.go

@@ -32,7 +32,7 @@ func NewDebugClient() (*DebugClient, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	c := &DebugClient{
 		c: &http.Client{

app/vmalert/remotewrite/init.go

@@ -9,12 +9,13 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 var (
 	addr = flag.String("remoteWrite.url", "", "Optional URL to VictoriaMetrics or vminsert where to persist alerts state "+
 		"and recording rules results in form of timeseries. "+
-		"Supports address in the form of IP address with a port (e.g., 127.0.0.1:8428) or DNS SRV record. "+
+		"Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. "+
 		"For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, "+
 		"then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
@@ -71,10 +72,10 @@ func Init(ctx context.Context) (*Client, error) {
 
 	t, err := httputils.Transport(*addr, *tlsCertFile, *tlsKeyFile, *tlsCAFile, *tlsServerName, *tlsInsecureSkipVerify)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create transport: %w", err)
+		return nil, fmt.Errorf("failed to create transport for -remoteWrite.url=%q: %w", *addr, err)
 	}
 	t.IdleConnTimeout = *idleConnectionTimeout
-	t.DialContext = httputils.GetStatDialFunc("vmalert_remotewrite")
+	t.DialContext = netutil.NewStatDialFunc("vmalert_remotewrite")
 
 	endpointParams, err := flagutil.ParseJSONMap(*oauth2EndpointParams)
 	if err != nil {

app/vmalert/replay_test.go

@@ -39,135 +39,102 @@ func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to ti
 }
 
 func TestReplay(t *testing.T) {
-	testCases := []struct {
-		name     string
-		from, to string
-		maxDP    int
-		cfg      []config.Group
-		qb       *fakeReplayQuerier
-	}{
-		{
-			name:  "one rule + one response",
-			from:  "2021-01-01T12:00:00.000Z",
-			to:    "2021-01-01T12:02:00.000Z",
-			maxDP: 10,
-			cfg: []config.Group{
-				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-			},
-			qb: &fakeReplayQuerier{
-				registry: map[string]map[string]struct{}{
-					"sum(up)": {"12:00:00+12:02:00": {}},
-				},
-			},
-		},
-		{
-			name:  "one rule + multiple responses",
-			from:  "2021-01-01T12:00:00.000Z",
-			to:    "2021-01-01T12:02:30.000Z",
-			maxDP: 1,
-			cfg: []config.Group{
-				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-			},
-			qb: &fakeReplayQuerier{
-				registry: map[string]map[string]struct{}{
-					"sum(up)": {
-						"12:00:00+12:01:00": {},
-						"12:01:00+12:02:00": {},
-						"12:02:00+12:02:30": {},
-					},
-				},
-			},
-		},
-		{
-			name:  "datapoints per step",
-			from:  "2021-01-01T12:00:00.000Z",
-			to:    "2021-01-01T15:02:30.000Z",
-			maxDP: 60,
-			cfg: []config.Group{
-				{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-			},
-			qb: &fakeReplayQuerier{
-				registry: map[string]map[string]struct{}{
-					"sum(up)": {
-						"12:00:00+13:00:00": {},
-						"13:00:00+14:00:00": {},
-						"14:00:00+15:00:00": {},
-						"15:00:00+15:02:30": {},
-					},
-				},
-			},
-		},
-		{
-			name:  "multiple recording rules + multiple responses",
-			from:  "2021-01-01T12:00:00.000Z",
-			to:    "2021-01-01T12:02:30.000Z",
-			maxDP: 1,
-			cfg: []config.Group{
-				{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
-				{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
-			},
-			qb: &fakeReplayQuerier{
-				registry: map[string]map[string]struct{}{
-					"sum(up)": {
-						"12:00:00+12:01:00": {},
-						"12:01:00+12:02:00": {},
-						"12:02:00+12:02:30": {},
-					},
-					"max(up)": {
-						"12:00:00+12:01:00": {},
-						"12:01:00+12:02:00": {},
-						"12:02:00+12:02:30": {},
-					},
-				},
-			},
-		},
-		{
-			name:  "multiple alerting rules + multiple responses",
-			from:  "2021-01-01T12:00:00.000Z",
-			to:    "2021-01-01T12:02:30.000Z",
-			maxDP: 1,
-			cfg: []config.Group{
-				{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
-				{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
-			},
-			qb: &fakeReplayQuerier{
-				registry: map[string]map[string]struct{}{
-					"sum(up) > 1": {
-						"12:00:00+12:01:00": {},
-						"12:01:00+12:02:00": {},
-						"12:02:00+12:02:30": {},
-					},
-					"max(up) < 1": {
-						"12:00:00+12:01:00": {},
-						"12:01:00+12:02:00": {},
-						"12:02:00+12:02:30": {},
-					},
-				},
-			},
-		},
+	f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
+		t.Helper()
+
+		fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
+		retriesOrig, delayOrig := *replayRuleRetryAttempts, *replayRulesDelay
+		defer func() {
+			*replayFrom, *replayTo = fromOrig, toOrig
+			*replayMaxDatapoints, *replayRuleRetryAttempts = maxDatapointsOrig, retriesOrig
+			*replayRulesDelay = delayOrig
+		}()
+
+		*replayRuleRetryAttempts = 1
+		*replayRulesDelay = time.Millisecond
+		rwb := &remotewrite.DebugClient{}
+		*replayFrom = from
+		*replayTo = to
+		*replayMaxDatapoints = maxDP
+		if err := replay(cfg, qb, rwb); err != nil {
+			t.Fatalf("replay failed: %s", err)
+		}
+		if len(qb.registry) > 0 {
+			t.Fatalf("not all requests were sent: %#v", qb.registry)
+		}
 	}
 
-	from, to, maxDP := *replayFrom, *replayTo, *replayMaxDatapoints
-	retries, delay := *replayRuleRetryAttempts, *replayRulesDelay
-	defer func() {
-		*replayFrom, *replayTo = from, to
-		*replayMaxDatapoints, *replayRuleRetryAttempts = maxDP, retries
-		*replayRulesDelay = delay
-	}()
+	// one rule + one response
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string]struct{}{
+			"sum(up)": {"12:00:00+12:02:00": {}},
+		},
+	})
 
-	*replayRuleRetryAttempts = 1
-	*replayRulesDelay = time.Millisecond
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			*replayFrom = tc.from
-			*replayTo = tc.to
-			*replayMaxDatapoints = tc.maxDP
-			if err := replay(tc.cfg, tc.qb, &remotewrite.DebugClient{}); err != nil {
-				t.Fatalf("replay failed: %s", err)
-			}
-			if len(tc.qb.registry) > 0 {
-				t.Fatalf("not all requests were sent: %#v", tc.qb.registry)
-			}
-		})
-	}
+	// one rule + multiple responses
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string]struct{}{
+			"sum(up)": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	})
+
+	// datapoints per step
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
+		{Interval: promutils.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string]struct{}{
+			"sum(up)": {
+				"12:00:00+13:00:00": {},
+				"13:00:00+14:00:00": {},
+				"14:00:00+15:00:00": {},
+				"15:00:00+15:02:30": {},
+			},
+		},
+	})
+
+	// multiple recording rules + multiple responses
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
+		{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
+		{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string]struct{}{
+			"sum(up)": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+			"max(up)": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	})
+
+	// multiple alerting rules + multiple responses
+	f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
+		{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
+		{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
+	}, &fakeReplayQuerier{
+		registry: map[string]map[string]struct{}{
+			"sum(up) > 1": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+			"max(up) < 1": {
+				"12:00:00+12:01:00": {},
+				"12:01:00+12:02:00": {},
+				"12:02:00+12:02:30": {},
+			},
+		},
+	})
 }

app/vmalert/rule/alerting.go

@@ -183,7 +183,7 @@ func (ar *AlertingRule) GetAlert(id uint64) *notifier.Alert {
 	return ar.alerts[id]
 }
 
-func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...interface{}) {
+func (ar *AlertingRule) logDebugf(at time.Time, a *notifier.Alert, format string, args ...any) {
 	if !ar.Debug {
 		return
 	}
@@ -441,9 +441,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			}
 			a.Value = m.Values[0]
 			a.Annotations = annotations
-			if err != nil {
-				return nil, err
-			}
 			a.KeepFiringSince = time.Time{}
 			continue
 		}

app/vmalert/rule/group.go

@@ -475,7 +475,7 @@ func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *
 	return randSleep
 }
 
-func (g *Group) infof(format string, args ...interface{}) {
+func (g *Group) infof(format string, args ...any) {
 	msg := fmt.Sprintf(format, args...)
 	logger.Infof("group %q %s; interval=%v; eval_offset=%v; concurrency=%d",
 		g.Name, msg, g.Interval, g.EvalOffset, g.Concurrency)

app/vmalert/rule/group_test.go

@@ -37,158 +37,142 @@ func TestMain(m *testing.M) {
 }
 
 func TestUpdateWith(t *testing.T) {
-	testCases := []struct {
-		name         string
-		currentRules []config.Rule
-		newRules     []config.Rule
-	}{
-		{
-			"new rule",
-			nil,
-			[]config.Rule{{Alert: "bar"}},
-		},
-		{
-			"update alerting rule",
-			[]config.Rule{
-				{
-					Alert: "foo",
-					Expr:  "up > 0",
-					For:   promutils.NewDuration(time.Second),
-					Labels: map[string]string{
-						"bar": "baz",
-					},
-					Annotations: map[string]string{
-						"summary":     "{{ $value|humanize }}",
-						"description": "{{$labels}}",
-					},
-				},
-				{
-					Alert: "bar",
-					Expr:  "up > 0",
-					For:   promutils.NewDuration(time.Second),
-					Labels: map[string]string{
-						"bar": "baz",
-					},
-				},
-			},
-			[]config.Rule{
-				{
-					Alert: "foo",
-					Expr:  "up > 10",
-					For:   promutils.NewDuration(time.Second),
-					Labels: map[string]string{
-						"baz": "bar",
-					},
-					Annotations: map[string]string{
-						"summary": "none",
-					},
-				},
-				{
-					Alert:         "bar",
-					Expr:          "up > 0",
-					For:           promutils.NewDuration(2 * time.Second),
-					KeepFiringFor: promutils.NewDuration(time.Minute),
-					Labels: map[string]string{
-						"bar": "baz",
-					},
-				},
-			},
-		},
-		{
-			"update recording rule",
-			[]config.Rule{{
-				Record: "foo",
-				Expr:   "max(up)",
-				Labels: map[string]string{
-					"bar": "baz",
-				},
-			}},
-			[]config.Rule{{
-				Record: "foo",
-				Expr:   "min(up)",
-				Labels: map[string]string{
-					"baz": "bar",
-				},
-			}},
-		},
-		{
-			"empty rule",
-			[]config.Rule{{Alert: "foo"}, {Record: "bar"}},
-			nil,
-		},
-		{
-			"multiple rules",
-			[]config.Rule{
-				{Alert: "bar"},
-				{Alert: "baz"},
-				{Alert: "foo"},
-			},
-			[]config.Rule{
-				{Alert: "baz"},
-				{Record: "foo"},
-			},
-		},
-		{
-			"replace rule",
-			[]config.Rule{{Alert: "foo1"}},
-			[]config.Rule{{Alert: "foo2"}},
-		},
-		{
-			"replace multiple rules",
-			[]config.Rule{
-				{Alert: "foo1"},
-				{Record: "foo2"},
-				{Alert: "foo3"},
-			},
-			[]config.Rule{
-				{Alert: "foo3"},
-				{Alert: "foo4"},
-				{Record: "foo5"},
-			},
-		},
-	}
+	f := func(currentRules, newRules []config.Rule) {
+		t.Helper()
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			g := &Group{Name: "test"}
-			qb := &datasource.FakeQuerier{}
-			for _, r := range tc.currentRules {
-				r.ID = config.HashRule(r)
-				g.Rules = append(g.Rules, g.newRule(qb, r))
-			}
+		g := &Group{
+			Name: "test",
+		}
+		qb := &datasource.FakeQuerier{}
+		for _, r := range currentRules {
+			r.ID = config.HashRule(r)
+			g.Rules = append(g.Rules, g.newRule(qb, r))
+		}
 
-			ng := &Group{Name: "test"}
-			for _, r := range tc.newRules {
-				r.ID = config.HashRule(r)
-				ng.Rules = append(ng.Rules, ng.newRule(qb, r))
-			}
+		ng := &Group{
+			Name: "test",
+		}
+		for _, r := range newRules {
+			r.ID = config.HashRule(r)
+			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
+		}
 
-			err := g.updateWith(ng)
-			if err != nil {
-				t.Fatal(err)
-			}
+		err := g.updateWith(ng)
+		if err != nil {
+			t.Fatalf("cannot update rule: %s", err)
+		}
 
-			if len(g.Rules) != len(tc.newRules) {
-				t.Fatalf("expected to have %d rules; got: %d",
-					len(g.Rules), len(tc.newRules))
-			}
-			sort.Slice(g.Rules, func(i, j int) bool {
-				return g.Rules[i].ID() < g.Rules[j].ID()
-			})
-			sort.Slice(ng.Rules, func(i, j int) bool {
-				return ng.Rules[i].ID() < ng.Rules[j].ID()
-			})
-			for i, r := range g.Rules {
-				got, want := r, ng.Rules[i]
-				if got.ID() != want.ID() {
-					t.Fatalf("expected to have rule %q; got %q", want, got)
-				}
-				if err := CompareRules(t, got, want); err != nil {
-					t.Fatalf("comparison error: %s", err)
-				}
-			}
+		if len(g.Rules) != len(newRules) {
+			t.Fatalf("expected to have %d rules; got: %d", len(g.Rules), len(newRules))
+		}
+		sort.Slice(g.Rules, func(i, j int) bool {
+			return g.Rules[i].ID() < g.Rules[j].ID()
 		})
+		sort.Slice(ng.Rules, func(i, j int) bool {
+			return ng.Rules[i].ID() < ng.Rules[j].ID()
+		})
+		for i, r := range g.Rules {
+			got, want := r, ng.Rules[i]
+			if got.ID() != want.ID() {
+				t.Fatalf("expected to have rule %q; got %q", want, got)
+			}
+			if err := CompareRules(t, got, want); err != nil {
+				t.Fatalf("comparison error: %s", err)
+			}
+		}
 	}
+
+	// new rule
+	f(nil, []config.Rule{
+		{Alert: "bar"},
+	})
+
+	// update alerting rule
+	f([]config.Rule{
+		{
+			Alert: "foo",
+			Expr:  "up > 0",
+			For:   promutils.NewDuration(time.Second),
+			Labels: map[string]string{
+				"bar": "baz",
+			},
+			Annotations: map[string]string{
+				"summary":     "{{ $value|humanize }}",
+				"description": "{{$labels}}",
+			},
+		},
+		{
+			Alert: "bar",
+			Expr:  "up > 0",
+			For:   promutils.NewDuration(time.Second),
+			Labels: map[string]string{
+				"bar": "baz",
+			},
+		},
+	}, []config.Rule{
+		{
+			Alert: "foo",
+			Expr:  "up > 10",
+			For:   promutils.NewDuration(time.Second),
+			Labels: map[string]string{
+				"baz": "bar",
+			},
+			Annotations: map[string]string{
+				"summary": "none",
+			},
+		},
+		{
+			Alert:         "bar",
+			Expr:          "up > 0",
+			For:           promutils.NewDuration(2 * time.Second),
+			KeepFiringFor: promutils.NewDuration(time.Minute),
+			Labels: map[string]string{
+				"bar": "baz",
+			},
+		},
+	})
+
+	// update recording rule
+	f([]config.Rule{{
+		Record: "foo",
+		Expr:   "max(up)",
+		Labels: map[string]string{
+			"bar": "baz",
+		},
+	}}, []config.Rule{{
+		Record: "foo",
+		Expr:   "min(up)",
+		Labels: map[string]string{
+			"baz": "bar",
+		},
+	}})
+
+	// empty rule
+	f([]config.Rule{{Alert: "foo"}, {Record: "bar"}}, nil)
+
+	// multiple rules
+	f([]config.Rule{
+		{Alert: "bar"},
+		{Alert: "baz"},
+		{Alert: "foo"},
+	}, []config.Rule{
+		{Alert: "baz"},
+		{Record: "foo"},
+	})
+
+	// replace rule
+	f([]config.Rule{{Alert: "foo1"}}, []config.Rule{{Alert: "foo2"}})
+
+	// replace multiple rules
+	f([]config.Rule{
+		{Alert: "foo1"},
+		{Record: "foo2"},
+		{Alert: "foo3"},
+	}, []config.Rule{
+		{Alert: "foo3"},
+		{Alert: "foo4"},
+		{Record: "foo5"},
+	})
 }
 
 func TestGroupStart(t *testing.T) {
@@ -312,30 +296,23 @@ func TestGroupStart(t *testing.T) {
 	<-finished
 }
 
-func TestResolveDuration(t *testing.T) {
-	testCases := []struct {
-		groupInterval time.Duration
-		maxDuration   time.Duration
-		resendDelay   time.Duration
-		expected      time.Duration
-	}{
-		{time.Minute, 0, 0, 4 * time.Minute},
-		{time.Minute, 0, 2 * time.Minute, 8 * time.Minute},
-		{time.Minute, 4 * time.Minute, 4 * time.Minute, 4 * time.Minute},
-		{2 * time.Minute, time.Minute, 2 * time.Minute, time.Minute},
-		{time.Minute, 2 * time.Minute, 1 * time.Minute, 2 * time.Minute},
-		{2 * time.Minute, 0, 1 * time.Minute, 8 * time.Minute},
-		{0, 0, 0, 0},
+func TestGetResolveDuration(t *testing.T) {
+	f := func(groupInterval, maxDuration, resendDelay, resultExpected time.Duration) {
+		t.Helper()
+
+		result := getResolveDuration(groupInterval, resendDelay, maxDuration)
+		if result != resultExpected {
+			t.Fatalf("unexpected result; got %s; want %s", result, resultExpected)
+		}
 	}
 
-	for _, tc := range testCases {
-		t.Run(fmt.Sprintf("%v-%v-%v", tc.groupInterval, tc.expected, tc.maxDuration), func(t *testing.T) {
-			got := getResolveDuration(tc.groupInterval, tc.resendDelay, tc.maxDuration)
-			if got != tc.expected {
-				t.Errorf("expected to have %v; got %v", tc.expected, got)
-			}
-		})
-	}
+	f(0, 0, 0, 0)
+	f(time.Minute, 0, 0, 4*time.Minute)
+	f(time.Minute, 0, 2*time.Minute, 8*time.Minute)
+	f(time.Minute, 4*time.Minute, 4*time.Minute, 4*time.Minute)
+	f(2*time.Minute, time.Minute, 2*time.Minute, time.Minute)
+	f(time.Minute, 2*time.Minute, 1*time.Minute, 2*time.Minute)
+	f(2*time.Minute, 0, 1*time.Minute, 8*time.Minute)
 }
 
 func TestGetStaleSeries(t *testing.T) {
@@ -345,6 +322,7 @@ func TestGetStaleSeries(t *testing.T) {
 	}
 	f := func(r Rule, labels, expLabels [][]prompbmarshal.Label) {
 		t.Helper()
+
 		var tss []prompbmarshal.TimeSeries
 		for _, l := range labels {
 			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
@@ -606,7 +584,7 @@ func TestGroupStartDelay(t *testing.T) {
 		delay := delayBeforeStart(at, key, g.Interval, g.EvalOffset)
 		gotStart := at.Add(delay)
 		if expTS != gotStart {
-			t.Errorf("expected to get %v; got %v instead", expTS, gotStart)
+			t.Fatalf("expected to get %v; got %v instead", expTS, gotStart)
 		}
 	}
 
@@ -647,157 +625,118 @@ func TestGroupStartDelay(t *testing.T) {
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {
+	f := func(g *Group, tsOrigin, tsExpected string) {
+		t.Helper()
+
+		originT, _ := time.Parse(time.RFC3339, tsOrigin)
+		expT, _ := time.Parse(time.RFC3339, tsExpected)
+		gotTS := g.adjustReqTimestamp(originT)
+		if !gotTS.Equal(expT) {
+			t.Fatalf("get wrong prometheus request timestamp: %s; want %s", gotTS, expT)
+		}
+	}
+
 	offset := 30 * time.Minute
 	evalDelay := 1 * time.Minute
 	disableAlign := false
-	testCases := []struct {
-		name            string
-		g               *Group
-		originTS, expTS string
-	}{
-		{
-			"with query align + default evalDelay",
-			&Group{
-				Interval: time.Hour,
-			},
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T11:00:00+00:00",
-		},
-		{
-			"without query align + default evalDelay",
-			&Group{
-				Interval:      time.Hour,
-				evalAlignment: &disableAlign,
-			},
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T11:10:30+00:00",
-		},
-		{
-			"with eval_offset, find previous offset point + default evalDelay",
-			&Group{
-				EvalOffset: &offset,
-				Interval:   time.Hour,
-			},
-			"2023-08-28T11:11:00+00:00",
-			"2023-08-28T10:30:00+00:00",
-		},
-		{
-			"with eval_offset + default evalDelay",
-			&Group{
-				EvalOffset: &offset,
-				Interval:   time.Hour,
-			},
-			"2023-08-28T11:41:00+00:00",
-			"2023-08-28T11:30:00+00:00",
-		},
-		{
-			"1h interval with eval_delay",
-			&Group{
-				EvalDelay: &evalDelay,
-				Interval:  time.Hour,
-			},
-			"2023-08-28T11:41:00+00:00",
-			"2023-08-28T11:00:00+00:00",
-		},
-		{
-			"1m interval with eval_delay",
-			&Group{
-				EvalDelay: &evalDelay,
-				Interval:  time.Minute,
-			},
-			"2023-08-28T11:41:13+00:00",
-			"2023-08-28T11:40:00+00:00",
-		},
-		{
-			"disable alignment with eval_delay",
-			&Group{
-				EvalDelay:     &evalDelay,
-				Interval:      time.Hour,
-				evalAlignment: &disableAlign,
-			},
-			"2023-08-28T11:41:00+00:00",
-			"2023-08-28T11:40:00+00:00",
-		},
-	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			originT, _ := time.Parse(time.RFC3339, tc.originTS)
-			expT, _ := time.Parse(time.RFC3339, tc.expTS)
-			gotTS := tc.g.adjustReqTimestamp(originT)
-			if !gotTS.Equal(expT) {
-				t.Fatalf("get wrong prometheus request timestamp, expect %s, got %s", expT, gotTS)
-			}
-		})
-	}
+
+	// with query align + default evalDelay
+	f(&Group{
+		Interval: time.Hour,
+	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:00:00+00:00")
+
+	// without query align + default evalDelay
+	f(&Group{
+		Interval:      time.Hour,
+		evalAlignment: &disableAlign,
+	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:10:30+00:00")
+
+	// with eval_offset, find previous offset point + default evalDelay
+	f(&Group{
+		EvalOffset: &offset,
+		Interval:   time.Hour,
+	}, "2023-08-28T11:11:00+00:00", "2023-08-28T10:30:00+00:00")
+
+	// with eval_offset + default evalDelay
+	f(&Group{
+		EvalOffset: &offset,
+		Interval:   time.Hour,
+	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:30:00+00:00")
+
+	// 1h interval with eval_delay
+	f(&Group{
+		EvalDelay: &evalDelay,
+		Interval:  time.Hour,
+	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:00:00+00:00")
+
+	// 1m interval with eval_delay
+	f(&Group{
+		EvalDelay: &evalDelay,
+		Interval:  time.Minute,
+	}, "2023-08-28T11:41:13+00:00", "2023-08-28T11:40:00+00:00")
+
+	// disable alignment with eval_delay
+	f(&Group{
+		EvalDelay:     &evalDelay,
+		Interval:      time.Hour,
+		evalAlignment: &disableAlign,
+	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:40:00+00:00")
 }
 
 func TestRangeIterator(t *testing.T) {
-	testCases := []struct {
-		ri     rangeIterator
-		result [][2]time.Time
-	}{
-		{
-			ri: rangeIterator{
-				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
-				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
-				step:  5 * time.Minute,
-			},
-			result: [][2]time.Time{
-				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
-				{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
-				{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
-				{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
-				{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
-				{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-			},
-		},
-		{
-			ri: rangeIterator{
-				start: parseTime(t, "2021-01-01T12:00:00.000Z"),
-				end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
-				step:  45 * time.Minute,
-			},
-			result: [][2]time.Time{
-				{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-				{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
-			},
-		},
-		{
-			ri: rangeIterator{
-				start: parseTime(t, "2021-01-01T12:00:12.000Z"),
-				end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
-				step:  time.Second,
-			},
-			result: [][2]time.Time{
-				{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
-				{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
-				{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
-				{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
-				{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
-			},
-		},
+	f := func(ri rangeIterator, resultExpected [][2]time.Time) {
+		t.Helper()
+
+		var j int
+		for ri.next() {
+			if len(resultExpected) < j+1 {
+				t.Fatalf("unexpected result for iterator on step %d: %v - %v", j, ri.s, ri.e)
+			}
+			s, e := ri.s, ri.e
+			expS, expE := resultExpected[j][0], resultExpected[j][1]
+			if s != expS {
+				t.Fatalf("expected to get start=%v; got %v", expS, s)
+			}
+			if e != expE {
+				t.Fatalf("expected to get end=%v; got %v", expE, e)
+			}
+			j++
+		}
 	}
 
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
-			var j int
-			for tc.ri.next() {
-				if len(tc.result) < j+1 {
-					t.Fatalf("unexpected result for iterator on step %d: %v - %v",
-						j, tc.ri.s, tc.ri.e)
-				}
-				s, e := tc.ri.s, tc.ri.e
-				expS, expE := tc.result[j][0], tc.result[j][1]
-				if s != expS {
-					t.Fatalf("expected to get start=%v; got %v", expS, s)
-				}
-				if e != expE {
-					t.Fatalf("expected to get end=%v; got %v", expE, e)
-				}
-				j++
-			}
-		})
-	}
+	f(rangeIterator{
+		start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+		end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+		step:  5 * time.Minute,
+	}, [][2]time.Time{
+		{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:05:00.000Z")},
+		{parseTime(t, "2021-01-01T12:05:00.000Z"), parseTime(t, "2021-01-01T12:10:00.000Z")},
+		{parseTime(t, "2021-01-01T12:10:00.000Z"), parseTime(t, "2021-01-01T12:15:00.000Z")},
+		{parseTime(t, "2021-01-01T12:15:00.000Z"), parseTime(t, "2021-01-01T12:20:00.000Z")},
+		{parseTime(t, "2021-01-01T12:20:00.000Z"), parseTime(t, "2021-01-01T12:25:00.000Z")},
+		{parseTime(t, "2021-01-01T12:25:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+	})
+
+	f(rangeIterator{
+		start: parseTime(t, "2021-01-01T12:00:00.000Z"),
+		end:   parseTime(t, "2021-01-01T12:30:00.000Z"),
+		step:  45 * time.Minute,
+	}, [][2]time.Time{
+		{parseTime(t, "2021-01-01T12:00:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+		{parseTime(t, "2021-01-01T12:30:00.000Z"), parseTime(t, "2021-01-01T12:30:00.000Z")},
+	})
+
+	f(rangeIterator{
+		start: parseTime(t, "2021-01-01T12:00:12.000Z"),
+		end:   parseTime(t, "2021-01-01T12:00:17.000Z"),
+		step:  time.Second,
+	}, [][2]time.Time{
+		{parseTime(t, "2021-01-01T12:00:12.000Z"), parseTime(t, "2021-01-01T12:00:13.000Z")},
+		{parseTime(t, "2021-01-01T12:00:13.000Z"), parseTime(t, "2021-01-01T12:00:14.000Z")},
+		{parseTime(t, "2021-01-01T12:00:14.000Z"), parseTime(t, "2021-01-01T12:00:15.000Z")},
+		{parseTime(t, "2021-01-01T12:00:15.000Z"), parseTime(t, "2021-01-01T12:00:16.000Z")},
+		{parseTime(t, "2021-01-01T12:00:16.000Z"), parseTime(t, "2021-01-01T12:00:17.000Z")},
+	})
 }
 
 func parseTime(t *testing.T, s string) time.Time {

app/vmalert/rule/recording_test.go

@@ -13,218 +13,225 @@ import (
 )
 
 func TestRecordingRule_Exec(t *testing.T) {
+	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
+		t.Helper()
+
+		fq := &datasource.FakeQuerier{}
+		fq.Add(metrics...)
+		rule.q = fq
+		rule.state = &ruleState{
+			entries: make([]StateEntry, 10),
+		}
+		tss, err := rule.exec(context.TODO(), time.Now(), 0)
+		if err != nil {
+			t.Fatalf("unexpected RecordingRule.exec error: %s", err)
+		}
+		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
+			t.Fatalf("timeseries missmatch: %s", err)
+		}
+	}
+
 	timestamp := time.Now()
-	testCases := []struct {
-		rule    *RecordingRule
-		metrics []datasource.Metric
-		expTS   []prompbmarshal.TimeSeries
-	}{
-		{
-			&RecordingRule{Name: "foo"},
-			[]datasource.Metric{metricWithValueAndLabels(t, 10,
-				"__name__", "bar",
-			)},
-			[]prompbmarshal.TimeSeries{
-				newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "foo",
-				}),
-			},
+
+	f(&RecordingRule{
+		Name: "foo",
+	}, []datasource.Metric{
+		metricWithValueAndLabels(t, 10, "__name__", "bar"),
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foo",
+		}),
+	})
+
+	f(&RecordingRule{
+		Name: "foobarbaz",
+	}, []datasource.Metric{
+		metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+		metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+		metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "foo",
+		}),
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "bar",
+		}),
+		newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "baz",
+		}),
+	})
+
+	f(&RecordingRule{
+		Name: "job:foo",
+		Labels: map[string]string{
+			"source": "test",
 		},
-		{
-			&RecordingRule{Name: "foobarbaz"},
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
-				metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
-				metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
-			},
-			[]prompbmarshal.TimeSeries{
-				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "foobarbaz",
-					"job":      "foo",
-				}),
-				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "foobarbaz",
-					"job":      "bar",
-				}),
-				newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "foobarbaz",
-					"job":      "baz",
-				}),
-			},
-		},
-		{
-			&RecordingRule{
-				Name: "job:foo",
-				Labels: map[string]string{
-					"source": "test",
-				},
-			},
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
-			},
-			[]prompbmarshal.TimeSeries{
-				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "job:foo",
-					"job":      "foo",
-					"source":   "test",
-				}),
-				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__":        "job:foo",
-					"job":             "bar",
-					"source":          "test",
-					"exported_source": "origin",
-				}),
-			},
-		},
-	}
-	for _, tc := range testCases {
-		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &datasource.FakeQuerier{}
-			fq.Add(tc.metrics...)
-			tc.rule.q = fq
-			tc.rule.state = &ruleState{entries: make([]StateEntry, 10)}
-			tss, err := tc.rule.exec(context.TODO(), time.Now(), 0)
-			if err != nil {
-				t.Fatalf("unexpected Exec err: %s", err)
-			}
-			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
-				t.Fatalf("timeseries missmatch: %s", err)
-			}
-		})
-	}
+	}, []datasource.Metric{
+		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "job:foo",
+			"job":      "foo",
+			"source":   "test",
+		}),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":        "job:foo",
+			"job":             "bar",
+			"source":          "test",
+			"exported_source": "origin",
+		}),
+	})
 }
 
 func TestRecordingRule_ExecRange(t *testing.T) {
-	timestamp := time.Now()
-	testCases := []struct {
-		rule    *RecordingRule
-		metrics []datasource.Metric
-		expTS   []prompbmarshal.TimeSeries
-	}{
-		{
-			&RecordingRule{Name: "foo"},
-			[]datasource.Metric{metricWithValuesAndLabels(t, []float64{10, 20, 30},
-				"__name__", "bar",
-			)},
-			[]prompbmarshal.TimeSeries{
-				newTimeSeries([]float64{10, 20, 30},
-					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
-					map[string]string{
-						"__name__": "foo",
-					}),
-			},
-		},
-		{
-			&RecordingRule{Name: "foobarbaz"},
-			[]datasource.Metric{
-				metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
-				metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
-				metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-			},
-			[]prompbmarshal.TimeSeries{
-				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "foobarbaz",
-					"job":      "foo",
-				}),
-				newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
-					"__name__": "foobarbaz",
-					"job":      "bar",
-				}),
-				newTimeSeries([]float64{4, 5, 6},
-					[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
-					map[string]string{
-						"__name__": "foobarbaz",
-						"job":      "baz",
-					}),
-			},
-		},
-		{
-			&RecordingRule{Name: "job:foo", Labels: map[string]string{
-				"source": "test",
-			}},
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
-				metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
-			},
-			[]prompbmarshal.TimeSeries{
-				newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "job:foo",
-					"job":      "foo",
-					"source":   "test",
-				}),
-				newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
-					"__name__": "job:foo",
-					"job":      "bar",
-					"source":   "test",
-				}),
-			},
-		},
-	}
-	for _, tc := range testCases {
-		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &datasource.FakeQuerier{}
-			fq.Add(tc.metrics...)
-			tc.rule.q = fq
-			tss, err := tc.rule.execRange(context.TODO(), time.Now(), time.Now())
-			if err != nil {
-				t.Fatalf("unexpected Exec err: %s", err)
-			}
-			if err := compareTimeSeries(t, tc.expTS, tss); err != nil {
-				t.Fatalf("timeseries missmatch: %s", err)
-			}
-		})
-	}
-}
+	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
+		t.Helper()
 
-func TestRecordingRuleLimit(t *testing.T) {
-	timestamp := time.Now()
-	testCases := []struct {
-		limit int
-		err   string
-	}{
-		{
-			limit: 0,
-		},
-		{
-			limit: -1,
-		},
-		{
-			limit: 1,
-			err:   "exec exceeded limit of 1 with 3 series",
-		},
-		{
-			limit: 2,
-			err:   "exec exceeded limit of 2 with 3 series",
-		},
+		fq := &datasource.FakeQuerier{}
+		fq.Add(metrics...)
+		rule.q = fq
+		tss, err := rule.execRange(context.TODO(), time.Now(), time.Now())
+		if err != nil {
+			t.Fatalf("unexpected RecordingRule.execRange error: %s", err)
+		}
+		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
+			t.Fatalf("timeseries missmatch: %s", err)
+		}
 	}
-	testMetrics := []datasource.Metric{
+
+	timestamp := time.Now()
+
+	f(&RecordingRule{
+		Name: "foo",
+	}, []datasource.Metric{
+		metricWithValuesAndLabels(t, []float64{10, 20, 30}, "__name__", "bar"),
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+			"__name__": "foo",
+		}),
+	})
+
+	f(&RecordingRule{
+		Name: "foobarbaz",
+	}, []datasource.Metric{
 		metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
-	}
-	rule := &RecordingRule{Name: "job:foo",
-		state: &ruleState{entries: make([]StateEntry, 10)},
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "foo",
+		}),
+		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "bar",
+		}),
+		newTimeSeries([]float64{4, 5, 6},
+			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+				"__name__": "foobarbaz",
+				"job":      "baz",
+			}),
+	})
+
+	f(&RecordingRule{
+		Name: "job:foo",
 		Labels: map[string]string{
-			"source": "test_limit",
+			"source": "test",
 		},
-		metrics: &recordingRuleMetrics{
-			errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-		},
-	}
-	var err error
-	for _, testCase := range testCases {
-		fq := &datasource.FakeQuerier{}
-		fq.Add(testMetrics...)
-		rule.q = fq
-		_, err = rule.exec(context.TODO(), timestamp, testCase.limit)
-		if err != nil && !strings.EqualFold(err.Error(), testCase.err) {
-			t.Fatal(err)
-		}
-	}
+	}, []datasource.Metric{
+		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
+		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "job:foo",
+			"job":      "foo",
+			"source":   "test",
+		}),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "job:foo",
+			"job":      "bar",
+			"source":   "test",
+		}),
+	})
 }
 
-func TestRecordingRule_ExecNegative(t *testing.T) {
+func TestRecordingRuleLimit_Failure(t *testing.T) {
+	f := func(limit int, errStrExpected string) {
+		t.Helper()
+
+		testMetrics := []datasource.Metric{
+			metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+			metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+			metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+		}
+
+		fq := &datasource.FakeQuerier{}
+		fq.Add(testMetrics...)
+
+		rule := &RecordingRule{Name: "job:foo",
+			state: &ruleState{entries: make([]StateEntry, 10)},
+			Labels: map[string]string{
+				"source": "test_limit",
+			},
+			metrics: &recordingRuleMetrics{
+				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
+			},
+		}
+		rule.q = fq
+
+		_, err := rule.exec(context.TODO(), time.Now(), limit)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		errStr := err.Error()
+		if !strings.Contains(errStr, errStrExpected) {
+			t.Fatalf("missing %q in the error %q", errStrExpected, errStr)
+		}
+	}
+
+	f(1, "exec exceeded limit of 1 with 3 series")
+	f(2, "exec exceeded limit of 2 with 3 series")
+}
+
+func TestRecordingRuleLimit_Success(t *testing.T) {
+	f := func(limit int) {
+		t.Helper()
+
+		testMetrics := []datasource.Metric{
+			metricWithValuesAndLabels(t, []float64{1}, "__name__", "foo", "job", "foo"),
+			metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
+			metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
+		}
+
+		fq := &datasource.FakeQuerier{}
+		fq.Add(testMetrics...)
+
+		rule := &RecordingRule{Name: "job:foo",
+			state: &ruleState{entries: make([]StateEntry, 10)},
+			Labels: map[string]string{
+				"source": "test_limit",
+			},
+			metrics: &recordingRuleMetrics{
+				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
+			},
+		}
+		rule.q = fq
+
+		_, err := rule.exec(context.TODO(), time.Now(), limit)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+	}
+
+	f(0)
+	f(-1)
+}
+
+func TestRecordingRuleExec_Negative(t *testing.T) {
 	rr := &RecordingRule{
 		Name: "job:foo",
 		Labels: map[string]string{
@@ -256,6 +263,6 @@ func TestRecordingRule_ExecNegative(t *testing.T) {
 
 	_, err = rr.exec(context.TODO(), time.Now(), 0)
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("cannot execute recroding rule: %s", err)
 	}
 }

app/vmalert/templates/template.go

@@ -316,7 +316,7 @@ func templateFuncs() textTpl.FuncMap {
 
 		// humanize converts given number to a human readable format
 		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(i interface{}) (string, error) {
+		"humanize": func(i any) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -347,7 +347,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(i interface{}) (string, error) {
+		"humanize1024": func(i any) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -359,7 +359,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizeDuration converts given seconds to a human-readable duration
-		"humanizeDuration": func(i interface{}) (string, error) {
+		"humanizeDuration": func(i any) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -405,7 +405,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(i interface{}) (string, error) {
+		"humanizePercentage": func(i any) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -414,7 +414,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(i interface{}) (string, error) {
+		"humanizeTimestamp": func(i any) (string, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return "", err
@@ -427,7 +427,7 @@ func templateFuncs() textTpl.FuncMap {
 		},
 
 		// toTime converts given timestamp to a time.Time.
-		"toTime": func(i interface{}) (time.Time, error) {
+		"toTime": func(i any) (time.Time, error) {
 			v, err := toFloat64(i)
 			if err != nil {
 				return time.Time{}, err
@@ -524,8 +524,8 @@ func templateFuncs() textTpl.FuncMap {
 
 		// Converts a list of objects to a map with keys arg0, arg1 etc.
 		// This is intended to allow multiple arguments to be passed to templates.
-		"args": func(args ...interface{}) map[string]interface{} {
-			result := make(map[string]interface{})
+		"args": func(args ...any) map[string]any {
+			result := make(map[string]any)
 			for i, a := range args {
 				result[fmt.Sprintf("arg%d", i)] = a
 			}
@@ -565,7 +565,7 @@ func (t Time) Time() time.Time {
 	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
 }
 
-func toFloat64(v interface{}) (float64, error) {
+func toFloat64(v any) (float64, error) {
 	switch i := v.(type) {
 	case float64:
 		return i, nil

app/vmalert/templates/template_test.go

@@ -7,10 +7,11 @@ import (
 	textTpl "text/template"
 )
 
-func TestTemplateFuncs(t *testing.T) {
-	funcs := templateFuncs()
+func TestTemplateFuncs_StringConversion(t *testing.T) {
 	f := func(funcName, s, resultExpected string) {
 		t.Helper()
+
+		funcs := templateFuncs()
 		v := funcs[funcName]
 		fLocal := v.(func(s string) string)
 		result := fLocal(s)
@@ -18,6 +19,7 @@ func TestTemplateFuncs(t *testing.T) {
 			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
 		}
 	}
+
 	f("title", "foo bar", "Foo Bar")
 	f("toUpper", "foo", "FOO")
 	f("toLower", "FOO", "foo")
@@ -31,7 +33,10 @@ func TestTemplateFuncs(t *testing.T) {
 	f("stripPort", "foo:1234", "foo")
 	f("stripDomain", "foo.bar.baz", "foo")
 	f("stripDomain", "foo.bar:123", "foo:123")
+}
 
+func TestTemplateFuncs_Match(t *testing.T) {
+	funcs := templateFuncs()
 	// check "match" func
 	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
 	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
@@ -51,11 +56,15 @@ func TestTemplateFuncs(t *testing.T) {
 	if !ok {
 		t.Fatalf("unexpected mismatch")
 	}
+}
 
-	formatting := func(funcName string, p interface{}, resultExpected string) {
+func TestTemplateFuncs_Formatting(t *testing.T) {
+	f := func(funcName string, p any, resultExpected string) {
 		t.Helper()
+
+		funcs := templateFuncs()
 		v := funcs[funcName]
-		fLocal := v.(func(s interface{}) (string, error))
+		fLocal := v.(func(s any) (string, error))
 		result, err := fLocal(p)
 		if err != nil {
 			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
@@ -64,35 +73,36 @@ func TestTemplateFuncs(t *testing.T) {
 			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
 		}
 	}
-	formatting("humanize1024", float64(0), "0")
-	formatting("humanize1024", math.Inf(0), "+Inf")
-	formatting("humanize1024", math.NaN(), "NaN")
-	formatting("humanize1024", float64(127087), "124.1ki")
-	formatting("humanize1024", float64(130137088), "124.1Mi")
-	formatting("humanize1024", float64(133260378112), "124.1Gi")
-	formatting("humanize1024", float64(136458627186688), "124.1Ti")
-	formatting("humanize1024", float64(139733634239168512), "124.1Pi")
-	formatting("humanize1024", float64(143087241460908556288), "124.1Ei")
-	formatting("humanize1024", float64(146521335255970361638912), "124.1Zi")
-	formatting("humanize1024", float64(150037847302113650318245888), "124.1Yi")
-	formatting("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 
-	formatting("humanize", float64(127087), "127.1k")
-	formatting("humanize", float64(136458627186688), "136.5T")
+	f("humanize1024", float64(0), "0")
+	f("humanize1024", math.Inf(0), "+Inf")
+	f("humanize1024", math.NaN(), "NaN")
+	f("humanize1024", float64(127087), "124.1ki")
+	f("humanize1024", float64(130137088), "124.1Mi")
+	f("humanize1024", float64(133260378112), "124.1Gi")
+	f("humanize1024", float64(136458627186688), "124.1Ti")
+	f("humanize1024", float64(139733634239168512), "124.1Pi")
+	f("humanize1024", float64(143087241460908556288), "124.1Ei")
+	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 
-	formatting("humanizeDuration", 1, "1s")
-	formatting("humanizeDuration", 0.2, "200ms")
-	formatting("humanizeDuration", 42000, "11h 40m 0s")
-	formatting("humanizeDuration", 16790555, "194d 8h 2m 35s")
+	f("humanize", float64(127087), "127.1k")
+	f("humanize", float64(136458627186688), "136.5T")
 
-	formatting("humanizePercentage", 1, "100%")
-	formatting("humanizePercentage", 0.8, "80%")
-	formatting("humanizePercentage", 0.015, "1.5%")
+	f("humanizeDuration", 1, "1s")
+	f("humanizeDuration", 0.2, "200ms")
+	f("humanizeDuration", 42000, "11h 40m 0s")
+	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
 
-	formatting("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
+	f("humanizePercentage", 1, "100%")
+	f("humanizePercentage", 0.8, "80%")
+	f("humanizePercentage", 0.015, "1.5%")
+
+	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
 }
 
-func mkTemplate(current, replacement interface{}) textTemplate {
+func mkTemplate(current, replacement any) textTemplate {
 	tmpl := textTemplate{}
 	if current != nil {
 		switch val := current.(type) {
@@ -138,224 +148,201 @@ func equalTemplates(tmpls ...*textTpl.Template) bool {
 	return true
 }
 
-func TestTemplates_Load(t *testing.T) {
-	testCases := []struct {
-		name             string
-		initialTemplate  textTemplate
-		pathPatterns     []string
-		overwrite        bool
-		expectedTemplate textTemplate
-		expErr           string
-	}{
-		{
-			"non existing path undefined template override",
-			mkTemplate(nil, nil),
-			[]string{
-				"templates/non-existing/good-*.tpl",
-				"templates/absent/good-*.tpl",
-			},
-			true,
-			mkTemplate(``, nil),
-			"",
-		},
-		{
-			"non existing path defined template override",
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{- printf "value" -}}
-				{{- end -}}
-			`, nil),
-			[]string{
-				"templates/non-existing/good-*.tpl",
-				"templates/absent/good-*.tpl",
-			},
-			true,
-			mkTemplate(``, nil),
-			"",
-		},
-		{
-			"existing path undefined template override",
-			mkTemplate(nil, nil),
-			[]string{
-				"templates/other/nested/good0-*.tpl",
-				"templates/test/good0-*.tpl",
-			},
-			false,
-			mkTemplate(`
-				{{- define "good0-test.tpl" -}}{{- end -}}
-				{{- define "test.0" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.1" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.2" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.3" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-			`, nil),
-			"",
-		},
-		{
-			"existing path defined template override",
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{ printf "Hello %s!" "world" }}
-				{{- end -}}
-			`, nil),
-			[]string{
-				"templates/other/nested/good0-*.tpl",
-				"templates/test/good0-*.tpl",
-			},
-			false,
-			mkTemplate(`
-				{{- define "good0-test.tpl" -}}{{- end -}}
-				{{- define "test.0" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.1" -}}
-					{{ printf "Hello %s!" "world" }}
-				{{- end -}}
-				{{- define "test.2" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.3" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				`, `
-				{{- define "good0-test.tpl" -}}{{- end -}}
-				{{- define "test.0" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.1" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.2" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-				{{- define "test.3" -}}
-					{{ printf "Hello %s!" externalURL }}
-				{{- end -}}
-			`),
-			"",
-		},
-		{
-			"load template with syntax error",
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{ printf "Hello %s!" "world" }}
-				{{- end -}}
-			`, nil),
-			[]string{
-				"templates/other/nested/bad0-*.tpl",
-				"templates/test/good0-*.tpl",
-			},
-			false,
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{ printf "Hello %s!" "world" }}
-				{{- end -}}
-			`, nil),
-			"failed to parse template glob",
-		},
+func TestTemplatesLoad_Failure(t *testing.T) {
+	f := func(pathPatterns []string, expectedErrStr string) {
+		t.Helper()
+
+		err := Load(pathPatterns, false)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		errStr := err.Error()
+		if !strings.Contains(errStr, expectedErrStr) {
+			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
+		}
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			masterTmpl = tc.initialTemplate
-			err := Load(tc.pathPatterns, tc.overwrite)
-			if tc.expErr == "" && err != nil {
-				t.Error("happened error that wasn't expected: %w", err)
-			}
-			if tc.expErr != "" && err == nil {
-				t.Error("%+w", err)
-				t.Error("expected error that didn't happened")
-			}
-			if err != nil && !strings.Contains(err.Error(), tc.expErr) {
-				t.Error("%+w", err)
-				t.Error("expected string doesn't exist in error message")
-			}
-			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
-				t.Fatalf("replacement template is not as expected")
-			}
-			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
-				t.Fatalf("current template is not as expected")
-			}
-		})
-	}
+	// load template with syntax error
+	f([]string{
+		"templates/other/nested/bad0-*.tpl",
+		"templates/test/good0-*.tpl",
+	}, "failed to parse template glob")
 }
 
-func TestTemplates_Reload(t *testing.T) {
-	testCases := []struct {
-		name             string
-		initialTemplate  textTemplate
-		expectedTemplate textTemplate
-	}{
-		{
-			"empty current and replacement templates",
-			mkTemplate(nil, nil),
-			mkTemplate(nil, nil),
-		},
-		{
-			"empty current template only",
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{- printf "value" -}}
-				{{- end -}}
-			`, nil),
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{- printf "value" -}}
-				{{- end -}}
-			`, nil),
-		},
-		{
-			"empty replacement template only",
-			mkTemplate(nil, `
-				{{- define "test.1" -}}
-					{{- printf "value" -}}
-				{{- end -}}
-			`),
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{- printf "value" -}}
-				{{- end -}}
-			`, nil),
-		},
-		{
-			"defined both templates",
-			mkTemplate(`
-				{{- define "test.0" -}}
-					{{- printf "value" -}}
-				{{- end -}}
-				{{- define "test.1" -}}
-					{{- printf "before" -}}
-				{{- end -}}
-			`, `
-				{{- define "test.1" -}}
-					{{- printf "after" -}}
-				{{- end -}}
-			`),
-			mkTemplate(`
-				{{- define "test.1" -}}
-					{{- printf "after" -}}
-				{{- end -}}
-			`, nil),
-		},
+func TestTemplatesLoad_Success(t *testing.T) {
+	f := func(initialTmpl textTemplate, pathPatterns []string, overwrite bool, expectedTmpl textTemplate) {
+		t.Helper()
+
+		masterTmplOrig := masterTmpl
+		masterTmpl = initialTmpl
+		defer func() {
+			masterTmpl = masterTmplOrig
+		}()
+
+		if err := Load(pathPatterns, overwrite); err != nil {
+			t.Fatalf("cannot load templates: %s", err)
+		}
+
+		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
+			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
+		}
+		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
+			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
+		}
 	}
 
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			masterTmpl = tc.initialTemplate
-			Reload()
-			if !equalTemplates(masterTmpl.replacement, tc.expectedTemplate.replacement) {
-				t.Fatalf("replacement template is not as expected")
-			}
-			if !equalTemplates(masterTmpl.current, tc.expectedTemplate.current) {
-				t.Fatalf("current template is not as expected")
-			}
-		})
+	// non existing path undefined template override
+	initialTmpl := mkTemplate(nil, nil)
+	pathPatterns := []string{
+		"templates/non-existing/good-*.tpl",
+		"templates/absent/good-*.tpl",
 	}
+	overwrite := true
+	expectedTmpl := mkTemplate(``, nil)
+	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
+
+	// non existing path defined template override
+	initialTmpl = mkTemplate(`
+		{{- define "test.1" -}}
+			{{- printf "value" -}}
+		{{- end -}}
+	`, nil)
+	pathPatterns = []string{
+		"templates/non-existing/good-*.tpl",
+		"templates/absent/good-*.tpl",
+	}
+	overwrite = true
+	expectedTmpl = mkTemplate(``, nil)
+	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
+
+	// existing path undefined template override
+	initialTmpl = mkTemplate(nil, nil)
+	pathPatterns = []string{
+		"templates/other/nested/good0-*.tpl",
+		"templates/test/good0-*.tpl",
+	}
+	overwrite = false
+	expectedTmpl = mkTemplate(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.2" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`, nil)
+	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
+
+	// existing path defined template override
+	initialTmpl = mkTemplate(`
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" "world" }}
+		{{- end -}}
+	`, nil)
+	pathPatterns = []string{
+		"templates/other/nested/good0-*.tpl",
+		"templates/test/good0-*.tpl",
+	}
+	overwrite = false
+	expectedTmpl = mkTemplate(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" "world" }}
+		{{- end -}}
+		{{- define "test.2" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`, `
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.2" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`)
+	f(initialTmpl, pathPatterns, overwrite, expectedTmpl)
+}
+
+func TestTemplatesReload(t *testing.T) {
+	f := func(initialTmpl, expectedTmpl textTemplate) {
+		t.Helper()
+
+		masterTmplOrig := masterTmpl
+		masterTmpl = initialTmpl
+		defer func() {
+			masterTmpl = masterTmplOrig
+		}()
+
+		Reload()
+
+		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
+			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
+		}
+		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
+			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
+		}
+	}
+
+	// empty current and replacement templates
+	f(mkTemplate(nil, nil), mkTemplate(nil, nil))
+
+	// empty current template only
+	f(mkTemplate(`
+		{{- define "test.1" -}}
+			{{- printf "value" -}}
+		{{- end -}}
+	`, nil), mkTemplate(`
+		{{- define "test.1" -}}
+			{{- printf "value" -}}
+		{{- end -}}
+	`, nil))
+
+	// empty replacement template only
+	f(mkTemplate(nil, `
+		{{- define "test.1" -}}
+			{{- printf "value" -}}
+		{{- end -}}
+	`), mkTemplate(`
+		{{- define "test.1" -}}
+			{{- printf "value" -}}
+		{{- end -}}
+	`, nil))
+
+	// defined both templates
+	f(mkTemplate(`
+		{{- define "test.0" -}}
+			{{- printf "value" -}}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{- printf "before" -}}
+		{{- end -}}
+	`, `
+		{{- define "test.1" -}}
+			{{- printf "after" -}}
+		{{- end -}}
+	`), mkTemplate(`
+		{{- define "test.1" -}}
+			{{- printf "after" -}}
+		{{- end -}}
+	`, nil))
 }

app/vmalert/utils/err_group_test.go

@@ -7,35 +7,31 @@ import (
 )
 
 func TestErrGroup(t *testing.T) {
-	testCases := []struct {
-		errs []error
-		exp  string
-	}{
-		{nil, ""},
-		{[]error{errors.New("timeout")}, "errors(1): timeout"},
-		{
-			[]error{errors.New("timeout"), errors.New("deadline")},
-			"errors(2): timeout\ndeadline",
-		},
-	}
-	for _, tc := range testCases {
-		eg := new(ErrGroup)
-		for _, err := range tc.errs {
+	f := func(errs []error, resultExpected string) {
+		t.Helper()
+
+		eg := &ErrGroup{}
+		for _, err := range errs {
 			eg.Add(err)
 		}
-		if len(tc.errs) == 0 {
+		if len(errs) == 0 {
 			if eg.Err() != nil {
 				t.Fatalf("expected to get nil error")
 			}
-			continue
+			return
 		}
 		if eg.Err() == nil {
 			t.Fatalf("expected to get non-nil error")
 		}
-		if eg.Error() != tc.exp {
-			t.Fatalf("expected to have: \n%q\ngot:\n%q", tc.exp, eg.Error())
+		result := eg.Error()
+		if result != resultExpected {
+			t.Fatalf("unexpected result\ngot\n%v\nwant\n%v", result, resultExpected)
 		}
 	}
+
+	f(nil, "")
+	f([]error{errors.New("timeout")}, "errors(1): timeout")
+	f([]error{errors.New("timeout"), errors.New("deadline")}, "errors(2): timeout\ndeadline")
 }
 
 // TestErrGroupConcurrent supposed to test concurrent

app/vmalert/web.go

@@ -19,7 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
-var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+var reloadAuthKey = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 
 var (
 	apiLinks = [][2]string{
@@ -35,7 +35,7 @@ var (
 		{"-/reload", "reload configuration"},
 	}
 	navItems = []tpl.NavItem{
-		{Name: "vmalert", Url: "."},
+		{Name: "vmalert", Url: "../vmalert"},
 		{Name: "Groups", Url: "groups"},
 		{Name: "Alerts", Url: "alerts"},
 		{Name: "Notifiers", Url: "notifiers"},
@@ -167,7 +167,7 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		w.Write(data)
 		return true
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
 		}
 		logger.Infof("api config reload was called, sending sighup")

app/vmalert/web_test.go

@@ -36,23 +36,23 @@ func TestHandler(t *testing.T) {
 	}}
 	rh := &requestHandler{m: m}
 
-	getResp := func(t *testing.T, url string, to interface{}, code int) {
+	getResp := func(t *testing.T, url string, to any, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
 			t.Fatalf("unexpected err %s", err)
 		}
 		if code != resp.StatusCode {
-			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
+			t.Fatalf("unexpected status code %d want %d", resp.StatusCode, code)
 		}
 		defer func() {
 			if err := resp.Body.Close(); err != nil {
-				t.Errorf("err closing body %s", err)
+				t.Fatalf("err closing body %s", err)
 			}
 		}()
 		if to != nil {
 			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
-				t.Errorf("unexpected err %s", err)
+				t.Fatalf("unexpected err %s", err)
 			}
 		}
 	}
@@ -92,13 +92,13 @@ func TestHandler(t *testing.T) {
 		lr := listAlertsResponse{}
 		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
-			t.Errorf("expected 1 alert got %d", length)
+			t.Fatalf("expected 1 alert got %d", length)
 		}
 
 		lr = listAlertsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if length := len(lr.Data.Alerts); length != 1 {
-			t.Errorf("expected 1 alert got %d", length)
+			t.Fatalf("expected 1 alert got %d", length)
 		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
@@ -106,13 +106,13 @@ func TestHandler(t *testing.T) {
 		alert := &apiAlert{}
 		getResp(t, ts.URL+"/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
-			t.Errorf("expected %v is equal to %v", alert, expAlert)
+			t.Fatalf("expected %v is equal to %v", alert, expAlert)
 		}
 
 		alert = &apiAlert{}
 		getResp(t, ts.URL+"/vmalert/"+expAlert.APILink(), alert, 200)
 		if !reflect.DeepEqual(alert, expAlert) {
-			t.Errorf("expected %v is equal to %v", alert, expAlert)
+			t.Fatalf("expected %v is equal to %v", alert, expAlert)
 		}
 	})
 
@@ -135,13 +135,13 @@ func TestHandler(t *testing.T) {
 		lr := listGroupsResponse{}
 		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
-			t.Errorf("expected 1 group got %d", length)
+			t.Fatalf("expected 1 group got %d", length)
 		}
 
 		lr = listGroupsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if length := len(lr.Data.Groups); length != 1 {
-			t.Errorf("expected 1 group got %d", length)
+			t.Fatalf("expected 1 group got %d", length)
 		}
 	})
 	t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
@@ -150,14 +150,14 @@ func TestHandler(t *testing.T) {
 		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
-			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
+			t.Fatalf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRule = apiRule{}
 		getResp(t, ts.URL+"/vmalert/"+expRule.APILink(), &gotRule, 200)
 
 		if expRule.ID != gotRule.ID {
-			t.Errorf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
+			t.Fatalf("expected to get Rule %q; got %q instead", expRule.ID, gotRule.ID)
 		}
 
 		gotRuleWithUpdates := apiRuleWithUpdates{}
@@ -173,7 +173,7 @@ func TestHandler(t *testing.T) {
 			lr := listGroupsResponse{}
 			getResp(t, ts.URL+url, &lr, 200)
 			if length := len(lr.Data.Groups); length != expGroups {
-				t.Errorf("expected %d groups got %d", expGroups, length)
+				t.Fatalf("expected %d groups got %d", expGroups, length)
 			}
 			if len(lr.Data.Groups) < 1 {
 				return
@@ -183,7 +183,7 @@ func TestHandler(t *testing.T) {
 				rulesN += len(gr.Rules)
 			}
 			if rulesN != expRules {
-				t.Errorf("expected %d rules got %d", expRules, rulesN)
+				t.Fatalf("expected %d rules got %d", expRules, rulesN)
 			}
 		}
 
@@ -241,23 +241,23 @@ func TestEmptyResponse(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rhWithNoGroups.handler(w, r) }))
 	defer ts.Close()
 
-	getResp := func(t *testing.T, url string, to interface{}, code int) {
+	getResp := func(t *testing.T, url string, to any, code int) {
 		t.Helper()
 		resp, err := http.Get(url)
 		if err != nil {
 			t.Fatalf("unexpected err %s", err)
 		}
 		if code != resp.StatusCode {
-			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
+			t.Fatalf("unexpected status code %d want %d", resp.StatusCode, code)
 		}
 		defer func() {
 			if err := resp.Body.Close(); err != nil {
-				t.Errorf("err closing body %s", err)
+				t.Fatalf("err closing body %s", err)
 			}
 		}()
 		if to != nil {
 			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
-				t.Errorf("unexpected err %s", err)
+				t.Fatalf("unexpected err %s", err)
 			}
 		}
 	}
@@ -266,13 +266,13 @@ func TestEmptyResponse(t *testing.T) {
 		lr := listAlertsResponse{}
 		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
-			t.Errorf("expected /api/v1/alerts response to have non-nil data")
+			t.Fatalf("expected /api/v1/alerts response to have non-nil data")
 		}
 
 		lr = listAlertsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
 		if lr.Data.Alerts == nil {
-			t.Errorf("expected /api/v1/alerts response to have non-nil data")
+			t.Fatalf("expected /api/v1/alerts response to have non-nil data")
 		}
 	})
 
@@ -280,13 +280,13 @@ func TestEmptyResponse(t *testing.T) {
 		lr := listGroupsResponse{}
 		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
-			t.Errorf("expected /api/v1/rules response to have non-nil data")
+			t.Fatalf("expected /api/v1/rules response to have non-nil data")
 		}
 
 		lr = listGroupsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
 		if lr.Data.Groups == nil {
-			t.Errorf("expected /api/v1/rules response to have non-nil data")
+			t.Fatalf("expected /api/v1/rules response to have non-nil data")
 		}
 	})
 

app/vmalert/web_types.go

@@ -183,7 +183,7 @@ func (ar apiRule) WebLink() string {
 		paramGroupID, ar.GroupID, paramRuleID, ar.ID)
 }
 
-func ruleToAPI(r interface{}) apiRule {
+func ruleToAPI(r any) apiRule {
 	if ar, ok := r.(*rule.AlertingRule); ok {
 		return alertingToAPI(ar)
 	}

app/vmalert/web_types_test.go

@@ -13,11 +13,11 @@ func TestUrlValuesToStrings(t *testing.T) {
 	res := urlValuesToStrings(mapQueryParams)
 
 	if len(res) != len(expectedRes) {
-		t.Errorf("Expected length %d, but got %d", len(expectedRes), len(res))
+		t.Fatalf("Expected length %d, but got %d", len(expectedRes), len(res))
 	}
 	for ind, val := range expectedRes {
 		if val != res[ind] {
-			t.Errorf("Expected %v; but got %v", val, res[ind])
+			t.Fatalf("Expected %v; but got %v", val, res[ind])
 		}
 	}
 }

app/vmauth/auth_config.go

@@ -83,7 +83,6 @@ type UserInfo struct {
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
-	overrideHostHeader      bool
 
 	rt http.RoundTripper
 
@@ -94,8 +93,9 @@ type UserInfo struct {
 
 // HeadersConf represents config for request and response headers.
 type HeadersConf struct {
-	RequestHeaders  []*Header `yaml:"headers,omitempty"`
-	ResponseHeaders []*Header `yaml:"response_headers,omitempty"`
+	RequestHeaders   []*Header `yaml:"headers,omitempty"`
+	ResponseHeaders  []*Header `yaml:"response_headers,omitempty"`
+	KeepOriginalHost *bool     `yaml:"keep_original_host,omitempty"`
 }
 
 func (ui *UserInfo) beginConcurrencyLimit() error {
@@ -129,7 +129,7 @@ type Header struct {
 }
 
 // UnmarshalYAML unmarshals h from f.
-func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
+func (h *Header) UnmarshalYAML(f func(any) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@@ -146,19 +146,10 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
 }
 
 // MarshalYAML marshals h to yaml.
-func (h *Header) MarshalYAML() (interface{}, error) {
+func (h *Header) MarshalYAML() (any, error) {
 	return h.sOriginal, nil
 }
 
-func overrideHostHeader(headers []*Header) bool {
-	for _, h := range headers {
-		if h.Name == "Host" && h.Value == "" {
-			return true
-		}
-	}
-	return false
-}
-
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
 	// SrcPaths is an optional list of regular expressions, which must match the request path.
@@ -201,7 +192,7 @@ type QueryArg struct {
 }
 
 // UnmarshalYAML unmarshals qa from yaml.
-func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
+func (qa *QueryArg) UnmarshalYAML(f func(any) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@@ -230,7 +221,7 @@ func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
 }
 
 // MarshalYAML marshals qa to yaml.
-func (qa *QueryArg) MarshalYAML() (interface{}, error) {
+func (qa *QueryArg) MarshalYAML() (any, error) {
 	return qa.sOriginal, nil
 }
 
@@ -263,7 +254,7 @@ type URLPrefix struct {
 	nextDiscoveryDeadline atomic.Uint64
 
 	// vOriginal contains the original yaml value for URLPrefix.
-	vOriginal interface{}
+	vOriginal any
 }
 
 func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
@@ -310,12 +301,18 @@ func (up *URLPrefix) getBackendsCount() int {
 
 // getBackendURL returns the backendURL depending on the load balance policy.
 //
+// It can return nil if there are no backend urls available at the moment.
+//
 // backendURL.put() must be called on the returned backendURL after the request is complete.
 func (up *URLPrefix) getBackendURL() *backendURL {
 	up.discoverBackendAddrsIfNeeded()
 
 	pbus := up.bus.Load()
 	bus := *pbus
+	if len(bus) == 0 {
+		return nil
+	}
+
 	if up.loadBalancingPolicy == "first_available" {
 		return getFirstAvailableBackendURL(bus)
 	}
@@ -354,11 +351,12 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 			// ips for the given host have been already discovered
 			continue
 		}
+
 		var resolvedAddrs []string
 		if strings.HasPrefix(host, "srv+") {
 			// The host has the format 'srv+realhost'. Strip 'srv+' prefix before performing the lookup.
-			host = strings.TrimPrefix(host, "srv+")
-			_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", host)
+			srvHost := strings.TrimPrefix(host, "srv+")
+			_, addrs, err := netutil.Resolver.LookupSRV(ctx, "", "", srvHost)
 			if err != nil {
 				logger.Warnf("cannot discover backend SRV records for %s: %s; use it literally", bu, err)
 				resolvedAddrs = []string{host}
@@ -390,14 +388,13 @@ func (up *URLPrefix) discoverBackendAddrsIfNeeded() {
 	var busNew []*backendURL
 	for _, bu := range up.busOriginal {
 		host := bu.Hostname()
-		host = strings.TrimPrefix(host, "srv+")
 		port := bu.Port()
 		for _, addr := range hostToAddrs[host] {
 			buCopy := *bu
 			buCopy.Host = addr
 			if port != "" {
 				if n := strings.IndexByte(buCopy.Host, ':'); n >= 0 {
-					// Drop the discovered port and substitute it the the port specified in bu.
+					// Drop the discovered port and substitute it the port specified in bu.
 					buCopy.Host = buCopy.Host[:n]
 				}
 				buCopy.Host += ":" + port
@@ -464,7 +461,7 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 	}
 
 	// Slow path - select other backend urls.
-	n := atomicCounter.Add(1)
+	n := atomicCounter.Add(1) - 1
 
 	for i := uint32(0); i < uint32(len(bus)); i++ {
 		idx := (n + i) % uint32(len(bus))
@@ -497,8 +494,8 @@ func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *
 }
 
 // UnmarshalYAML unmarshals up from yaml.
-func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
-	var v interface{}
+func (up *URLPrefix) UnmarshalYAML(f func(any) error) error {
+	var v any
 	if err := f(&v); err != nil {
 		return err
 	}
@@ -508,7 +505,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 	switch x := v.(type) {
 	case string:
 		urls = []string{x}
-	case []interface{}:
+	case []any:
 		if len(x) == 0 {
 			return fmt.Errorf("`url_prefix` must contain at least a single url")
 		}
@@ -538,7 +535,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
 }
 
 // MarshalYAML marshals up to yaml.
-func (up *URLPrefix) MarshalYAML() (interface{}, error) {
+func (up *URLPrefix) MarshalYAML() (any, error) {
 	return up.vOriginal, nil
 }
 
@@ -562,7 +559,7 @@ func (r *Regex) match(s string) bool {
 }
 
 // UnmarshalYAML implements yaml.Unmarshaler
-func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
+func (r *Regex) UnmarshalYAML(f func(any) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@@ -579,7 +576,7 @@ func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
 }
 
 // MarshalYAML implements yaml.Marshaler.
-func (r *Regex) MarshalYAML() (interface{}, error) {
+func (r *Regex) MarshalYAML() (any, error) {
 	return r.sOriginal, nil
 }
 
@@ -600,7 +597,7 @@ func initAuthConfig() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	_, err := loadAuthConfig()
+	_, err := reloadAuthConfig()
 	if err != nil {
 		logger.Fatalf("cannot load auth config: %s", err)
 	}
@@ -632,7 +629,7 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 
 	updateFn := func() {
 		configReloads.Inc()
-		updated, err := loadAuthConfig()
+		updated, err := reloadAuthConfig()
 		if err != nil {
 			logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
 			configSuccess.Set(0)
@@ -658,27 +655,45 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 	}
 }
 
-// authConfigData stores the yaml definition for this config.
-// authConfigData needs to be updated each time authConfig is updated.
-var authConfigData atomic.Pointer[[]byte]
-
 var (
-	authConfig   atomic.Pointer[AuthConfig]
-	authUsers    atomic.Pointer[map[string]*UserInfo]
+	// authConfigData stores the yaml definition for this config.
+	// authConfigData needs to be updated each time authConfig is updated.
+	authConfigData atomic.Pointer[[]byte]
+
+	// authConfig contains the currently loaded auth config
+	authConfig atomic.Pointer[AuthConfig]
+
+	// authUsers contains the currently loaded auth users
+	authUsers atomic.Pointer[map[string]*UserInfo]
+
 	authConfigWG sync.WaitGroup
 	stopCh       chan struct{}
 )
 
-// loadAuthConfig loads and applies the config from *authConfigPath.
+// reloadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
 // The config can be not applied if there is a parsing error
 // or if there are no changes to the current authConfig.
-func loadAuthConfig() (bool, error) {
+func reloadAuthConfig() (bool, error) {
 	data, err := fscore.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
 		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
 	}
 
+	ok, err := reloadAuthConfigData(data)
+	if err != nil {
+		return false, fmt.Errorf("failed to pars -auth.config=%q: %w", *authConfigPath, err)
+	}
+	if !ok {
+		return false, nil
+	}
+
+	mp := authUsers.Load()
+	logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
+	return true, nil
+}
+
+func reloadAuthConfigData(data []byte) (bool, error) {
 	oldData := authConfigData.Load()
 	if oldData != nil && bytes.Equal(data, *oldData) {
 		// there are no updates in the config - skip reloading.
@@ -687,31 +702,23 @@ func loadAuthConfig() (bool, error) {
 
 	ac, err := parseAuthConfig(data)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse auth config: %w", err)
 	}
 
 	m, err := parseAuthConfigUsers(ac)
 	if err != nil {
-		return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse users from auth config: %w", err)
 	}
-	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
-	prevAc := authConfig.Load()
-	if prevAc != nil {
-		metrics.UnregisterSet(prevAc.ms)
+	acPrev := authConfig.Load()
+	if acPrev != nil {
+		metrics.UnregisterSet(acPrev.ms, true)
 	}
 	metrics.RegisterSet(ac.ms)
+
 	authConfig.Store(ac)
 	authConfigData.Store(&data)
 	authUsers.Store(&m)
-	if prevAc != nil {
-		// explicilty unregister metrics, since all summary type metrics
-		// are registered at global state of metrics package
-		// and must be removed from it to release memory.
-		// Metrics must be unregistered only after atomic.Value.Store calls above
-		// Otherwise it may lead to metric gaps, since UnregisterAllMetrics is slow operation
-		prevAc.ms.UnregisterAllMetrics()
-	}
 
 	return true, nil
 }
@@ -748,7 +755,6 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
 
 		metricLabels, err := ui.getMetricLabels()
 		if err != nil {
@@ -813,7 +819,6 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		_ = ac.ms.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
-		ui.overrideHostHeader = overrideHostHeader(ui.HeadersConf.RequestHeaders)
 
 		rt, err := newRoundTripper(ui.TLSCAFile, ui.TLSCertFile, ui.TLSKeyFile, ui.TLSServerName, ui.TLSInsecureSkipVerify)
 		if err != nil {
@@ -1004,6 +1009,14 @@ func getAuthTokensFromRequest(r *http.Request) []string {
 		}
 	}
 
+	// Authorization via http://user:pass@hosname/path
+	if u := r.URL.User; u != nil && u.Username() != "" {
+		username := u.Username()
+		password, _ := u.Password()
+		at := getHTTPAuthBasicToken(username, password)
+		ats = append(ats, at)
+	}
+
 	return ats
 }
 
@@ -1024,17 +1037,11 @@ func (up *URLPrefix) sanitizeAndInitialize() error {
 		}
 	}
 	up.bus.Store(&bus)
-	up.nextDiscoveryDeadline.Store(0)
-	up.n.Store(0)
 
 	return nil
 }
 
 func sanitizeURLPrefix(urlPrefix *url.URL) (*url.URL, error) {
-	// Remove trailing '/' from urlPrefix
-	for strings.HasSuffix(urlPrefix.Path, "/") {
-		urlPrefix.Path = urlPrefix.Path[:len(urlPrefix.Path)-1]
-	}
 	// Validate urlPrefix
 	if urlPrefix.Scheme != "http" && urlPrefix.Scheme != "https" {
 		return nil, fmt.Errorf("unsupported scheme for `url_prefix: %q`: %q; must be `http` or `https`", urlPrefix, urlPrefix.Scheme)

app/vmauth/auth_config_test.go

@@ -82,6 +82,14 @@ users:
   headers: foobar
 `)
 
+	// Invalid keep_original_host value
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  keep_original_host: foobar
+`)
+
 	// empty url_prefix
 	f(`
 users:
@@ -410,7 +418,7 @@ users:
 				HeadersConf: HeadersConf{
 					RequestHeaders: []*Header{
 						mustNewHeader("'foo: bar'"),
-						mustNewHeader("'xxx: y'"),
+						mustNewHeader("'xxx:'"),
 					},
 				},
 			},
@@ -429,7 +437,7 @@ users:
     url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
     headers:
     - "foo: bar"
-    - "xxx: y"
+    - "xxx:"
 `, map[string]*UserInfo{
 		getHTTPAuthBearerToken("foo"):    sharedUserInfo,
 		getHTTPAuthBasicToken("foo", ""): sharedUserInfo,
@@ -458,6 +466,7 @@ users:
 	})
 
 	// with default url
+	keepOriginalHost := true
 	f(`
 users:
 - bearer_token: foo
@@ -469,6 +478,7 @@ users:
     headers:
     - "foo: bar"
     - "xxx: y"
+    keep_original_host: true
   default_url:
   - http://default1/select/0/prometheus
   - http://default2/select/0/prometheus
@@ -491,6 +501,7 @@ users:
 							mustNewHeader("'foo: bar'"),
 							mustNewHeader("'xxx: y'"),
 						},
+						KeepOriginalHost: &keepOriginalHost,
 					},
 				},
 			},
@@ -517,6 +528,7 @@ users:
 							mustNewHeader("'foo: bar'"),
 							mustNewHeader("'xxx: y'"),
 						},
+						KeepOriginalHost: &keepOriginalHost,
 					},
 				},
 			},
@@ -536,12 +548,17 @@ users:
   metric_labels:
     dc: eu
     team: dev
+  keep_original_host: true
 - username: foo-same
   password: bar
   url_prefix: https://bar/x
   metric_labels:
     backend_env: test
     team: accounting
+  headers:
+  - "foo: bar"
+  response_headers:
+  - "Abc: def"
 `, map[string]*UserInfo{
 		getHTTPAuthBasicToken("foo-same", "baz"): {
 			Username:  "foo-same",
@@ -551,6 +568,9 @@ users:
 				"dc":   "eu",
 				"team": "dev",
 			},
+			HeadersConf: HeadersConf{
+				KeepOriginalHost: &keepOriginalHost,
+			},
 		},
 		getHTTPAuthBasicToken("foo-same", "bar"): {
 			Username:  "foo-same",
@@ -560,6 +580,14 @@ users:
 				"backend_env": "test",
 				"team":        "accounting",
 			},
+			HeadersConf: HeadersConf{
+				RequestHeaders: []*Header{
+					mustNewHeader("'foo: bar'"),
+				},
+				ResponseHeaders: []*Header{
+					mustNewHeader("'Abc: def'"),
+				},
+			},
 		},
 	})
 }
@@ -695,9 +723,9 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 
 	fn := func(ns ...int) {
 		t.Helper()
-		bus := up.bus.Load()
-		pbus := *bus
-		for i, b := range pbus {
+		pbus := up.bus.Load()
+		bus := *pbus
+		for i, b := range bus {
 			got := int(b.concurrentRequests.Load())
 			exp := ns[i]
 			if got != exp {
@@ -707,39 +735,39 @@ func TestGetLeastLoadedBackendURL(t *testing.T) {
 	}
 
 	up.getBackendURL()
-	fn(0, 1, 0)
+	fn(1, 0, 0)
 	up.getBackendURL()
-	fn(0, 1, 1)
+	fn(1, 1, 0)
 	up.getBackendURL()
 	fn(1, 1, 1)
 
 	up.getBackendURL()
 	up.getBackendURL()
-	fn(1, 2, 2)
+	fn(2, 2, 1)
 
 	bus := up.bus.Load()
 	pbus := *bus
 	pbus[0].concurrentRequests.Add(2)
 	pbus[2].concurrentRequests.Add(5)
-	fn(3, 2, 7)
+	fn(4, 2, 6)
 
 	up.getBackendURL()
-	fn(3, 3, 7)
+	fn(4, 3, 6)
 
 	up.getBackendURL()
-	fn(3, 4, 7)
+	fn(4, 4, 6)
 
 	up.getBackendURL()
-	fn(4, 4, 7)
+	fn(4, 5, 6)
 
 	up.getBackendURL()
-	fn(5, 4, 7)
+	fn(5, 5, 6)
 
 	up.getBackendURL()
-	fn(5, 5, 7)
+	fn(6, 5, 6)
 
 	up.getBackendURL()
-	fn(6, 5, 7)
+	fn(6, 6, 6)
 
 	up.getBackendURL()
 	fn(6, 6, 7)

app/vmauth/main.go

@@ -37,20 +37,20 @@ var (
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
 		"See also -maxConcurrentRequests")
-	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, `Defines a duration for idle (keep-alive connections) to exist.
-    Consider setting this value less than "-http.idleConnTimeout". It must prevent possible "write: broken pipe" and "read: connection reset by peer" errors.`)
+	idleConnTimeout = flag.Duration("idleConnTimeout", 50*time.Second, "The timeout for HTTP keep-alive connections to backend services. "+
+		"It is recommended setting this value to values smaller than -http.idleConnTimeout set at backend services")
 	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
 	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
 		"'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options")
 	maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+
 		"Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+
 		"in per-user config")
-	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings.")
+	reloadAuthKey        = flagutil.NewPassword("reloadAuthKey", "Auth key for /-/reload http endpoint. It must be passed via authKey query arg. It overrides -httpAuth.*")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
 	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
-		"Bigger values may require more memory. Negative or zero values disable request body caching and retries.")
+		"Bigger values may require more memory. Zero or negative value disables caching of request body. This may be useful when proxying data ingestion requests")
 	backendTLSInsecureSkipVerify = flag.Bool("backend.tlsInsecureSkipVerify", false, "Whether to skip TLS verification when connecting to backends over HTTPS. "+
 		"See https://docs.victoriametrics.com/vmauth/#backend-tls-setup")
 	backendTLSCAFile = flag.String("backend.TLSCAFile", "", "Optional path to TLS root CA file, which is used for TLS verification when connecting to backends over HTTPS. "+
@@ -99,7 +99,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey.Get(), "reloadAuthKey") {
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
 			return true
 		}
 		configReloadRequests.Inc()
@@ -117,8 +117,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 
-		w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-		http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
+		handleMissingAuthorizationError(w)
 		return true
 	}
 
@@ -189,26 +188,27 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			// to a route that is not in the configuration for unauthorized user.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5236
 			if ui.BearerToken == "" && ui.Username == "" && len(*authUsers.Load()) > 0 {
-				w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-				http.Error(w, "missing `Authorization` request header", http.StatusUnauthorized)
+				handleMissingAuthorizationError(w)
 				return
 			}
 			missingRouteRequests.Inc()
-			httpserver.Errorf(w, r, "missing route for %q", u.String())
+			httpserver.Errorf(w, r, "missing route for %s", u.String())
 			return
 		}
 		up, hc = ui.DefaultURL, ui.HeadersConf
 		isDefault = true
 	}
-	// caching makes sense only for positive non zero size
-	if maxRequestBodySizeToRetry.IntN() > 0 {
-		rtb := getReadTrackingBody(r.Body, int(r.ContentLength))
-		defer putReadTrackingBody(rtb)
-		r.Body = rtb
-	}
+
+	rtb := getReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
+	defer putReadTrackingBody(rtb)
+	r.Body = rtb
+
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getBackendURL()
+		if bu == nil {
+			break
+		}
 		targetURL := bu.url
 		// Don't change path and add request_path query param for default route.
 		if isDefault {
@@ -218,7 +218,15 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+
+		wasLocalRetry := false
+	again:
+		ok, needLocalRetry := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
+		if needLocalRetry && !wasLocalRetry {
+			wasLocalRetry = true
+			goto again
+		}
+
 		bu.put()
 		if ok {
 			return
@@ -226,29 +234,28 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		bu.setBroken()
 	}
 	err := &httpserver.ErrorWithStatusCode{
-		Err:        fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
-		StatusCode: http.StatusServiceUnavailable,
+		Err:        fmt.Errorf("all the %d backends for the user %q are unavailable", up.getBackendsCount(), ui.name()),
+		StatusCode: http.StatusBadGateway,
 	}
 	httpserver.Errorf(w, r, "%s", err)
 	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
-	// This code has been copied from net/http/httputil/reverseproxy.go
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) (bool, bool) {
 	req := sanitizeRequestHeaders(r)
-	req.URL = targetURL
 
-	if req.URL.Scheme == "https" || ui.overrideHostHeader {
-		// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
-		// so it expects the targetURL.Host in the request.
-		// There is no need in overriding the req.Host for http requests, since it is expected that backend server
-		// may properly process queries with the original req.Host.
-		req.Host = targetURL.Host
-	}
+	req.URL = targetURL
+	req.Header.Set("User-Agent", "vmauth")
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	var trivialRetries int
+	if hc.KeepOriginalHost == nil || !*hc.KeepOriginalHost {
+		if host := getHostHeader(hc.RequestHeaders); host != "" {
+			req.Host = host
+		} else {
+			req.Host = targetURL.Host
+		}
+	}
+
 	rtb, rtbOK := req.Body.(*readTrackingBody)
-again:
 	res, err := ui.rt.RoundTrip(req)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -260,7 +267,7 @@ again:
 				// Timed out request must be counted as errors, since this usually means that the backend is slow.
 				ui.backendErrors.Inc()
 			}
-			return true
+			return true, false
 		}
 		if !rtbOK || !rtb.canRetry() {
 			// Request body cannot be re-sent to another backend. Return the error to the client then.
@@ -270,20 +277,19 @@ again:
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
-			return true
+			return true, false
 		}
-		// one time retry trivial network errors, such as proxy idle timeout misconfiguration
-		// or socket close by OS
-		if (netutil.IsTrivialNetworkError(err) || errors.Is(err, io.EOF)) && trivialRetries < 1 {
-			trivialRetries++
-			goto again
+		if netutil.IsTrivialNetworkError(err) {
+			// Retry request at the same backend on trivial network errors, such as proxy idle timeout misconfiguration or socket close by OS
+			return false, true
 		}
+
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
 		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
-		return false
+		return false, false
 	}
 	if slices.Contains(retryStatusCodes, res.StatusCode) {
 		_ = res.Body.Close()
@@ -297,7 +303,7 @@ again:
 			}
 			httpserver.Errorf(w, r, "%s", err)
 			ui.backendErrors.Inc()
-			return true
+			return true, false
 		}
 		// Retry requests at other backends if it matches retryStatusCodes.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
@@ -306,7 +312,7 @@ again:
 		// it explicitly reads request body, which may fail retries.
 		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
 			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
-		return false
+		return false, false
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
@@ -322,9 +328,9 @@ again:
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		requestURI := httpserver.GetRequestURI(r)
 		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
-		return true
+		return true, false
 	}
-	return true
+	return true, false
 }
 
 var copyBufPool bytesutil.ByteBufferPool
@@ -337,12 +343,21 @@ func copyHeader(dst, src http.Header) {
 	}
 }
 
-func updateHeadersByConfig(headers http.Header, config []*Header) {
-	for _, h := range config {
+func getHostHeader(headers []*Header) string {
+	for _, h := range headers {
+		if h.Name == "Host" {
+			return h.Value
+		}
+	}
+	return ""
+}
+
+func updateHeadersByConfig(dst http.Header, src []*Header) {
+	for _, h := range src {
 		if h.Value == "" {
-			headers.Del(h.Name)
+			dst.Del(h.Name)
 		} else {
-			headers.Set(h.Name, h.Value)
+			dst.Set(h.Name, h.Value)
 		}
 	}
 }
@@ -450,7 +465,7 @@ func newRoundTripper(caFileOpt, certFileOpt, keyFileOpt, serverNameOpt string, i
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
-	tr.DialContext = netutil.DialMaybeSRV
+	tr.DialContext = netutil.NewStatDialFunc("vmauth_backend")
 
 	rt := cfg.NewRoundTripper(tr)
 	return rt, nil
@@ -482,6 +497,11 @@ See the docs at https://docs.victoriametrics.com/vmauth/ .
 	flagutil.Usage(s)
 }
 
+func handleMissingAuthorizationError(w http.ResponseWriter) {
+	w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
+	http.Error(w, "missing 'Authorization' request header", http.StatusUnauthorized)
+}
+
 func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err error) {
 	w.Header().Add("Retry-After", "10")
 	err = &httpserver.ErrorWithStatusCode{
@@ -491,85 +511,124 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 	httpserver.Errorf(w, r, "%s", err)
 }
 
+// readTrackingBody must be obtained via getReadTrackingBody()
 type readTrackingBody struct {
+	// maxBodySize is the maximum body size to cache in buf.
+	//
+	// Bigger bodies cannot be retried.
+	maxBodySize int
+
 	// r contains reader for initial data reading
 	r io.ReadCloser
 
-	// buf is a buffer for data read from r. Buf size is limited by maxRequestBodySizeToRetry.
-	// If more than maxRequestBodySizeToRetry is read from r, then cannotRetry is set to true.
+	// buf is a buffer for data read from r. Buf size is limited by maxBodySize.
+	// If more than maxBodySize is read from r, then cannotRetry is set to true.
 	buf []byte
 
-	// cannotRetry is set to true when more than maxRequestBodySizeToRetry are read from r.
+	// readBuf points to the cached data at buf, which must be read in the next call to Read().
+	readBuf []byte
+
+	// cannotRetry is set to true when more than maxBodySize bytes are read from r.
 	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
 	cannotRetry bool
 
 	// bufComplete is set to true when buf contains complete request body read from r.
 	bufComplete bool
-
-	// offset is an offset at buf for the next data read if needReadBuf is set to true.
-	offset int
 }
 
-// Read implements io.Reader interface
-// tracks body reading requests
+func (rtb *readTrackingBody) reset() {
+	rtb.maxBodySize = 0
+	rtb.r = nil
+	rtb.buf = rtb.buf[:0]
+	rtb.readBuf = nil
+	rtb.cannotRetry = false
+	rtb.bufComplete = false
+}
+
+func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
+	v := readTrackingBodyPool.Get()
+	if v == nil {
+		v = &readTrackingBody{}
+	}
+	rtb := v.(*readTrackingBody)
+
+	if maxBodySize < 0 {
+		maxBodySize = 0
+	}
+	rtb.maxBodySize = maxBodySize
+
+	if r == nil {
+		// This is GET request without request body
+		r = (*zeroReader)(nil)
+	}
+	rtb.r = r
+	return rtb
+}
+
+type zeroReader struct{}
+
+func (r *zeroReader) Read(_ []byte) (int, error) {
+	return 0, io.EOF
+}
+func (r *zeroReader) Close() error {
+	return nil
+}
+
+func putReadTrackingBody(rtb *readTrackingBody) {
+	rtb.reset()
+	readTrackingBodyPool.Put(rtb)
+}
+
+var readTrackingBodyPool sync.Pool
+
+// Read implements io.Reader interface.
 func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if rtb.offset < len(rtb.buf) {
-		if rtb.cannotRetry {
-			return 0, fmt.Errorf("cannot retry reading data from buf")
-		}
-		nb := copy(p, rtb.buf[rtb.offset:])
-		rtb.offset += nb
-		if rtb.bufComplete {
-			if rtb.offset == len(rtb.buf) {
-				return nb, io.EOF
-			}
-			return nb, nil
-		}
-		if nb < len(p) {
-			nr, err := rtb.readFromStream(p[nb:])
-			return nb + nr, err
-		}
-		return nb, nil
+	if len(rtb.readBuf) > 0 {
+		n := copy(p, rtb.readBuf)
+		rtb.readBuf = rtb.readBuf[n:]
+		return n, nil
 	}
-	if rtb.bufComplete {
-		return 0, io.EOF
-	}
-	return rtb.readFromStream(p)
-}
 
-func (rtb *readTrackingBody) readFromStream(p []byte) (int, error) {
 	if rtb.r == nil {
-		return 0, fmt.Errorf("cannot read data after closing the reader")
+		if rtb.bufComplete {
+			return 0, io.EOF
+		}
+		return 0, fmt.Errorf("cannot read client request body after closing client reader")
 	}
+
 	n, err := rtb.r.Read(p)
 	if rtb.cannotRetry {
 		return n, err
 	}
-	if rtb.offset+n > maxRequestBodySizeToRetry.IntN() {
-		rtb.cannotRetry = true
-	}
-	if n > 0 {
-		rtb.offset += n
-		rtb.buf = append(rtb.buf, p[:n]...)
-	}
-	if err != nil {
-		if err == io.EOF {
-			rtb.bufComplete = true
-			return n, err
-		}
+
+	if len(rtb.buf)+n > rtb.maxBodySize {
 		rtb.cannotRetry = true
 		return n, err
 	}
-	return n, nil
+	rtb.buf = append(rtb.buf, p[:n]...)
+	if err == io.EOF {
+		rtb.bufComplete = true
+	}
+	return n, err
 }
 
 func (rtb *readTrackingBody) canRetry() bool {
-	return !rtb.cannotRetry
+	if rtb.cannotRetry {
+		return false
+	}
+	if rtb.bufComplete {
+		return true
+	}
+	return rtb.r != nil
 }
 
 // Close implements io.Closer interface.
 func (rtb *readTrackingBody) Close() error {
-	rtb.offset = 0
+	if !rtb.cannotRetry {
+		rtb.readBuf = rtb.buf
+	} else {
+		rtb.readBuf = nil
+	}
 
 	// Close rtb.r only if the request body is completely read or if it is too big.
 	// http.Roundtrip performs body.Close call even without any Read calls,
@@ -585,38 +644,3 @@ func (rtb *readTrackingBody) Close() error {
 
 	return nil
 }
-
-var readTrackingBodyPool sync.Pool
-
-func getReadTrackingBody(origin io.ReadCloser, b int) *readTrackingBody {
-	bufSize := 1024
-	if b > 0 && b < maxRequestBodySizeToRetry.IntN() {
-		bufSize = b
-	}
-	v := readTrackingBodyPool.Get()
-	if v == nil {
-		v = &readTrackingBody{
-			buf: make([]byte, 0, bufSize),
-		}
-	}
-	rtb := v.(*readTrackingBody)
-	rtb.r = origin
-	if bufSize > cap(rtb.buf) {
-		rtb.buf = make([]byte, 0, bufSize)
-	}
-
-	return rtb
-}
-
-func putReadTrackingBody(rtb *readTrackingBody) {
-	if rtb.r != nil {
-		_ = rtb.r.Close()
-	}
-	rtb.r = nil
-	rtb.buf = rtb.buf[:0]
-	rtb.offset = 0
-	rtb.cannotRetry = false
-	rtb.bufComplete = false
-
-	readTrackingBodyPool.Put(rtb)
-}

app/vmauth/main_test.go

@@ -2,18 +2,520 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
-func TestReadTrackingBodyRetrySuccess(t *testing.T) {
-	f := func(s string) {
+func TestRequestHandler(t *testing.T) {
+	f := func(cfgStr, requestURL string, backendHandler http.HandlerFunc, responseExpected string) {
 		t.Helper()
-		rtb := &readTrackingBody{
-			r: io.NopCloser(bytes.NewBufferString(s)),
+
+		ts := httptest.NewServer(backendHandler)
+		defer ts.Close()
+
+		cfgStr = strings.ReplaceAll(cfgStr, "{BACKEND}", ts.URL)
+		responseExpected = strings.ReplaceAll(responseExpected, "{BACKEND}", ts.URL)
+
+		cfgOrigP := authConfigData.Load()
+		if _, err := reloadAuthConfigData([]byte(cfgStr)); err != nil {
+			t.Fatalf("cannot load config data: %s", err)
 		}
+		defer func() {
+			cfgOrig := []byte("unauthorized_user:\n  url_prefix: http://foo/bar")
+			if cfgOrigP != nil {
+				cfgOrig = *cfgOrigP
+			}
+			_, err := reloadAuthConfigData(cfgOrig)
+			if err != nil {
+				t.Fatalf("cannot load the original config: %s", err)
+			}
+		}()
+
+		r, err := http.NewRequest(http.MethodGet, requestURL, nil)
+		if err != nil {
+			t.Fatalf("cannot initialize http request: %s", err)
+		}
+
+		r.RequestURI = r.URL.RequestURI()
+		r.RemoteAddr = "42.2.3.84:6789"
+		r.Header.Set("X-Forwarded-For", "12.34.56.78")
+		r.Header.Set("Connection", "Some-Header,Other-Header")
+		r.Header.Set("Some-Header", "foobar")
+		r.Header.Set("Pass-Header", "abc")
+
+		w := &fakeResponseWriter{}
+		if !requestHandler(w, r) {
+			t.Fatalf("unexpected false is returned from requestHandler")
+		}
+
+		response := w.getResponse()
+		response = strings.ReplaceAll(response, "\r\n", "\n")
+		response = strings.TrimSpace(response)
+		responseExpected = strings.TrimSpace(responseExpected)
+		if response != responseExpected {
+			t.Fatalf("unexpected response\ngot\n%s\nwant\n%s", response, responseExpected)
+		}
+	}
+
+	// regular url_prefix
+	cfgStr := `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz`
+	requestURL := "http://some-host.com/abc/def?some_arg=some_value"
+	backendHandler := func(w http.ResponseWriter, r *http.Request) {
+		h := w.Header()
+		h.Set("Connection", "close")
+		h.Set("Foo", "bar")
+
+		var bb bytes.Buffer
+		if err := r.Header.Write(&bb); err != nil {
+			panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
+	}
+	responseExpected := `
+statusCode=200
+Foo: bar
+requested_url={BACKEND}/foo/abc/def?bar=baz&some_arg=some_value
+Pass-Header: abc
+User-Agent: vmauth
+X-Forwarded-For: 12.34.56.78, 42.2.3.84`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// keep_original_host
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  keep_original_host: true`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url=http://some-host.com/foo/abc/def?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// override user-agent header
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "User-Agent: foobar"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/abc/def?bar=baz
+User-Agent=foobar`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// delete user-agent header
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "User-Agent:"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s\nUser-Agent=%s", r.Host, r.URL, r.Header.Get("User-Agent"))
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/abc/def?bar=baz
+User-Agent=Go-http-client/1.1`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// override request host with non-empty host
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "Host: other-host:12345"
+  - "abc:"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url=http://other-host:12345/foo/abc/def?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// override request host with empty host
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo?bar=baz"
+  headers:
+  - "Host:"`
+	requestURL = "http://some-host.com/abc/def"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/abc/def?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// /-/reload handler failure
+	origAuthKey := reloadAuthKey.Get()
+	if err := reloadAuthKey.Set("secret"); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	cfgStr = `
+unauthorized_user:
+  url_prefix: "{BACKEND}/foo"`
+	requestURL = "http://some-host.com/-/reload"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+The provided authKey doesn't match -reloadAuthKey`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	if err := reloadAuthKey.Set(origAuthKey); err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	// missing authorization
+	cfgStr = `
+users:
+- username: foo
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://some-host.com/a/b"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+Www-Authenticate: Basic realm="Restricted"
+missing 'Authorization' request header`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// incorrect authorization
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:invalid-secret@some-host.com/a/b"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+Unauthorized`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// incorrect authorization with logging invalid auth tokens
+	origLogInvalidAuthTokens := *logInvalidAuthTokens
+	*logInvalidAuthTokens = true
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:invalid-secret@some-host.com/a/b?c=d"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /a/b?c=d; cannot authorize request with auth tokens ["http_auth:Basic Zm9vOmludmFsaWQtc2VjcmV0"]`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	*logInvalidAuthTokens = origLogInvalidAuthTokens
+
+	// correct authorization
+	cfgStr = `
+users:
+- username: foo
+  password: secret
+  url_prefix: "{BACKEND}/bar"`
+	requestURL = "http://foo:secret@some-host.com/a/b"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/bar/a/b`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz`
+	requestURL = "http://some-host.com/../../a//.///bar/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/a/bar/?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works for url without path
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo?bar=baz`
+	requestURL = "http://some-host.com/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works for url without path if url_prefix path ends with /
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/foo/?bar=baz`
+	requestURL = "http://some-host.com/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/foo/?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify how path cleanup works for url without path and the url_prefix without path prefix
+	cfgStr = `
+unauthorized_user:
+  url_prefix: {BACKEND}/?bar=baz`
+	requestURL = "http://some-host.com/"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/?bar=baz`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify routing to default_url
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/
+  default_url: {BACKEND}/404.html`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/404.html?request_path=http%3A%2F%2Fsome-host.com%2Fabc%3Fde%3Dfg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// verify routing to default url_prefix
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/
+  url_prefix: {BACKEND}/default`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/default/abc?de=fg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// missing default_url and default url_prefix for unauthorized user
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=400
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /abc?de=fg; missing route for http://some-host.com/abc?de=fg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// missing default_url and default url_prefix for unauthorized user when there are configs for authorized users
+	cfgStr = `
+users:
+- username: some-user
+  url_map:
+  - src_paths: ["/foo/.+"]
+    url_prefix: {BACKEND}/x-foo/
+unauthorized_user:
+  url_map:
+  - src_paths: ["/abc/.*"]
+    url_prefix: {BACKEND}/x-bar`
+	requestURL = "http://some-host.com/abc?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=401
+Www-Authenticate: Basic realm="Restricted"
+missing 'Authorization' request header`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// all the backend_urls are unavailable for unauthorized user
+	cfgStr = `
+unauthorized_user:
+  url_map:
+  - src_paths: ["/foo/.*"]
+    url_prefix:
+    - http://127.0.0.1:1/
+    - http://127.0.0.1:2/`
+	requestURL = "http://some-host.com/foo/?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// all the backend_urls are unavailable for authorized user
+	cfgStr = `
+users:
+- username: some-user
+  url_map:
+  - src_paths: ["/foo/.*"]
+    url_prefix:
+    - http://127.0.0.1:1/
+    - http://127.0.0.1:2/`
+	requestURL = "http://some-user@some-host.com/foo/?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "some-user" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+
+	// zero discovered backend IPs
+	customResolver := &fakeResolver{
+		Resolver: &net.Resolver{},
+		lookupIPAddrResults: map[string][]net.IPAddr{
+			"some-addr": {},
+		},
+	}
+	origResolver := netutil.Resolver
+	netutil.Resolver = customResolver
+	cfgStr = `
+unauthorized_user:
+  url_prefix: ['http://some-addr:1234/foo/bar']
+  discover_backend_ips: true`
+	requestURL = "http://abc.com/def/?de=fg"
+	backendHandler = func(_ http.ResponseWriter, _ *http.Request) {
+		panic(fmt.Errorf("backend handler shouldn't be called"))
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /def/?de=fg; all the 0 backends for the user "" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	netutil.Resolver = origResolver
+
+	// retry_status_codes failure
+	var retries atomic.Int64
+	cfgStr = `
+unauthorized_user:
+  url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
+  retry_status_codes: [500, 502]`
+	requestURL = "http://some-host.com/foo/?de=fg"
+	backendHandler = func(w http.ResponseWriter, _ *http.Request) {
+		retries.Add(1)
+		w.WriteHeader(500)
+	}
+	responseExpected = `
+statusCode=502
+remoteAddr: "42.2.3.84:6789, X-Forwarded-For: 12.34.56.78"; requestURI: /foo/?de=fg; all the 2 backends for the user "" are unavailable`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	if n := retries.Load(); n != 2 {
+		t.Fatalf("unexpected number of retries; got %d; want 2", n)
+	}
+
+	// retry_status_codes success
+	retries.Store(0)
+	cfgStr = `
+unauthorized_user:
+  url_prefix: ['{BACKEND}/path1', '{BACKEND}/path2']
+  retry_status_codes: [500, 502]`
+	requestURL = "http://some-host.com/foo/?de=fg"
+	backendHandler = func(w http.ResponseWriter, r *http.Request) {
+		if n := retries.Add(1); n < 2 {
+			w.WriteHeader(500)
+			return
+		}
+		fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
+	}
+	responseExpected = `
+statusCode=200
+requested_url={BACKEND}/path2/foo/?de=fg`
+	f(cfgStr, requestURL, backendHandler, responseExpected)
+	if n := retries.Load(); n != 2 {
+		t.Fatalf("unexpected number of retries; got %d; want 2", n)
+	}
+}
+
+type fakeResponseWriter struct {
+	h http.Header
+
+	bb bytes.Buffer
+}
+
+func (w *fakeResponseWriter) getResponse() string {
+	return w.bb.String()
+}
+
+func (w *fakeResponseWriter) Header() http.Header {
+	if w.h == nil {
+		w.h = http.Header{}
+	}
+	return w.h
+}
+
+func (w *fakeResponseWriter) Write(p []byte) (int, error) {
+	return w.bb.Write(p)
+}
+
+func (w *fakeResponseWriter) WriteHeader(statusCode int) {
+	fmt.Fprintf(&w.bb, "statusCode=%d\n", statusCode)
+	if w.h == nil {
+		return
+	}
+	err := w.h.WriteSubset(&w.bb, map[string]bool{
+		"Content-Length":         true,
+		"Content-Type":           true,
+		"Date":                   true,
+		"X-Content-Type-Options": true,
+	})
+	if err != nil {
+		panic(fmt.Errorf("cannot marshal headers: %s", err))
+	}
+}
+
+func TestReadTrackingBody_RetrySuccess(t *testing.T) {
+	f := func(s string, maxBodySize int) {
+		t.Helper()
+
+		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defer putReadTrackingBody(rtb)
+
 		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true")
+			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		for i := 0; i < 5; i++ {
 			data, err := io.ReadAll(rtb)
@@ -32,41 +534,99 @@ func TestReadTrackingBodyRetrySuccess(t *testing.T) {
 		}
 	}
 
-	f("")
-	f("foo")
-	f("foobar")
-	f(newTestString(maxRequestBodySizeToRetry.IntN()))
+	f("", 0)
+	f("", -1)
+	f("", 100)
+	f("foo", 100)
+	f("foobar", 100)
+	f(newTestString(1000), 1000)
 }
 
-func TestReadTrackingBodyRetryFailure(t *testing.T) {
-	f := func(s string) {
+func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
+	f := func(s string, maxBodySize int) {
 		t.Helper()
-		rtb := &readTrackingBody{
-			r: io.NopCloser(bytes.NewBufferString(s)),
+
+		// Check the case with partial read
+		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defer putReadTrackingBody(rtb)
+
+		for i := 0; i < len(s); i++ {
+			buf := make([]byte, i)
+			n, err := io.ReadFull(rtb, buf)
+			if err != nil {
+				t.Fatalf("unexpected error when reading %d bytes: %s", i, err)
+			}
+			if n != i {
+				t.Fatalf("unexpected number of bytes read; got %d; want %d", n, i)
+			}
+			if string(buf) != s[:i] {
+				t.Fatalf("unexpected data read with the length %d\ngot\n%s\nwant\n%s", i, buf, s[:i])
+			}
+			if err := rtb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing reader after reading %d bytes", i)
+			}
+			if !rtb.canRetry() {
+				t.Fatalf("canRetry() must return true after closing the reader after reading %d bytes", i)
+			}
+		}
+
+		data, err := io.ReadAll(rtb)
+		if err != nil {
+			t.Fatalf("unexpected error when reading all the data: %s", err)
+		}
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
+		}
+		if err := rtb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
 		}
 		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true")
+			t.Fatalf("canRetry() must return true after closing the reader after reading all the input")
+		}
+	}
+
+	f("", 0)
+	f("", -1)
+	f("", 100)
+	f("foo", 100)
+	f("foobar", 100)
+	f(newTestString(1000), 1000)
+}
+
+func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
+	f := func(s string, maxBodySize int) {
+		t.Helper()
+
+		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defer putReadTrackingBody(rtb)
+
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true before reading anything")
 		}
 		buf := make([]byte, 1)
-		n, err := rtb.Read(buf)
+		n, err := io.ReadFull(rtb, buf)
 		if err != nil {
 			t.Fatalf("unexpected error when reading a single byte: %s", err)
 		}
 		if n != 1 {
 			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
 		}
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true after reading one byte")
+		}
 		data, err := io.ReadAll(rtb)
 		if err != nil {
 			t.Fatalf("unexpected error when reading all the data: %s", err)
 		}
-		if string(buf)+string(data) != s {
-			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", string(buf)+string(data), s)
+		dataRead := string(buf) + string(data)
+		if dataRead != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", dataRead, s)
 		}
 		if err := rtb.Close(); err != nil {
 			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
 		}
 		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false")
+			t.Fatalf("canRetry() must return false after closing the reader")
 		}
 
 		data, err = io.ReadAll(rtb)
@@ -78,132 +638,55 @@ func TestReadTrackingBodyRetryFailure(t *testing.T) {
 		}
 	}
 
-	f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
-	f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
+	const maxBodySize = 1000
+	f(newTestString(maxBodySize+1), maxBodySize)
+	f(newTestString(2*maxBodySize), maxBodySize)
 }
 
-// request body not over maxRequestBodySizeToRetry
-// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
-// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
-// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
-func TestRetryReadSuccessAfterPartialRead(t *testing.T) {
-	f := func(s string) {
-		rtb := &readTrackingBody{
-			r:   io.NopCloser(bytes.NewBufferString(s)),
-			buf: make([]byte, 0, len(s)),
-		}
+func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
+	f := func(s string, maxBodySize int) {
+		t.Helper()
 
-		var data []byte
-		var err error
-		halfSize := len(s) / 2
-		if halfSize == 0 {
-			halfSize = 100
-		}
-		buf := make([]byte, halfSize)
-		var n int
+		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
+		defer putReadTrackingBody(rtb)
 
-		// read part of the data
-		n, err = rtb.Read(buf[:])
-		data = append(data, buf[:n]...)
-		if err != nil && err != io.EOF {
-			t.Fatalf("unexpected error: %s", err)
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true before reading anything")
+		}
+		data, err := io.ReadAll(rtb)
+		if err != nil {
+			t.Fatalf("unexpected error when reading all the data: %s", err)
+		}
+		if string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", data, s)
 		}
-
-		// request failed when output stream is closed (eg: server connection reset)
-		// would close the reader
 		if err := rtb.Close(); err != nil {
 			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
 		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true")
-		}
 
-		// retry read (read buf + remaining data)
-		data = data[:0]
-		err = nil
-		for err == nil {
-			n, err = rtb.Read(buf[:])
-			data = append(data, buf[:n]...)
-		}
-		if err != io.EOF {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if string(data) != s {
-			t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
-		}
-		// cannotRetry return false
-		// because the request data is not over maxRequestBodySizeToRetry limit
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true")
-		}
-	}
-
-	f("")
-	f("foo")
-	f("foobar")
-	f(newTestString(maxRequestBodySizeToRetry.IntN()))
-}
-
-// request body over maxRequestBodySizeToRetry
-// 1. When writing data downstream, buf only caches part of the data because the downstream connection is disconnected.
-// 2. retry request: because buf caches some data, first read buf and then read stream when retrying
-// 3. retry request: the data has been read to buf in the second step. if the request fails, retry to read all buf later.
-func TestRetryReadSuccessAfterPartialReadAndCannotRetryAgain(t *testing.T) {
-	f := func(s string) {
-		rtb := &readTrackingBody{
-			r:   io.NopCloser(bytes.NewBufferString(s)),
-			buf: make([]byte, 0, len(s)),
-		}
-
-		var data []byte
-		var err error
-		halfSize := len(s) / 2
-		if halfSize == 0 {
-			halfSize = 100
-		}
-		buf := make([]byte, halfSize)
-		var n int
-
-		// read part of the data
-		n, err = rtb.Read(buf[:])
-		data = append(data, buf[:n]...)
-		if err != nil && err != io.EOF {
-			t.Fatalf("unexpected error: %s", err)
-		}
-
-		// request failed when output stream is closed (eg: server connection reset)
-		if err := rtb.Close(); err != nil {
-			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
-		}
-		if !rtb.canRetry() {
-			t.Fatalf("canRetry() must return true")
-		}
-
-		// retry read (read buf + remaining data)
-		data = data[:0]
-		err = nil
-		for err == nil {
-			n, err = rtb.Read(buf[:])
-			data = append(data, buf[:n]...)
-		}
-		if err != io.EOF {
-			t.Fatalf("unexpected error: %s", err)
-		}
-		if string(data) != s {
-			t.Fatalf("unexpected data read; got\n%s\nwant\n%s", data, s)
-		}
-
-		// cannotRetry returns true
-		// because the request data is over maxRequestBodySizeToRetry limit
 		if rtb.canRetry() {
-			t.Fatalf("canRetry() must return false")
+			t.Fatalf("canRetry() must return false after closing the reader")
+		}
+		data, err = io.ReadAll(rtb)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if len(data) != 0 {
+			t.Fatalf("unexpected non-empty data read: %q", data)
 		}
 	}
 
-	f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
-	f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
+	f("foobar", 0)
+	f(newTestString(1000), 0)
+
+	f("foobar", -1)
+	f(newTestString(1000), -1)
 }
 
 func newTestString(sLen int) string {
-	return string(make([]byte, sLen))
+	data := make([]byte, sLen)
+	for i := range data {
+		data[i] = byte(i)
+	}
+	return string(data)
 }

app/vmauth/target_url_test.go

@@ -86,6 +86,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	f := func(ui *UserInfo, requestURI, expectedTarget, expectedRequestHeaders, expectedResponseHeaders string,
 		expectedRetryStatusCodes []int, expectedLoadBalancingPolicy string, expectedDropSrcPathPrefixParts int) {
 		t.Helper()
+
 		if err := ui.initURLs(); err != nil {
 			t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
 		}
@@ -122,6 +123,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			t.Fatalf("unexpected dropSrcPathPrefixParts; got %d; want %d", up.dropSrcPathPrefixParts, expectedDropSrcPathPrefixParts)
 		}
 	}
+
 	// Simple routing with `url_prefix`
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
@@ -260,7 +262,32 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	f(ui, `/api/v1/query?query=up{env="prod"}`, `http://vmselect/1/prometheus/api/v1/query?query=up%7Benv%3D%22prod%22%7D`, "", "", nil, "least_loaded", 0)
 	f(ui, `/api/v1/query?query=up{foo="bar",env="dev",pod!=""}`, `http://vmselect/0/prometheus/api/v1/query?query=up%7Bfoo%3D%22bar%22%2Cenv%3D%22dev%22%2Cpod%21%3D%22%22%7D`, "", "", nil, "least_loaded", 0)
 	f(ui, `/api/v1/query?query=up{foo="bar"}`, `http://default-server/api/v1/query?query=up%7Bfoo%3D%22bar%22%7D`, "", "", nil, "least_loaded", 0)
+}
 
+func TestUserInfoGetBackendURL_SRV(t *testing.T) {
+	f := func(ui *UserInfo, requestURI, expectedTarget string) {
+		t.Helper()
+
+		u, err := url.Parse(requestURI)
+		if err != nil {
+			t.Fatalf("cannot parse %q: %s", requestURI, err)
+		}
+		u = normalizeURL(u)
+		up, _ := ui.getURLPrefixAndHeaders(u, nil)
+		if up == nil {
+			t.Fatalf("cannot match available backend: %s", err)
+		}
+		bu := up.getBackendURL()
+		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
+		bu.put()
+
+		gotTarget := target.String()
+		if gotTarget != expectedTarget {
+			t.Fatalf("unexpected target\ngot:\n%q\nwant\n%q", gotTarget, expectedTarget)
+		}
+	}
+
+	// Discover backendURL with SRV hostnames
 	customResolver := &fakeResolver{
 		Resolver: &net.Resolver{},
 		lookupSRVResults: map[string][]*net.SRV{
@@ -283,11 +310,14 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 	}
+	origResolver := netutil.Resolver
 	netutil.Resolver = customResolver
+	defer func() {
+		netutil.Resolver = origResolver
+	}()
 
-	// Discover backendURL
 	allowed := true
-	ui = &UserInfo{
+	ui := &UserInfo{
 		URLMaps: []URLMap{
 			{
 				SrcPaths:  getRegexs([]string{"/select/.+"}),
@@ -301,12 +331,64 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		DiscoverBackendIPs: &allowed,
 		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
 	}
-	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
-	// url_prefix counter will be reset, still go to 10.6.142.51
-	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
-	f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write", "", "", nil, "least_loaded", 0)
+	if err := ui.initURLs(); err != nil {
+		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
+	}
+
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.50:8481/select/0/prometheus/api/v1/query?query=up")
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`, "http://10.6.142.51:8481/select/0/prometheus/api/v1/query?query=up")
+	f(ui, `/insert/0/prometheus/api/v1/write`, "http://10.6.142.52:8480/insert/0/prometheus/api/v1/write")
 	// unsuccessful dns resolve
-	f(ui, `/test`, "http://non-exist-dns-addr/test", "", "", nil, "least_loaded", 0)
+	f(ui, `/test`, "http://non-exist-dns-addr/test")
+}
+
+func TestUserInfoGetBackendURL_SRVZeroBackends(t *testing.T) {
+	f := func(ui *UserInfo, requestURI string) {
+		t.Helper()
+
+		u, err := url.Parse(requestURI)
+		if err != nil {
+			t.Fatalf("cannot parse %q: %s", requestURI, err)
+		}
+		u = normalizeURL(u)
+		up, _ := ui.getURLPrefixAndHeaders(u, nil)
+		if up == nil {
+			t.Fatalf("cannot match available backend: %s", err)
+		}
+		bu := up.getBackendURL()
+		if bu != nil {
+			t.Fatalf("expecting nil backendURL; got %v", bu)
+		}
+	}
+
+	customResolver := &fakeResolver{
+		Resolver: &net.Resolver{},
+		lookupSRVResults: map[string][]*net.SRV{
+			"vmselect": {},
+		},
+	}
+	origResolver := netutil.Resolver
+	netutil.Resolver = customResolver
+	defer func() {
+		netutil.Resolver = origResolver
+	}()
+
+	allowed := true
+	ui := &UserInfo{
+		URLMaps: []URLMap{
+			{
+				SrcPaths:  getRegexs([]string{"/select/.+"}),
+				URLPrefix: mustParseURL("http://srv+vmselect"),
+			},
+		},
+		DiscoverBackendIPs: &allowed,
+		URLPrefix:          mustParseURL("http://non-exist-dns-addr"),
+	}
+	if err := ui.initURLs(); err != nil {
+		t.Fatalf("cannot initialize urls inside UserInfo: %s", err)
+	}
+
+	f(ui, `/select/0/prometheus/api/v1/query?query=up`)
 }
 
 func TestCreateTargetURLFailure(t *testing.T) {

app/vmbackup/main_test.go

@@ -8,15 +8,18 @@ import (
 func TestHasFilepathPrefix(t *testing.T) {
 	f := func(dst, storageDataPath string, resultExpected bool) {
 		t.Helper()
+
 		result := hasFilepathPrefix(dst, storageDataPath)
 		if result != resultExpected {
-			t.Errorf("unexpected hasFilepathPrefix(%q, %q); got: %v; want: %v", dst, storageDataPath, result, resultExpected)
+			t.Fatalf("unexpected hasFilepathPrefix(%q, %q); got: %v; want: %v", dst, storageDataPath, result, resultExpected)
 		}
 	}
+
 	pwd, err := filepath.Abs("")
 	if err != nil {
 		t.Fatalf("cannot determine working directory: %s", err)
 	}
+
 	f("s3://foo/bar", "foo", false)
 	f("fs://"+pwd+"/foo", "foo", true)
 	f("fs://"+pwd+"/foo", "foo/bar", false)

app/vmctl/backoff/backoff.go

@@ -10,12 +10,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-const (
-	backoffRetries     = 10
-	backoffFactor      = 1.8
-	backoffMinDuration = time.Second * 2
-)
-
 // retryableFunc describes call back which will repeat on errors
 type retryableFunc func() error
 
@@ -30,12 +24,22 @@ type Backoff struct {
 }
 
 // New initialize backoff object
-func New() *Backoff {
-	return &Backoff{
-		retries:     backoffRetries,
-		factor:      backoffFactor,
-		minDuration: backoffMinDuration,
+func New(retries int, factor float64, minDuration time.Duration) (*Backoff, error) {
+	if retries <= 0 {
+		return nil, fmt.Errorf("number of backoff retries must be greater than 0")
 	}
+	if factor <= 1 {
+		return nil, fmt.Errorf("backoff retry factor must be greater than 1")
+	}
+	if minDuration <= 0 {
+		return nil, fmt.Errorf("backoff retry minimum duration must be greater than 0")
+	}
+
+	return &Backoff{
+		retries:     retries,
+		factor:      factor,
+		minDuration: minDuration,
+	}, nil
 }
 
 // Retry process retries until all attempts are completed

app/vmctl/backoff/backoff_test.go

@@ -3,107 +3,140 @@ package backoff
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 )
 
-func TestRetry_Do(t *testing.T) {
-	counter := 0
-	tests := []struct {
-		name               string
-		backoffRetries     int
-		backoffFactor      float64
-		backoffMinDuration time.Duration
-		retryableFunc      retryableFunc
-		cancelTimeout      time.Duration
-		want               uint64
-		wantErr            bool
-	}{
-		{
-			name: "return bad request",
-			retryableFunc: func() error {
-				return ErrBadRequest
-			},
-			want:    0,
-			wantErr: true,
-		},
-		{
-			name: "empty retries values",
-			retryableFunc: func() error {
-				time.Sleep(time.Millisecond * 100)
-				return nil
-			},
-			want:    0,
-			wantErr: true,
-		},
-		{
-			name:               "only one retry test",
-			backoffRetries:     5,
-			backoffFactor:      1.7,
-			backoffMinDuration: time.Millisecond * 10,
-			retryableFunc: func() error {
-				t := time.NewTicker(time.Millisecond * 5)
-				defer t.Stop()
-				for range t.C {
-					counter++
-					if counter%2 == 0 {
-						return fmt.Errorf("got some error")
-					}
-					if counter%3 == 0 {
-						return nil
-					}
-				}
-				return nil
-			},
-			want:    1,
-			wantErr: false,
-		},
-		{
-			name:               "all retries failed test",
-			backoffRetries:     5,
-			backoffFactor:      0.1,
-			backoffMinDuration: time.Millisecond * 10,
-			retryableFunc: func() error {
-				t := time.NewTicker(time.Millisecond * 5)
-				defer t.Stop()
-				for range t.C {
-					return fmt.Errorf("got some error")
-				}
-				return nil
-			},
-			want:    5,
-			wantErr: true,
-		},
-		{
-			name:               "cancel context",
-			backoffRetries:     5,
-			backoffFactor:      1.7,
-			backoffMinDuration: time.Millisecond * 10,
-			retryableFunc: func() error {
-				return fmt.Errorf("got some error")
-			},
-			cancelTimeout: time.Millisecond * 40,
-			want:          3,
-			wantErr:       true,
-		},
+func TestBackoffRetry_Failure(t *testing.T) {
+	f := func(backoffFactor float64, backoffRetries int, cancelTimeout time.Duration, retryFunc func() error, resultExpected int) {
+		t.Helper()
+
+		r := &Backoff{
+			retries:     backoffRetries,
+			factor:      backoffFactor,
+			minDuration: time.Millisecond * 10,
+		}
+		ctx := context.Background()
+		if cancelTimeout != 0 {
+			newCtx, cancelFn := context.WithTimeout(context.Background(), cancelTimeout)
+			ctx = newCtx
+			defer cancelFn()
+		}
+
+		result, err := r.Retry(ctx, retryFunc)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if result != uint64(resultExpected) {
+			t.Fatalf("unexpected result: got %d; want %d", result, resultExpected)
+		}
 	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := &Backoff{retries: tt.backoffRetries, factor: tt.backoffFactor, minDuration: tt.backoffMinDuration}
-			ctx := context.Background()
-			if tt.cancelTimeout != 0 {
-				newCtx, cancelFn := context.WithTimeout(context.Background(), tt.cancelTimeout)
-				ctx = newCtx
-				defer cancelFn()
-			}
-			got, err := r.Retry(ctx, tt.retryableFunc)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Retry() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if got != tt.want {
-				t.Errorf("Retry() got = %v, want %v", got, tt.want)
-			}
-		})
+
+	// return bad request
+	retryFunc := func() error {
+		return ErrBadRequest
 	}
+	f(0, 0, 0, retryFunc, 0)
+
+	// empty retries values
+	retryFunc = func() error {
+		time.Sleep(time.Millisecond * 100)
+		return nil
+	}
+	f(0, 0, 0, retryFunc, 0)
+
+	// all retries failed test
+	backoffFactor := 0.1
+	backoffRetries := 5
+	cancelTimeout := time.Second * 0
+	retryFunc = func() error {
+		t := time.NewTicker(time.Millisecond * 5)
+		defer t.Stop()
+		for range t.C {
+			return fmt.Errorf("got some error")
+		}
+		return nil
+	}
+	resultExpected := 5
+	f(backoffFactor, backoffRetries, cancelTimeout, retryFunc, resultExpected)
+
+	// cancel context
+	backoffFactor = 1.7
+	backoffRetries = 5
+	cancelTimeout = time.Millisecond * 40
+	retryFunc = func() error {
+		return fmt.Errorf("got some error")
+	}
+	resultExpected = 3
+	f(backoffFactor, backoffRetries, cancelTimeout, retryFunc, resultExpected)
+}
+
+func TestBackoffRetry_Success(t *testing.T) {
+	f := func(retryFunc func() error, resultExpected int) {
+		t.Helper()
+
+		r := &Backoff{
+			retries:     5,
+			factor:      1.7,
+			minDuration: time.Millisecond * 10,
+		}
+		ctx := context.Background()
+
+		result, err := r.Retry(ctx, retryFunc)
+		if err != nil {
+			t.Fatalf("Retry() error: %s", err)
+		}
+		if result != uint64(resultExpected) {
+			t.Fatalf("unexpected result: got %d; want %d", result, resultExpected)
+		}
+	}
+
+	// only one retry test
+	counter := 0
+	retryFunc := func() error {
+		t := time.NewTicker(time.Millisecond * 5)
+		defer t.Stop()
+		for range t.C {
+			counter++
+			if counter%2 == 0 {
+				return fmt.Errorf("got some error")
+			}
+			if counter%3 == 0 {
+				return nil
+			}
+		}
+		return nil
+	}
+	resultExpected := 1
+	f(retryFunc, resultExpected)
+}
+
+func TestBackoff_New(t *testing.T) {
+	f := func(retries int, factor float64, minDuration time.Duration, errExpected string) {
+		t.Helper()
+
+		_, err := New(retries, factor, minDuration)
+		if err == nil {
+			if errExpected != "" {
+				t.Fatalf("expecting non-nil error")
+			}
+			return
+		}
+		if !strings.Contains(err.Error(), errExpected) {
+			t.Fatalf("unexpected error: got %q; want %q", err.Error(), errExpected)
+		}
+	}
+
+	// empty retries
+	f(0, 1.1, time.Millisecond*10, "retries must be greater than 0")
+
+	// empty factor
+	f(1, 0, time.Millisecond*10, "factor must be greater than 1")
+
+	// empty minDuration
+	f(1, 1.1, 0, "minimum duration must be greater than 0")
+
+	// no errors
+	f(1, 1.1, time.Millisecond*10, "")
 }

app/vmctl/flags.go

@@ -56,6 +56,10 @@ const (
 	vmRateLimit  = "vm-rate-limit"
 
 	vmInterCluster = "vm-intercluster"
+
+	vmBackoffRetries     = "vm-backoff-retries"
+	vmBackoffFactor      = "vm-backoff-factor"
+	vmBackoffMinDuration = "vm-backoff-min-duration"
 )
 
 var (
@@ -146,6 +150,21 @@ var (
 			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  vmBackoffRetries,
+			Value: 10,
+			Usage: "How many import retries to perform before giving up.",
+		},
+		&cli.Float64Flag{
+			Name:  vmBackoffFactor,
+			Value: 1.8,
+			Usage: "Factor to multiply the base duration after each failed import retry. Must be greater than 1.0",
+		},
+		&cli.DurationFlag{
+			Name:  vmBackoffMinDuration,
+			Value: time.Second * 2,
+			Usage: "Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'.",
+		},
 	}
 )
 
@@ -430,6 +449,10 @@ const (
 	vmNativeDstCAFile             = "vm-native-dst-ca-file"
 	vmNativeDstServerName         = "vm-native-dst-server-name"
 	vmNativeDstInsecureSkipVerify = "vm-native-dst-insecure-skip-verify"
+
+	vmNativeBackoffRetries     = "vm-native-backoff-retries"
+	vmNativeBackoffFactor      = "vm-native-backoff-factor"
+	vmNativeBackoffMinDuration = "vm-native-backoff-min-duration"
 )
 
 var (
@@ -599,6 +622,21 @@ var (
 				"Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  vmNativeBackoffRetries,
+			Value: 10,
+			Usage: "How many export/import retries to perform before giving up.",
+		},
+		&cli.Float64Flag{
+			Name:  vmNativeBackoffFactor,
+			Value: 1.8,
+			Usage: "Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0",
+		},
+		&cli.DurationFlag{
+			Name:  vmNativeBackoffMinDuration,
+			Value: time.Second * 2,
+			Usage: "Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'.",
+		},
 	}
 )
 

app/vmctl/influx/influx_test.go

@@ -3,125 +3,97 @@ package influx
 import "testing"
 
 func TestFetchQuery(t *testing.T) {
-	testCases := []struct {
-		s          Series
-		timeFilter string
-		expected   string
-	}{
-		{
-			s: Series{
-				Measurement: "cpu",
-				Field:       "value",
-				LabelPairs: []LabelPair{
-					{
-						Name:  "foo",
-						Value: "bar",
-					},
-				},
-			},
-			expected: `select "value" from "cpu" where "foo"::tag='bar'`,
-		},
-		{
-			s: Series{
-				Measurement: "cpu",
-				Field:       "value",
-				LabelPairs: []LabelPair{
-					{
-						Name:  "foo",
-						Value: "bar",
-					},
-					{
-						Name:  "baz",
-						Value: "qux",
-					},
-				},
-			},
-			expected: `select "value" from "cpu" where "foo"::tag='bar' and "baz"::tag='qux'`,
-		},
-		{
-			s: Series{
-				Measurement: "cpu",
-				Field:       "value",
-				LabelPairs: []LabelPair{
-					{
-						Name:  "foo",
-						Value: "b'ar",
-					},
-				},
-			},
-			timeFilter: "time >= now()",
-			expected:   `select "value" from "cpu" where "foo"::tag='b\'ar' and time >= now()`,
-		},
-		{
-			s: Series{
-				Measurement: "cpu",
-				Field:       "value",
-				LabelPairs: []LabelPair{
-					{
-						Name:  "name",
-						Value: `dev-mapper-centos\x2dswap.swap`,
-					},
-					{
-						Name:  "state",
-						Value: "dev-mapp'er-c'en'tos",
-					},
-				},
-			},
-			timeFilter: "time >= now()",
-			expected:   `select "value" from "cpu" where "name"::tag='dev-mapper-centos\\x2dswap.swap' and "state"::tag='dev-mapp\'er-c\'en\'tos' and time >= now()`,
-		},
-		{
-			s: Series{
-				Measurement: "cpu",
-				Field:       "value",
-			},
-			timeFilter: "time >= now()",
-			expected:   `select "value" from "cpu" where time >= now()`,
-		},
-		{
-			s: Series{
-				Measurement: "cpu",
-				Field:       "value",
-			},
-			expected: `select "value" from "cpu"`,
-		},
-	}
+	f := func(s *Series, timeFilter, resultExpected string) {
+		t.Helper()
 
-	for _, tc := range testCases {
-		query := tc.s.fetchQuery(tc.timeFilter)
-		if query != tc.expected {
-			t.Fatalf("got: \n%s;\nexpected: \n%s", query, tc.expected)
+		result := s.fetchQuery(timeFilter)
+		if result != resultExpected {
+			t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
 		}
 	}
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value",
+		LabelPairs: []LabelPair{
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+		},
+	}, "", `select "value" from "cpu" where "foo"::tag='bar'`)
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value",
+		LabelPairs: []LabelPair{
+			{
+				Name:  "foo",
+				Value: "bar",
+			},
+			{
+				Name:  "baz",
+				Value: "qux",
+			},
+		},
+	}, "", `select "value" from "cpu" where "foo"::tag='bar' and "baz"::tag='qux'`)
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value",
+		LabelPairs: []LabelPair{
+			{
+				Name:  "foo",
+				Value: "b'ar",
+			},
+		},
+	}, "time >= now()", `select "value" from "cpu" where "foo"::tag='b\'ar' and time >= now()`)
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value",
+		LabelPairs: []LabelPair{
+			{
+				Name:  "name",
+				Value: `dev-mapper-centos\x2dswap.swap`,
+			},
+			{
+				Name:  "state",
+				Value: "dev-mapp'er-c'en'tos",
+			},
+		},
+	}, "time >= now()", `select "value" from "cpu" where "name"::tag='dev-mapper-centos\\x2dswap.swap' and "state"::tag='dev-mapp\'er-c\'en\'tos' and time >= now()`)
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value",
+	}, "time >= now()", `select "value" from "cpu" where time >= now()`)
+
+	f(&Series{
+		Measurement: "cpu",
+		Field:       "value",
+	}, "", `select "value" from "cpu"`)
 }
 
 func TestTimeFilter(t *testing.T) {
-	testCases := []struct {
-		start    string
-		end      string
-		expected string
-	}{
-		{
-			start:    "2020-01-01T20:07:00Z",
-			end:      "2020-01-01T21:07:00Z",
-			expected: "time >= '2020-01-01T20:07:00Z' and time <= '2020-01-01T21:07:00Z'",
-		},
-		{
-			expected: "",
-		},
-		{
-			start:    "2020-01-01T20:07:00Z",
-			expected: "time >= '2020-01-01T20:07:00Z'",
-		},
-		{
-			end:      "2020-01-01T21:07:00Z",
-			expected: "time <= '2020-01-01T21:07:00Z'",
-		},
-	}
-	for _, tc := range testCases {
-		f := timeFilter(tc.start, tc.end)
-		if f != tc.expected {
-			t.Fatalf("got: \n%q;\nexpected: \n%q", f, tc.expected)
+	f := func(start, end, resultExpected string) {
+		t.Helper()
+
+		result := timeFilter(start, end)
+		if result != resultExpected {
+			t.Fatalf("unexpected result\ngot\n%v\nwant\n%s", result, resultExpected)
 		}
 	}
+
+	// no start and end filters
+	f("", "", "")
+
+	// missing end filter
+	f("2020-01-01T20:07:00Z", "", "time >= '2020-01-01T20:07:00Z'")
+
+	// missing start filter
+	f("", "2020-01-01T21:07:00Z", "time <= '2020-01-01T21:07:00Z'")
+
+	// both start and end filters
+	f("2020-01-01T20:07:00Z", "2020-01-01T21:07:00Z", "time >= '2020-01-01T20:07:00Z' and time <= '2020-01-01T21:07:00Z'")
 }

app/vmctl/influx/parser.go

@@ -12,7 +12,7 @@ import (
 
 type queryValues struct {
 	name   string
-	values map[string][]interface{}
+	values map[string][]any
 }
 
 func parseResult(r influx.Result) ([]queryValues, error) {
@@ -21,7 +21,7 @@ func parseResult(r influx.Result) ([]queryValues, error) {
 	}
 	qValues := make([]queryValues, len(r.Series))
 	for i, row := range r.Series {
-		values := make(map[string][]interface{}, len(row.Values))
+		values := make(map[string][]any, len(row.Values))
 		for _, value := range row.Values {
 			for idx, v := range value {
 				key := row.Columns[idx]
@@ -36,7 +36,7 @@ func parseResult(r influx.Result) ([]queryValues, error) {
 	return qValues, nil
 }
 
-func toFloat64(v interface{}) (float64, error) {
+func toFloat64(v any) (float64, error) {
 	switch i := v.(type) {
 	case json.Number:
 		return i.Float64()

app/vmctl/influx/parser_test.go

@@ -6,71 +6,71 @@ import (
 	"testing"
 )
 
-func TestSeries_Unmarshal(t *testing.T) {
+func TestSeriesUnmarshal(t *testing.T) {
+	f := func(s string, resultExpected *Series) {
+		t.Helper()
+
+		result := &Series{}
+		if err := result.unmarshal(s); err != nil {
+			t.Fatalf("cannot unmarshal series from %q: %s", s, err)
+		}
+		if !reflect.DeepEqual(result, resultExpected) {
+			t.Fatalf("unexpected result\ngot\n%v\nwant\n%v", result, resultExpected)
+		}
+	}
+
 	tag := func(name, value string) LabelPair {
 		return LabelPair{
 			Name:  name,
 			Value: value,
 		}
 	}
-	series := func(measurement string, lp ...LabelPair) Series {
-		return Series{
+	series := func(measurement string, lp ...LabelPair) *Series {
+		return &Series{
 			Measurement: measurement,
 			LabelPairs:  lp,
 		}
 	}
-	testCases := []struct {
-		got  string
-		want Series
-	}{
-		{
-			got:  "cpu",
-			want: series("cpu"),
-		},
-		{
-			got:  "cpu,host=localhost",
-			want: series("cpu", tag("host", "localhost")),
-		},
-		{
-			got:  "cpu,host=localhost,instance=instance",
-			want: series("cpu", tag("host", "localhost"), tag("instance", "instance")),
-		},
-		{
-			got:  `fo\,bar\=baz,x\=\b=\\a\,\=\q\ `,
-			want: series("fo,bar=baz", tag(`x=\b`, `\a,=\q `)),
-		},
-		{
-			got:  "cpu,host=192.168.0.1,instance=fe80::fdc8:5e36:c2c6:baac%utun1",
-			want: series("cpu", tag("host", "192.168.0.1"), tag("instance", "fe80::fdc8:5e36:c2c6:baac%utun1")),
-		},
-		{
-			got: `cpu,db=db1,host=localhost,server=host\=localhost\ user\=user\ `,
-			want: series("cpu", tag("db", "db1"),
-				tag("host", "localhost"), tag("server", "host=localhost user=user ")),
-		},
-	}
-	for _, tc := range testCases {
-		s := Series{}
-		if err := s.unmarshal(tc.got); err != nil {
-			t.Fatalf("%q: unmarshal err: %s", tc.got, err)
-		}
-		if !reflect.DeepEqual(s, tc.want) {
-			t.Fatalf("%q: expected\n%#v\nto be equal\n%#v", tc.got, s, tc.want)
-		}
-	}
+
+	f("cpu", series("cpu"))
+
+	f("cpu,host=localhost", series("cpu", tag("host", "localhost")))
+
+	f("cpu,host=localhost,instance=instance", series("cpu", tag("host", "localhost"), tag("instance", "instance")))
+
+	f(`fo\,bar\=baz,x\=\b=\\a\,\=\q\ `, series("fo,bar=baz", tag(`x=\b`, `\a,=\q `)))
+
+	f("cpu,host=192.168.0.1,instance=fe80::fdc8:5e36:c2c6:baac%utun1", series("cpu", tag("host", "192.168.0.1"), tag("instance", "fe80::fdc8:5e36:c2c6:baac%utun1")))
+
+	f(`cpu,db=db1,host=localhost,server=host\=localhost\ user\=user\ `, series("cpu", tag("db", "db1"), tag("host", "localhost"), tag("server", "host=localhost user=user ")))
 }
 
-func TestToFloat64(t *testing.T) {
-	f := func(in interface{}, want float64) {
+func TestToFloat64_Failure(t *testing.T) {
+	f := func(in any) {
 		t.Helper()
-		got, err := toFloat64(in)
-		if err != nil {
-			t.Fatalf("unexpected err: %s", err)
-		}
-		if got != want {
-			t.Errorf("got %v; want %v", got, want)
+
+		_, err := toFloat64(in)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
 		}
 	}
+
+	f("text")
+}
+
+func TestToFloat64_Success(t *testing.T) {
+	f := func(in any, resultExpected float64) {
+		t.Helper()
+
+		result, err := toFloat64(in)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if result != resultExpected {
+			t.Fatalf("unexpected result: got %v; want %v", result, resultExpected)
+		}
+	}
+
 	f("123.4", 123.4)
 	f(float64(123.4), 123.4)
 	f(float32(12), 12)
@@ -78,9 +78,4 @@ func TestToFloat64(t *testing.T) {
 	f(true, 1)
 	f(false, 0)
 	f(json.Number("123456.789"), 123456.789)
-
-	_, err := toFloat64("text")
-	if err == nil {
-		t.Fatalf("expected to get err; got nil instead")
-	}
 }

app/vmctl/main.go

@@ -68,7 +68,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create Transport: %s", err)
+						return fmt.Errorf("failed to create transport for -%s=%q: %s", otsdbAddr, addr, err)
 					}
 					oCfg := opentsdb.Config{
 						Addr:       addr,
@@ -90,6 +90,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -143,6 +144,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -178,7 +180,7 @@ func main() {
 
 					tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 					if err != nil {
-						return fmt.Errorf("failed to create transport: %s", err)
+						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
 					}
 
 					rr, err := remoteread.NewClient(remoteread.Config{
@@ -201,6 +203,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -233,6 +236,7 @@ func main() {
 					if err != nil {
 						return fmt.Errorf("failed to init VM configuration: %s", err)
 					}
+
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -272,6 +276,14 @@ func main() {
 						return fmt.Errorf("flag %q can't be empty", vmNativeFilterMatch)
 					}
 
+					bfRetries := c.Int(vmNativeBackoffRetries)
+					bfFactor := c.Float64(vmNativeBackoffFactor)
+					bfMinDuration := c.Duration(vmNativeBackoffMinDuration)
+					bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
+					if err != nil {
+						return fmt.Errorf("failed to create backoff object: %s", err)
+					}
+
 					disableKeepAlive := c.Bool(vmNativeDisableHTTPKeepAlive)
 
 					var srcExtraLabels []string
@@ -350,7 +362,7 @@ func main() {
 							ExtraLabels: dstExtraLabels,
 							HTTPClient:  dstHTTPClient,
 						},
-						backoff:                  backoff.New(),
+						backoff:                  bf,
 						cc:                       c.Int(vmConcurrency),
 						disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
 						isNative:                 !c.Bool(vmNativeDisableBinaryProtocol),
@@ -426,7 +438,15 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 
 	tr, err := httputils.Transport(addr, certFile, keyFile, caFile, serverName, insecureSkipVerify)
 	if err != nil {
-		return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
+		return vm.Config{}, fmt.Errorf("failed to create transport for -%s=%q: %s", vmAddr, addr, err)
+	}
+
+	bfRetries := c.Int(vmBackoffRetries)
+	bfFactor := c.Float64(vmBackoffFactor)
+	bfMinDuration := c.Duration(vmBackoffMinDuration)
+	bf, err := backoff.New(bfRetries, bfFactor, bfMinDuration)
+	if err != nil {
+		return vm.Config{}, fmt.Errorf("failed to create backoff object: %s", err)
 	}
 
 	return vm.Config{
@@ -442,5 +462,6 @@ func initConfigVM(c *cli.Context) (vm.Config, error) {
 		RoundDigits:        c.Int(vmRoundDigits),
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
 		RateLimit:          c.Int64(vmRateLimit),
+		Backoff:            bf,
 	}, nil
 }

app/vmctl/opentsdb/opentsdb.go

@@ -75,12 +75,6 @@ type TimeRange struct {
 type MetaResults struct {
 	Type    string `json:"type"`
 	Results []Meta `json:"results"`
-	//metric       string
-	//tags         interface{}
-	//limit        int
-	//time         int
-	//startIndex   int
-	//totalResults int
 }
 
 // Meta A meta object about a metric
@@ -88,7 +82,6 @@ type MetaResults struct {
 type Meta struct {
 	Metric string            `json:"metric"`
 	Tags   map[string]string `json:"tags"`
-	//tsuid  string
 }
 
 // OtsdbMetric is a single series in OpenTSDB's returned format