docs/CHANGELOG.md: cut v1.113.0

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>

.github/workflows/docs.yaml

@@ -5,6 +5,7 @@ on:
       - 'master'
     paths:
       - 'docs/**'
+      - '.github/workflows/docs.yaml'
   workflow_dispatch: {}
 permissions:
   contents: read  # This is required for actions/checkout and to commit back image update
@@ -17,35 +18,41 @@ jobs:
       - name: Code checkout
         uses: actions/checkout@v4
         with:
-          path: main
+          path: __vm
+
       - name: Checkout private code
         uses: actions/checkout@v4
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: docs
+          path: __vm-docs
+
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v6
+        id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
           git_user_signingkey: true
           git_commit_gpgsign: true
-          workdir: docs
-      - name: Set short git commit SHA
-        id: vars
+          git_config_global: true
+
+      - name: Copy docs
+        id: update
         run: |
-          calculatedSha=$(git rev-parse --short ${{ github.sha }})
-          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
-        working-directory: main
-      - name: update code and commit
+          rsync -zarv \
+            --exclude="Makefile" \
+            docs/ ../__vm-docs/content/victoriametrics
+          echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
+        working-directory: __vm
+
+      - name: Push to vmdocs
         run: |
-          rm -rf content
-          cp -r ../main/docs content
-          make clean-after-copy
           git config --global user.name "${{ steps.import-gpg.outputs.email }}"
           git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          git add .
-          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
-          git push
-        working-directory: docs
+          if [[ -n $(git status --porcelain) ]]; then
+            git add .
+            git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.update.outputs.SHORT_SHA }}"
+            git push
+          fi
+        working-directory: __vm-docs

.github/workflows/main.yml

@@ -90,7 +90,7 @@ jobs:
       - name: Publish coverage
         uses: codecov/codecov-action@v5
         with:
-          file: ./coverage.txt
+          files: ./coverage.txt
 
   integration-test:
     name: integration-test

Makefile

@@ -513,19 +513,19 @@ check-all: fmt vet golangci-lint govulncheck
 clean-checkers: remove-golangci-lint remove-govulncheck
 
 test:
-	DISABLE_FSYNC_FOR_TESTING=1 go test ./lib/... ./app/...
+	go test ./lib/... ./app/...
 
 test-race:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -race ./lib/... ./app/...
+	go test -race ./lib/... ./app/...
 
 test-pure:
-	DISABLE_FSYNC_FOR_TESTING=1 CGO_ENABLED=0 go test ./lib/... ./app/...
+	CGO_ENABLED=0 go test ./lib/... ./app/...
 
 test-full:
-	DISABLE_FSYNC_FOR_TESTING=1 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 test-full-386:
-	DISABLE_FSYNC_FOR_TESTING=1 GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
+	GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...
 
 integration-test: victoria-metrics vmagent vmalert vmauth
 	go test ./apptest/... -skip="^TestCluster.*"
@@ -567,7 +567,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.5
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

README.md

@@ -3,7 +3,7 @@
 ![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)
 ![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
 ![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)
-![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg?link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
+![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
 ![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)
 ![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)
 ![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)
@@ -22,7 +22,7 @@ Here are some resources and information about VictoriaMetrics:
 
 - Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
 - Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/casestudies/).
-- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
+- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
 - Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/enterprise/)
 - Changelog: [CHANGELOG](https://docs.victoriametrics.com/changelog/), and [How to upgrade](https://docs.victoriametrics.com/#how-to-upgrade-victoriametrics)
 - Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)

SECURITY.md

@@ -8,7 +8,7 @@ The following versions of VictoriaMetrics receive regular security fixes:
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
 | v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.110.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.

app/vlinsert/elasticsearch/elasticsearch.go

@@ -60,7 +60,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	switch path {
-	case "/":
+	case "/", "":
 		switch r.Method {
 		case http.MethodGet:
 			// Return fake response for Elasticsearch ping request.

app/vlinsert/insertutils/common_params.go

@@ -199,8 +199,8 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
 	lmp.bytesIngestedTotal.Add(n)
 
 	if len(fields) > *MaxFieldsPerLine {
-		rf := logstorage.RowFormatter(fields)
-		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+		line := logstorage.MarshalFieldsToJSON(nil, fields)
+		logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, line)
 		rowsDroppedTotalTooManyFields.Inc()
 		return
 	}

app/vlinsert/insertutils/flags.go

@@ -8,8 +8,10 @@ import (
 
 var (
 	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
-	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have")
 
 	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
-	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers; "+
+		"see https://docs.victoriametrics.com/victorialogs/faq/#how-many-fields-a-single-log-entry-may-contain")
 )

app/vlinsert/insertutils/timestamp.go

@@ -2,20 +2,19 @@ package insertutils
 
 import (
 	"fmt"
-	"math"
 	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+// ExtractTimestampFromFields extracts timestamp in nanoseconds from the field with the name timeField at fields.
 //
 // The value for the timeField is set to empty string after returning from the function,
 // so it could be ignored during data ingestion.
 //
 // The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+func ExtractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
 	for i := range fields {
 		f := &fields[i]
 		if f.Name != timeField {
@@ -48,22 +47,24 @@ func parseTimestamp(s string) (int64, error) {
 	return nsecs, nil
 }
 
-// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
+// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
 func ParseUnixTimestamp(s string) (int64, error) {
 	n, err := strconv.ParseInt(s, 10, 64)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
 	}
 	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds. Convert it to milliseconds
-		n *= 1e3
+		// The timestamp is in seconds.
+		return n * 1e9, nil
 	}
-	if n > int64(math.MaxInt64)/1e6 {
-		return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
+		// The timestamp is in milliseconds.
+		return n * 1e6, nil
 	}
-	if n < int64(math.MinInt64)/1e6 {
-		return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
+		// The timestamp is in microseconds.
+		return n * 1e3, nil
 	}
-	n *= 1e6
+	// The timestamp is in nanoseconds
 	return n, nil
 }

app/vlinsert/insertutils/timestamp_test.go

@@ -6,11 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+func TestExtractTimestampFromFields_Success(t *testing.T) {
 	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
 		t.Helper()
 
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		nsecs, err := ExtractTimestampFromFields(timeField, fields)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -51,6 +51,18 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 		{Name: "foo", Value: "bar"},
 	}, 1718773640123456789)
 
+	// Unix timestamp in nanoseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123456789"},
+	}, 1718773640123456789)
+
+	// Unix timestamp in microseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123456"},
+	}, 1718773640123456000)
+
 	// Unix timestamp in milliseconds
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
@@ -64,14 +76,14 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 	}, 1718773640000000000)
 }
 
-func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+func TestExtractTimestampFromFields_Error(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 
 		fields := []logstorage.Field{
 			{Name: "time", Value: s},
 		}
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		nsecs, err := ExtractTimestampFromFields("time", fields)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -80,6 +92,7 @@ func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
 		}
 	}
 
+	// invalid time
 	f("foobar")
 
 	// incomplete time

app/vlinsert/journald/journald.go

@@ -30,7 +30,7 @@ const (
 
 var (
 	bodyBufferPool                bytesutil.ByteBufferPool
-	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]+`)
+	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
 )
 
 var (
@@ -129,6 +129,11 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 		return
 	}
 
+	// systemd starting release v258 will support compression, which starts working after negotiation: it expects supported compression
+	// algorithms list in Accept-Encoding response header in a format "<algorithm_1>[:<priority_1>][;<algorithm_2>:<priority_2>]"
+	// See https://github.com/systemd/systemd/pull/34822
+	w.Header().Set("Accept-Encoding", "zstd")
+
 	// update requestJournaldDuration only for successfully parsed requests
 	// There is no need in updating requestJournaldDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.

app/vlinsert/journald/journald_test.go

@@ -35,9 +35,9 @@ func TestPushJournaldOk(t *testing.T) {
 	)
 
 	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
+	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\nE=JobStateChanged\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
 		[]int64{1729698775704404000},
-		"{\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
+		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
 	)
 }
 

app/vlinsert/jsonline/jsonline.go

@@ -51,20 +51,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 
 	lmp := cp.NewLogMessageProcessor("jsonline")
 	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
+	processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
 	lmp.MustClose()
 
-	if err != nil {
-		logger.Errorf("jsonline: %s", err)
-	} else {
-		// update requestDuration only for successfully parsed requests.
-		// There is no need in updating requestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		requestDuration.UpdateDuration(startTime)
-	}
+	requestDuration.UpdateDuration(startTime)
 }
 
-func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) error {
+func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
@@ -76,10 +69,10 @@ func processStreamInternal(streamName string, r io.Reader, timeField string, msg
 		wcr.DecConcurrency()
 		if err != nil {
 			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
+			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			return nil
+			return
 		}
 		n++
 	}
@@ -96,16 +89,17 @@ func readLine(lr *insertutils.LineReader, timeField string, msgFields []string,
 	}
 
 	p := logstorage.GetJSONParser()
+	defer logstorage.PutJSONParser(p)
+
 	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
+		return true, fmt.Errorf("cannot parse json-encoded line: %w; line contents: %q", err, line)
 	}
-	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
+	ts, err := insertutils.ExtractTimestampFromFields(timeField, p.Fields)
 	if err != nil {
-		return false, fmt.Errorf("cannot get timestamp: %w", err)
+		return true, fmt.Errorf("cannot get timestamp from json-encoded line: %w; line contents: %q", err, line)
 	}
 	logstorage.RenameField(p.Fields, msgFields, "_msg")
 	lmp.AddRow(ts, p.Fields, nil)
-	logstorage.PutJSONParser(p)
 
 	return true, nil
 }

app/vlinsert/jsonline/jsonline_test.go

@@ -7,16 +7,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestProcessStreamInternal_Success(t *testing.T) {
+func TestProcessStreamInternal(t *testing.T) {
 	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
 		msgFields := []string{msgField}
 		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeField, msgFields, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
+		processStreamInternal("test", r, timeField, msgFields, tlp)
 
 		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
@@ -45,22 +43,37 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
 {"message":"baz"}`
 	f(data, timeField, msgField, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, "time", nil, tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
 
 	// invalid json
-	f("foobar")
+	data = "foobar"
+	timeField = "@timestamp"
+	msgField = "aaa"
+	timestampsExpected = nil
+	resultExpected = ``
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
 
 	// invalid timestamp field
-	f(`{"time":"foobar"}`)
+	data = `{"time":"foobar"}`
+	timeField = "time"
+	msgField = "abc"
+	timestampsExpected = nil
+	resultExpected = ``
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
+
+	// invalid lines among valid lines
+	data = `
+dsfodmasd
+
+{"time":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+invalid line
+{"time":"2023-06-06T04:48:12.735+01:00","message":"baz"}
+asbsdf
+
+`
+	timeField = "time"
+	msgField = "message"
+	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
+	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"_msg":"baz"}`
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
 }

app/vlinsert/main.go

@@ -46,7 +46,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	switch {
-	case strings.HasPrefix(path, "/elasticsearch/"):
+	case strings.HasPrefix(path, "/elasticsearch"):
+		// some clients may omit trailing slash
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
 		path = strings.TrimPrefix(path, "/elasticsearch")
 		return elasticsearch.RequestHandler(path, w, r)
 	case strings.HasPrefix(path, "/loki/"):

app/vlinsert/opentelemetry/opentelemetry.go

@@ -101,7 +101,7 @@ func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDe
 		commonFields = slicesutil.SetLength(commonFields, len(attributes))
 		for i, attr := range attributes {
 			commonFields[i].Name = attr.Key
-			commonFields[i].Value = attr.Value.FormatString()
+			commonFields[i].Value = attr.Value.FormatString(true)
 		}
 		commonFieldsLen := len(commonFields)
 		for _, sc := range rl.ScopeLogs {
@@ -118,12 +118,24 @@ func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field,
 		fields = fields[:len(commonFields)]
 		fields = append(fields, logstorage.Field{
 			Name:  "_msg",
-			Value: lr.Body.FormatString(),
+			Value: lr.Body.FormatString(true),
 		})
 		for _, attr := range lr.Attributes {
 			fields = append(fields, logstorage.Field{
 				Name:  attr.Key,
-				Value: attr.Value.FormatString(),
+				Value: attr.Value.FormatString(true),
+			})
+		}
+		if len(lr.TraceID) > 0 {
+			fields = append(fields, logstorage.Field{
+				Name:  "trace_id",
+				Value: lr.TraceID,
+			})
+		}
+		if len(lr.SpanID) > 0 {
+			fields = append(fields, logstorage.Field{
+				Name:  "span_id",
+				Value: lr.SpanID,
 			})
 		}
 		fields = append(fields, logstorage.Field{

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -66,9 +66,9 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
+		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Unspecified"}
+{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
 	)
 
 	// multi-scope with resource attributes and multi-line
@@ -113,8 +113,8 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
+		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Debug"}
 {"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}

app/vlinsert/syslog/syslog.go

@@ -560,7 +560,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	if useLocalTimestamp {
 		ts = time.Now().UnixNano()
 	} else {
-		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
+		nsecs, err := insertutils.ExtractTimestampFromFields("timestamp", p.Fields)
 		if err != nil {
 			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
 		}

app/vlogscli/json_prettifier.go

@@ -176,7 +176,7 @@ func writeCompactObject(w io.Writer, fields []logstorage.Field) error {
 		_, err := fmt.Fprintf(w, "%s\n", fields[0].Value)
 		return err
 	}
-	if len(fields) == 2 && fields[0].Name == "_time" || fields[1].Name == "_time" {
+	if len(fields) == 2 && (fields[0].Name == "_time" || fields[1].Name == "_time") {
 		// Write _time\tfieldValue as is
 		if fields[0].Name == "_time" {
 			_, err := fmt.Fprintf(w, "%s\t%s\n", fields[0].Value, fields[1].Value)

app/vlogscli/main.go

@@ -270,7 +270,7 @@ func printCommandsHelp(w io.Writer) {
 \h - show this help
 \s - singleline json output mode
 \m - multiline json output mode
-\c - compact output
+\c - compact output mode
 \logfmt - logfmt output mode
 \wrap_long_lines - toggles wrapping long lines
 \tail <query> - live tail <query> results

app/vlogsgenerator/main.go

@@ -18,7 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 var (
@@ -306,7 +306,7 @@ type timeFlag struct {
 }
 
 func (tf *timeFlag) Set(s string) error {
-	msec, err := promutils.ParseTimeMsec(s)
+	msec, err := timeutil.ParseTimeMsec(s)
 	if err != nil {
 		return fmt.Errorf("cannot parse time from %q: %w", s, err)
 	}

app/vlselect/logsql/logsql.go

@@ -22,7 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 // ProcessFacetsRequest handles /select/logsql/facets request.
@@ -116,7 +116,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	if stepStr == "" {
 		stepStr = "1d"
 	}
-	step, err := promutils.ParseDuration(stepStr)
+	step, err := timeutil.ParseDuration(stepStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse 'step' arg: %s", err)
 		return
@@ -131,7 +131,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	if offsetStr == "" {
 		offsetStr = "0s"
 	}
-	offset, err := promutils.ParseDuration(offsetStr)
+	offset, err := timeutil.ParseDuration(offsetStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse 'offset' arg: %s", err)
 		return
@@ -665,7 +665,7 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	if stepStr == "" {
 		stepStr = "1d"
 	}
-	step, err := promutils.ParseDuration(stepStr)
+	step, err := timeutil.ParseDuration(stepStr)
 	if err != nil {
 		err = fmt.Errorf("cannot parse 'step' arg: %s", err)
 		httpserver.SendPrometheusError(w, r, err)
@@ -694,13 +694,16 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 		for i := range timestamps {
-			timestamp := q.GetTimestamp()
+			// Do not move q.GetTimestamp() outside writeBlock, since ts
+			// must be initialized to query timestamp for every processed log row.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
+			ts := q.GetTimestamp()
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if c.Name == "_time" {
 					nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
 					if ok {
-						timestamp = nsec
+						ts = nsec
 						continue
 					}
 				}
@@ -721,7 +724,7 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 					dst = logstorage.MarshalFieldsToJSON(dst, labels)
 					key := string(dst)
 					p := statsPoint{
-						Timestamp: timestamp,
+						Timestamp: ts,
 						Value:     strings.Clone(c.Values[i]),
 					}
 
@@ -1119,7 +1122,7 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
 		return 0, false, nil
 	}
 	currentTimestamp := time.Now().UnixNano()
-	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
+	nsecs, err := timeutil.ParseTimeAt(s, currentTimestamp)
 	if err != nil {
 		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
 	}

app/vlselect/logsql/logsql_test.go

@@ -28,7 +28,7 @@ func TestParseExtraFilters_Success(t *testing.T) {
 	// LogsQL filter
 	f(`foobar`, `foobar`)
 	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
+	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
 }
 
 func TestParseExtraFilters_Failure(t *testing.T) {
@@ -77,7 +77,7 @@ func TestParseExtraStreamFilters_Success(t *testing.T) {
 	// LogsQL filter
 	f(`foobar`, `foobar`)
 	f(`foo:bar`, `foo:bar`)
-	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
+	f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `{foo="bar",baz="z"} (foo:bar or foo:baz) error _time:5m`)
 }
 
 func TestParseExtraStreamFilters_Failure(t *testing.T) {

app/vlselect/main.go

@@ -176,50 +176,62 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
 	httpserver.EnableCORS(w, r)
+	startTime := time.Now()
 	switch path {
 	case "/select/logsql/facets":
 		logsqlFacetsRequests.Inc()
 		logsql.ProcessFacetsRequest(ctx, w, r)
+		logsqlFacetsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/field_names":
 		logsqlFieldNamesRequests.Inc()
 		logsql.ProcessFieldNamesRequest(ctx, w, r)
+		logsqlFieldNamesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/field_values":
 		logsqlFieldValuesRequests.Inc()
 		logsql.ProcessFieldValuesRequest(ctx, w, r)
+		logsqlFieldValuesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/hits":
 		logsqlHitsRequests.Inc()
 		logsql.ProcessHitsRequest(ctx, w, r)
+		logsqlHitsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/query":
 		logsqlQueryRequests.Inc()
 		logsql.ProcessQueryRequest(ctx, w, r)
+		logsqlQueryDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stats_query":
 		logsqlStatsQueryRequests.Inc()
 		logsql.ProcessStatsQueryRequest(ctx, w, r)
+		logsqlStatsQueryDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stats_query_range":
 		logsqlStatsQueryRangeRequests.Inc()
 		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
+		logsqlStatsQueryRangeDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_field_names":
 		logsqlStreamFieldNamesRequests.Inc()
 		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
+		logsqlStreamFieldNamesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_field_values":
 		logsqlStreamFieldValuesRequests.Inc()
 		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
+		logsqlStreamFieldValuesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_ids":
 		logsqlStreamIDsRequests.Inc()
 		logsql.ProcessStreamIDsRequest(ctx, w, r)
+		logsqlStreamIDsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/streams":
 		logsqlStreamsRequests.Inc()
 		logsql.ProcessStreamsRequest(ctx, w, r)
+		logsqlStreamsDuration.UpdateDuration(startTime)
 		return true
 	default:
 		return false
@@ -240,16 +252,39 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
 }
 
 var (
-	logsqlFacetsRequests            = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
-	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
-	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlStatsQueryRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryRangeRequests   = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
-	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlFacetsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
+	logsqlFacetsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/facets"}`)
+
+	logsqlFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
+	logsqlFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_names"}`)
+
+	logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
+	logsqlFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_values"}`)
+
+	logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlHitsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/hits"}`)
+
+	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/query"}`)
+
+	logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
+	logsqlStatsQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query"}`)
+
+	logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
+	logsqlStatsQueryRangeDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query_range"}`)
+
+	logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlStreamFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_names"}`)
+
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
-	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
+	logsqlStreamFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_values"}`)
+
+	logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
+	logsqlStreamIDsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_ids"}`)
+
+	logsqlStreamsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
+	logsqlStreamsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/streams"}`)
+
+	// no need to track duration for tail requests, as they usually take long time
+	logsqlTailRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
 )

app/vlselect/vmui/asset-manifest.json

@@ -1,12 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.4aacd559.css",
-    "main.js": "./static/js/main.5ce54a05.js",
-    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.4aacd559.css",
-    "static/js/main.5ce54a05.js"
-  ]
-}

app/vlselect/vmui/assets/MetricsQL-DGylhASK.md

@@ -2349,4 +2349,4 @@ VictoriaMetrics performs the following implicit conversions for incoming queries
   is passed to [rollup function](#rollup-functions), then a [subquery](#subqueries) with `1i` lookbehind window and `1i` step is automatically formed.
   For example, `rate(sum(up))` is automatically converted to `rate((sum(default_rollup(up)))[1i:1i])`.
   This behavior can be disabled or logged via `-search.disableImplicitConversion` and `-search.logImplicitConversion` command-line flags
-  starting from [`v1.101.0` release](https://docs.victoriametrics.com/changelog/).
+  starting from [`v1.102.0-rc2` release](https://docs.victoriametrics.com/changelog/changelog_2024/#v11020-rc2).

app/vlselect/vmui/assets/vendor-D1GxaB_c.css

@@ -0,0 +1 @@
+.uplot,.uplot *,.uplot *:before,.uplot *:after{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol,"Noto Color Emoji";line-height:1.5;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:#00000012;position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{position:absolute;left:0;top:0;pointer-events:none;will-change:transform}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607D8B}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607D8B}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;background-clip:padding-box!important}.u-axis.u-off,.u-select.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-cursor-pt.u-off{display:none}

app/vlselect/vmui/index.html

@@ -1 +1,57 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.5ce54a05.js"></script><link href="./static/css/main.4aacd559.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="./favicon.svg" />
+  <link rel="apple-touch-icon" href="./favicon.svg" />
+  <link rel="mask-icon" href="./favicon.svg" color="#000000">
+
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta name="description" content="Explore your log data with VictoriaLogs UI"/>
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="./manifest.json"/>
+  <!--
+    Notice the use of in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>UI for VictoriaLogs</title>
+
+  <meta name="twitter:card" content="summary">
+  <meta name="twitter:title" content="UI for VictoriaLogs">
+  <meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/">
+  <meta name="twitter:description" content="Explore your log data with VictoriaLogs UI">
+  <meta name="twitter:image" content="./preview.jpg">
+
+  <meta property="og:type" content="website">
+  <meta property="og:title" content="UI for VictoriaLogs">
+  <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
+  <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
+  <script type="module" crossorigin src="./assets/index-C68hz-qY.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
+  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-B_R5bdPN.css">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlselect/vmui/static/js/main.5ce54a05.js.LICENSE.txt

@@ -1,38 +0,0 @@
-/*!
-	Copyright (c) 2018 Jed Watson.
-	Licensed under the MIT License (MIT), see
-	http://jedwatson.github.io/classnames
-*/
-
-/**
- * @remix-run/router v1.19.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router DOM v6.26.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router v6.26.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */

app/vmagent/remotewrite/remotewrite_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -53,7 +54,7 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 }
 
 func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
+	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
 		t.Helper()
 		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
 		if err != nil {
@@ -77,12 +78,15 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, enableWindows, dedupInterval, nil, "dedup-global")
 		}
 
 		if streamAggrConfig != "" {
 			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
+			opts := streamaggr.Options{
+				EnableWindows: enableWindows,
+			}
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, &opts, "global")
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
@@ -91,7 +95,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		}
 
 		offsetMsecs := time.Now().UnixMilli()
-		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
+		inputTss := prometheus.MustParsePromMetrics(input, offsetMsecs)
 		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
 		// copy inputTss to make sure it is not mutated during TryPush call
@@ -114,13 +118,13 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, 0, false, false, `
+`, false, 0, false, false, `
 metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
 metric{env="bar"} 25
 `)
-	f(``, ``, time.Hour, false, false, `
+	f(``, ``, true, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="foo"} 20
 metric{env="dev"} 15
@@ -130,7 +134,7 @@ metric{env="foo"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, false, false, `
+`, true, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
@@ -140,7 +144,7 @@ metric{env="bar"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, true, false, `
+`, true, time.Hour, true, false, `
 metric{env="test"} 10
 metric{env="dev"} 20
 metric{env="foo"} 15
@@ -150,7 +154,7 @@ metric{env="dev"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, false, true, `
+`, true, time.Hour, false, true, `
 metric{env="foo"} 10
 metric{env="dev"} 20
 metric{env="foo"} 15
@@ -160,7 +164,7 @@ metric{env="dev"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, true, true, `
+`, true, time.Hour, true, true, `
 metric{env="dev"} 10
 metric{env="test"} 20
 metric{env="dev"} 15

app/vmagent/remotewrite/streamaggr.go

@@ -35,6 +35,9 @@ var (
 		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
 		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+	streamAggrGlobalEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all global aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
 
 	// Per URL config
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
@@ -59,6 +62,9 @@ var (
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
 		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+	streamAggrEnableWindows = flagutil.NewArrayBool("remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
@@ -135,7 +141,7 @@ func initStreamAggrConfigGlobal() {
 	}
 	dedupInterval := *streamAggrGlobalDedupInterval
 	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
 }
 
@@ -161,7 +167,7 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
 			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
 		}
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, dropLabels, alias)
 	}
 }
 
@@ -207,6 +213,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
 		KeepInput:            *streamAggrGlobalKeepInput,
+		EnableWindows:        *streamAggrGlobalEnableWindows,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -240,6 +247,7 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
 		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
 		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
+		EnableWindows:        streamAggrEnableWindows.GetOptionalArg(idx),
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmalert-tool/main.go

@@ -51,6 +51,11 @@ Examples:
 						Usage:    `Optional external URL to template in rule's labels or annotations.`,
 						Required: false,
 					},
+					&cli.StringFlag{
+						Name:     "httpListenPort",
+						Usage:    `Optional local port for incoming HTTP requests. If not specified, a random unoccupied port will be used.`,
+						Required: false,
+					},
 					&cli.StringFlag{
 						Name:     "loggerLevel",
 						Usage:    `Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "ERROR").`,
@@ -58,7 +63,7 @@ Examples:
 					},
 				},
 				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url"), c.String("loggerLevel")); failed {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url"), c.String("httpListenPort"), c.String("loggerLevel")); failed {
 						return fmt.Errorf("unittest failed")
 					}
 					return nil

app/vmalert-tool/unittest/testdata/disable-group-label.yaml

@@ -8,8 +8,6 @@ tests:
     input_series:
       - series: 'up{job="vmagent2", instance="localhost:9090"}'
         values: "0+0x1440"
-      - series: "test_query"
-        values: "1x200"
 
     metricsql_expr_test:
       - expr: suquery_interval_test
@@ -34,8 +32,7 @@ tests:
       - eval_time: 0
         alertname: AlwaysFiring
         exp_alerts:
-          - exp_annotations:
-              queryValue: "1"
+          - {}
 
       - eval_time: 0
         alertname: InstanceDown

app/vmalert-tool/unittest/testdata/rules.yaml

@@ -12,8 +12,6 @@ groups:
           dashboard: '{{ $externalURL }}/d/dashboard?orgId=1'
       - alert: AlwaysFiring
         expr: 1
-        annotations:
-          queryValue: "{{ query \"test_query\" | first | value }}"
       - alert: SameAlertNameWithDifferentGroup
         expr: absent(test)
         for: 1m

app/vmalert-tool/unittest/testdata/test1.yaml

@@ -10,8 +10,6 @@ tests:
     input_series:
       - series: "test"
         values: "_x5 1x5 _ stale"
-      - series: "test_query"
-        values: "1x100"
 
     alert_rule_test:
       - eval_time: 1m
@@ -52,8 +50,6 @@ tests:
         values: "0+0x1440"
       - series: "test"
         values: "0+1x1440"
-      - series: "test_query"
-        values: "0+1x100"
 
     metricsql_expr_test:
       - expr: count(ALERTS) by (alertgroup, alertname, alertstate)
@@ -101,17 +97,6 @@ tests:
         exp_alerts:
           - exp_labels:
               cluster: prod
-            exp_annotations:
-              queryValue: "0"
-
-      - eval_time: 5m
-        groupname: group1
-        alertname: AlwaysFiring
-        exp_alerts:
-          - exp_labels:
-              cluster: prod
-            exp_annotations:
-              queryValue: "5"
 
       - eval_time: 0
         groupname: alerts

app/vmalert-tool/unittest/testdata/test2.yaml

@@ -8,8 +8,6 @@ tests:
     input_series:
       - series: 'up{job="vmagent2", instance="localhost:9090"}'
         values: "0+0x1440"
-      - series: "test_query"
-        values: "0+1x200"
 
     metricsql_expr_test:
       - expr: suquery_interval_test
@@ -39,8 +37,6 @@ tests:
         exp_alerts:
           - exp_labels:
               cluster: prod
-            exp_annotations:
-              queryValue: "0"
 
       - eval_time: 0
         groupname: group1

app/vmalert-tool/unittest/unittest.go

@@ -4,13 +4,18 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"gopkg.in/yaml.v2"
@@ -28,7 +33,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -38,15 +42,9 @@ import (
 
 var (
 	storagePath    string
-	httpListenAddr = ":8880"
+	httpListenAddr string
 	// insert series from 1970-01-01T00:00:00
-	testStartTime = time.Unix(0, 0).UTC()
-
-	testPromWriteHTTPPath = "http://127.0.0.1" + httpListenAddr + "/api/v1/write"
-	testDataSourcePath    = "http://127.0.0.1" + httpListenAddr + "/prometheus"
-	testRemoteWritePath   = "http://127.0.0.1" + httpListenAddr
-	testHealthHTTPPath    = "http://127.0.0.1" + httpListenAddr + "/health"
-
+	testStartTime          = time.Unix(0, 0).UTC()
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
@@ -56,7 +54,7 @@ const (
 )
 
 // UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL, logLevel string) bool {
+func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL, httpListenPort, logLevel string) bool {
 	if logLevel != "" {
 		testLogLevel = logLevel
 	}
@@ -64,21 +62,45 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if err != nil {
 		logger.Fatalf("failed to parse external URL: %w", err)
 	}
-	labels := make(map[string]string)
-	for _, s := range externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
-		}
-		labels[s[:n]] = s[n+1:]
-	}
-	if err := templates.Init([]string{}, labels, *eu); err != nil {
+	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
-	storagePath = filepath.Join(os.TempDir(), testStoragePath)
+
+	// set up http server
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/prometheus/api/v1/query":
+			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+			}
+		case "/prometheus/api/v1/write", "/api/v1/write":
+			if err := promremotewrite.InsertHandler(r); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+			}
+		default:
+		}
+	})
+	if httpListenPort == "" {
+		server := httptest.NewServer(handler)
+		httpListenAddr = strings.Split(server.URL, ":")[2]
+		defer server.Close()
+	} else {
+		httpListenAddr = httpListenPort
+		ln, err := net.Listen("tcp", fmt.Sprintf(":%s", httpListenPort))
+		if err != nil {
+			logger.Fatalf("cannot listen on port %s: %v", httpListenPort, err)
+		}
+		go func() {
+			err = http.Serve(ln, handler)
+			if err != nil {
+				logger.Fatalf("cannot start http server: %v", err)
+			}
+			defer ln.Close()
+		}()
+	}
+
+	// adding time.Now().UnixNano() to avoid possible file conflict when multiple processes run on a single host
+	storagePath = filepath.Join(os.TempDir(), testStoragePath, strconv.FormatInt(time.Now().UnixNano(), 10))
 	processFlags()
 	vminsert.Init()
 	vmselect.Init()
@@ -95,22 +117,52 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if len(testfiles) == 0 {
 		logger.Fatalf("no test file found")
 	}
-	_, err = notifier.Init(nil)
+
+	labels := make(map[string]string)
+	for _, s := range externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			logger.Fatalf("missing '=' in `-label`. It must contain label in the form `name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+	_, err = notifier.Init(nil, labels, externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init notifier: %v", err)
 	}
 
 	var failed bool
-	for fileName, file := range testfiles {
-		if err := ruleUnitTest(fileName, file, labels); err != nil {
-			fmt.Println("FAILED")
-			fmt.Printf("failed to run unit test for file %q: \n%v", fileName, err)
-			failed = true
-		} else {
-			fmt.Println("  SUCCESS")
+	runTest := func() bool {
+		for fileName, file := range testfiles {
+			if err := ruleUnitTest(fileName, file, labels); err != nil {
+				fmt.Println("FAILED")
+				fmt.Printf("failed to run unit test for file %q: \n%v", fileName, err)
+				return true
+			}
+			fmt.Println("SUCCESS")
 		}
+		return false
 	}
 
+	finishCh := make(chan struct{}, 1)
+	go func() {
+		failed = runTest()
+		finishCh <- struct{}{}
+	}()
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	select {
+	case sig := <-sigs:
+		fmt.Printf("received signal %s\n", sig)
+		failed = true
+		break
+	case <-finishCh:
+		break
+	}
 	return failed
 }
 
@@ -210,8 +262,8 @@ func processFlags() {
 		{flag: "search.disableCache", value: "true"},
 		// set storage retention time to 100 years, allow to store series from 1970-01-01T00:00:00.
 		{flag: "retentionPeriod", value: "100y"},
-		{flag: "datasource.url", value: testDataSourcePath},
-		{flag: "remoteWrite.url", value: testRemoteWritePath},
+		{flag: "datasource.url", value: fmt.Sprintf("http://127.0.0.1:%s/prometheus", httpListenAddr)},
+		{flag: "remoteWrite.url", value: fmt.Sprintf("http://127.0.0.1:%s", httpListenAddr)},
 		{flag: "notifier.blackhole", value: "true"},
 	} {
 		// panics if flag doesn't exist
@@ -223,27 +275,10 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	var ab flagutil.ArrayBool
-	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
-		switch r.URL.Path {
-		case "/prometheus/api/v1/query":
-			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		case "/prometheus/api/v1/write", "/api/v1/write":
-			if err := promremotewrite.InsertHandler(r); err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		default:
-		}
-		return false
-	})
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {
-		resp, err := http.Get(testHealthHTTPPath)
+		resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%s/health", httpListenAddr))
 		if err != nil {
 			return false
 		}
@@ -265,9 +300,6 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
-		logger.Errorf("cannot stop the webservice: %s", err)
-	}
 	vmstorage.Stop()
 	metrics.UnregisterAllMetrics()
 	fs.MustRemoveAll(storagePath)
@@ -282,7 +314,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	if tg.Interval == nil {
 		tg.Interval = promutils.NewDuration(evalInterval)
 	}
-	err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, testPromWriteHTTPPath)
+	err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, fmt.Sprintf("http://127.0.0.1:%s/api/v1/write", httpListenAddr))
 	if err != nil {
 		return []error{err}
 	}

app/vmalert-tool/unittest/unittest_test.go

@@ -8,7 +8,7 @@ func TestUnitTest_Failure(t *testing.T) {
 	f := func(files []string) {
 		t.Helper()
 
-		failed := UnitTest(files, false, nil, "", "")
+		failed := UnitTest(files, false, nil, "", "", "")
 		if !failed {
 			t.Fatalf("expecting failed test")
 		}
@@ -20,19 +20,20 @@ func TestUnitTest_Failure(t *testing.T) {
 }
 
 func TestUnitTest_Success(t *testing.T) {
-	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
+	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL, httpPort string) {
 		t.Helper()
 
-		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL, "")
+		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL, httpPort, "")
 		if failed {
 			t.Fatalf("unexpected failed test")
 		}
 	}
 
-	// run multi files
-	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
+	// run multi files with random http port
+	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000", "")
 
 	// disable group label
 	// template with null external values
-	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
+	// specify httpListenAddr
+	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "", "8880")
 }

app/vmalert/config/config.go

@@ -88,6 +88,9 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 	if g.EvalOffset.Duration() > g.Interval.Duration() {
 		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
 	}
+	if g.EvalOffset != nil && g.EvalDelay != nil {
+		return fmt.Errorf("eval_offset cannot be used with eval_delay")
+	}
 	if g.Limit < 0 {
 		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
 	}

app/vmalert/config/config_test.go

@@ -9,13 +9,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"gopkg.in/yaml.v2"
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -78,7 +79,7 @@ groups:
 		for i, u := range urls {
 			urls[i] = srv.URL + u
 		}
-		_, err := Parse(urls, templates.ValidateTemplates, true)
+		_, err := Parse(urls, notifier.ValidateTemplates, true)
 		if err != nil && !expErr {
 			t.Fatalf("error parsing URLs %s", err)
 		}
@@ -94,7 +95,7 @@ groups:
 }
 
 func TestParse_Success(t *testing.T) {
-	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, templates.ValidateTemplates, true)
+	_, err := Parse([]string{"testdata/rules/*good.rules", "testdata/dir/*good.*"}, notifier.ValidateTemplates, true)
 	if err != nil {
 		t.Fatalf("error parsing files %s", err)
 	}
@@ -104,7 +105,7 @@ func TestParse_Failure(t *testing.T) {
 	f := func(paths []string, errStrExpected string) {
 		t.Helper()
 
-		_, err := Parse(paths, templates.ValidateTemplates, true)
+		_, err := Parse(paths, notifier.ValidateTemplates, true)
 		if err == nil {
 			t.Fatalf("expected to get error")
 		}
@@ -115,7 +116,7 @@ func TestParse_Failure(t *testing.T) {
 
 	f([]string{"testdata/rules/rules_interval_bad.rules"}, "eval_offset should be smaller than interval")
 	f([]string{"testdata/rules/rules0-bad.rules"}, "unexpected token")
-	f([]string{"testdata/dir/rules0-bad.rules"}, "failed to parse text")
+	f([]string{"testdata/dir/rules0-bad.rules"}, "error parsing annotation")
 	f([]string{"testdata/dir/rules1-bad.rules"}, "duplicate in file")
 	f([]string{"testdata/dir/rules2-bad.rules"}, "function \"unknown\" not defined")
 	f([]string{"testdata/dir/rules3-bad.rules"}, "either `record` or `alert` must be set")
@@ -156,6 +157,19 @@ func TestGroupValidate_Failure(t *testing.T) {
 
 	f(&Group{}, false, "group name must be set")
 
+	f(&Group{
+		Name: "both record and alert are not set",
+		Rules: []Rule{
+			{
+				Expr: "sum(up == 0 ) by (host)",
+				For:  promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
+			},
+		},
+	}, false, "invalid rule")
+
 	f(&Group{
 		Name:     "negative interval",
 		Interval: promutils.NewDuration(-1),
@@ -239,59 +253,6 @@ func TestGroupValidate_Failure(t *testing.T) {
 		},
 	}, false, "duplicate")
 
-	f(&Group{
-		Name: "test graphite with prometheus expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
-
-	f(&Group{
-		Name: "test graphite inherit",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sum(up == 0 ) by (host)",
-			},
-		},
-	}, false, "either `record` or `alert` must be set")
-
-	f(&Group{
-		Name: "test vlogs with prometheus expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
-
-	// validate expressions
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "invalid expression")
-
 	f(&Group{
 		Name: "test thanos",
 		Type: NewRawType("thanos"),
@@ -302,8 +263,20 @@ func TestGroupValidate_Failure(t *testing.T) {
 		},
 	}, true, "unknown datasource type")
 
+	// validate expressions
 	f(&Group{
-		Name: "test graphite",
+		Name: "test prometheus expr",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, true, "bad prometheus expr")
+
+	f(&Group{
+		Name: "test graphite expr",
 		Type: NewGraphiteType(),
 		Rules: []Rule{
 			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
@@ -313,14 +286,63 @@ func TestGroupValidate_Failure(t *testing.T) {
 	}, true, "bad graphite expr")
 
 	f(&Group{
-		Name: "test vlogs",
+		Name: "test vlogs expr",
 		Type: NewVLogsType(),
 		Rules: []Rule{
-			{Alert: "alert", Expr: "stats count(*) as requests", Labels: map[string]string{
-				"description": "some-description",
-			}},
+			{Alert: "alert", Expr: "stats count(*) as requests"},
 		},
 	}, true, "bad LogsQL expr")
+
+	f(&Group{
+		Name: "test vlogs expr",
+		Type: NewVLogsType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"},
+		},
+	}, true, "bad LogsQL expr")
+
+	f(&Group{
+		Name: "test graphite with prometheus expr",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Record: "r1",
+				ID:     1,
+				Expr:   "sumSeries(time('foo.bar',10))",
+				For:    promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Record: "r2",
+				ID:     2,
+				Expr:   "sum(up == 0 ) by (host)",
+			},
+		},
+	}, true, "bad graphite expr")
+
+	f(&Group{
+		Name: "test vlogs with prometheus exp",
+		Type: NewVLogsType(),
+		Rules: []Rule{
+			{
+				Record: "r1",
+				Expr:   "sum(up == 0 ) by (host)",
+				For:    promutils.NewDuration(10 * time.Millisecond),
+			},
+		},
+	}, true, "bad LogsQL expr")
+
+	f(&Group{
+		Name: "test prometheus with vlogs exp",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{
+				Record: "r1",
+				Expr:   "* | stats by (path) count()",
+				For:    promutils.NewDuration(10 * time.Millisecond),
+			},
+		},
+	}, true, "bad prometheus expr")
+
 }
 
 func TestGroupValidate_Success(t *testing.T) {
@@ -329,7 +351,7 @@ func TestGroupValidate_Success(t *testing.T) {
 
 		var validateTplFn ValidateTplFn
 		if validateAnnotations {
-			validateTplFn = templates.ValidateTemplates
+			validateTplFn = notifier.ValidateTemplates
 		}
 		err := group.Validate(validateTplFn, validateExpressions)
 		if err != nil {

app/vmalert/config/types.go

@@ -71,9 +71,18 @@ func (t *Type) ValidateExpr(expr string) error {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
-		if _, err := logstorage.ParseStatsQuery(expr, 0); err != nil {
+		q, err := logstorage.ParseStatsQuery(expr, 0)
+		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
+		fields, _ := q.GetStatsByFields()
+		for i := range fields {
+			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
+			// making the result meaningless and may lead to cardinality issues.
+			if fields[i] == "_time" {
+				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
+			}
+		}
 	default:
 		return fmt.Errorf("unknown datasource type=%q", t.Name)
 	}

app/vmalert/main.go

@@ -83,6 +83,7 @@ absolute path to all .tpl files in root.
 
 var (
 	alertURLGeneratorFn notifier.AlertURLGenerator
+	extURL              *url.URL
 )
 
 func main() {
@@ -98,29 +99,18 @@ func main() {
 	logger.Init()
 
 	var err error
-	extURL, err := getExternalURL(*externalURL)
+	extURL, err = getExternalURL(*externalURL)
 	if err != nil {
 		logger.Fatalf("failed to init external.url %q: %s", *externalURL, err)
 	}
-	externalls := make(map[string]string)
-	for _, s := range *externalLabels {
-		if len(s) == 0 {
-			continue
-		}
-		n := strings.IndexByte(s, '=')
-		if n < 0 {
-			logger.Fatalf("wrong format in `-external.label`, it must contain label as `Name=value`; got %q", s)
-		}
-		externalls[s[:n]] = s[n+1:]
-	}
 
-	err = templates.Init(*ruleTemplatesPath, externalls, *extURL)
+	err = templates.Load(*ruleTemplatesPath, *extURL)
 	if err != nil {
 		logger.Fatalf("failed to load template %q: %s", *ruleTemplatesPath, err)
 	}
 
 	if *dryRun {
-		groups, err := config.Parse(*rulePath, templates.ValidateTemplates, true)
+		groups, err := config.Parse(*rulePath, notifier.ValidateTemplates, true)
 		if err != nil {
 			logger.Fatalf("failed to parse %q: %s", *rulePath, err)
 		}
@@ -137,7 +127,7 @@ func main() {
 
 	var validateTplFn config.ValidateTplFn
 	if *validateTemplates {
-		validateTplFn = templates.ValidateTemplates
+		validateTplFn = notifier.ValidateTemplates
 	}
 
 	if *replayFrom != "" {
@@ -166,7 +156,7 @@ func main() {
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	manager, err := newManager(ctx, externalls)
+	manager, err := newManager(ctx)
 	if err != nil {
 		logger.Fatalf("failed to init: %s", err)
 	}
@@ -213,13 +203,25 @@ var (
 	configTimestamp    = metrics.NewCounter(`vmalert_config_last_reload_success_timestamp_seconds`)
 )
 
-func newManager(ctx context.Context, externalls map[string]string) (*manager, error) {
+func newManager(ctx context.Context) (*manager, error) {
 	q, err := datasource.Init(nil)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init datasource: %w", err)
 	}
 
-	nts, err := notifier.Init(alertURLGeneratorFn)
+	labels := make(map[string]string)
+	for _, s := range *externalLabels {
+		if len(s) == 0 {
+			continue
+		}
+		n := strings.IndexByte(s, '=')
+		if n < 0 {
+			return nil, fmt.Errorf("missing '=' in `-label`. It must contain label in the form `Name=value`; got %q", s)
+		}
+		labels[s[:n]] = s[n+1:]
+	}
+
+	nts, err := notifier.Init(alertURLGeneratorFn, labels, *externalURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init notifier: %w", err)
 	}
@@ -227,7 +229,7 @@ func newManager(ctx context.Context, externalls map[string]string) (*manager, er
 		groups:         make(map[uint64]*rule.Group),
 		querierBuilder: q,
 		notifiers:      nts,
-		labels:         externalls,
+		labels:         labels,
 	}
 	rw, err := remotewrite.Init(ctx)
 	if err != nil {
@@ -291,36 +293,24 @@ func getAlertURLGenerator(externalURL *url.URL, externalAlertSource string, vali
 		}, nil
 	}
 	if validateTemplate {
-		if err := templates.ValidateTemplates(map[string]string{
+		if err := notifier.ValidateTemplates(map[string]string{
 			"tpl": externalAlertSource,
 		}); err != nil {
-			return nil, fmt.Errorf("cannot parse `external.alert.source` %q: %w", externalAlertSource, err)
+			return nil, fmt.Errorf("error validating source template %s: %w", externalAlertSource, err)
 		}
 	}
-	var err error
-	tmpl := templates.GetCurrentTmpl()
-	tmpl, err = templates.ParseWithFixedHeader(externalAlertSource, tmpl)
-	if err != nil {
-		return nil, err
+	m := map[string]string{
+		"tpl": externalAlertSource,
 	}
 	return func(alert notifier.Alert) string {
-		// recreate template if it was changed during config reload
-		cm := templates.GetCurrentTmpl()
-		if tmpl.Name() != cm.Name() {
-			tmpl = cm
-			tmpl, err = templates.ParseWithFixedHeader(externalAlertSource, tmpl)
-			if err != nil {
-				logger.Errorf("cannot parse `external.alert.source` %q: %w", externalAlertSource, err)
-				return fmt.Sprintf("%s/%s", externalURL, externalAlertSource)
-			}
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return nil, fmt.Errorf("`query` template isn't supported for alert source template")
 		}
-		tplData := alert.ToTplData()
-		rr, err := templates.ExecuteWithTemplate(tplData, tmpl)
+		templated, err := alert.ExecTemplate(qFn, alert.Labels, m)
 		if err != nil {
-			logger.Errorf("can not template alert source: %v", err)
-			return fmt.Sprintf("%s/%s", externalURL, externalAlertSource)
+			logger.Errorf("cannot template alert source: %s", err)
 		}
-		return fmt.Sprintf("%s/%s", externalURL, rr)
+		return fmt.Sprintf("%s/%s", externalURL, templated["tpl"])
 	}, nil
 }
 
@@ -344,7 +334,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 
 	var validateTplFn config.ValidateTplFn
 	if *validateTemplates {
-		validateTplFn = templates.ValidateTemplates
+		validateTplFn = notifier.ValidateTemplates
 	}
 
 	// init metrics for config state with positive values to improve alerting conditions
@@ -373,7 +363,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("failed to reload notifier config: %s", err)
 			continue
 		}
-		err := templates.LoadTemplateFile(*ruleTemplatesPath)
+		err := templates.Load(*ruleTemplatesPath, *extURL)
 		if err != nil {
 			setConfigError(err)
 			logger.Errorf("failed to load new templates: %s", err)
@@ -386,6 +376,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			continue
 		}
 		if configsEqual(newGroupsCfg, groupsCfg) {
+			templates.Reload()
 			// set success to 1 since previous reload could have been unsuccessful
 			// do not update configTimestamp as config version remains old.
 			configSuccess.Set(1)
@@ -399,6 +390,7 @@ func configReload(ctx context.Context, m *manager, groupsCfg []config.Group, sig
 			logger.Errorf("error while reloading rules: %s", err)
 			continue
 		}
+		templates.Reload()
 		groupsCfg = newGroupsCfg
 		setConfigSuccessAt(fasttime.UnixTimestamp())
 		logger.Infof("Rules reloaded successfully from %q", *rulePath)
@@ -413,6 +405,9 @@ func configsEqual(a, b []config.Group) bool {
 		if a[i].Checksum != b[i].Checksum {
 			return false
 		}
+		if a[i].File != b[i].File {
+			return false
+		}
 	}
 	return true
 }

app/vmalert/main_test.go

@@ -74,7 +74,10 @@ func TestGetAlertURLGenerator(t *testing.T) {
 
 func TestConfigReload(t *testing.T) {
 	originalRulePath := *rulePath
+	originalExternalURL := extURL
+	extURL = &url.URL{}
 	defer func() {
+		extURL = originalExternalURL
 		*rulePath = originalRulePath
 	}()
 

app/vmalert/manager.go

@@ -160,8 +160,8 @@ func (m *manager) update(ctx context.Context, groupsCfg []config.Group, restore
 			// it is important to call InterruptEval before the update, because cancel fn
 			// can be re-assigned during the update.
 			item.old.InterruptEval()
-			go func(old *rule.Group, new *rule.Group) {
-				old.UpdateWith(new)
+			go func(oldGroup *rule.Group, newGroup *rule.Group) {
+				oldGroup.UpdateWith(newGroup)
 				wg.Done()
 			}(item.old, item.new)
 		}

app/vmalert/manager_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())
@@ -72,7 +72,7 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
 				rnd := r.Intn(len(paths))
-				cfg, err := config.Parse([]string{paths[rnd]}, templates.ValidateTemplates, true)
+				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
@@ -135,7 +135,7 @@ func TestManagerUpdate_Success(t *testing.T) {
 			t.Fatalf("failed to complete initial rules update: %s", err)
 		}
 
-		cfgUpdate, err := config.Parse([]string{updatePath}, templates.ValidateTemplates, true)
+		cfgUpdate, err := config.Parse([]string{updatePath}, notifier.ValidateTemplates, true)
 		if err == nil { // update can fail and that's expected
 			_ = m.update(ctx, cfgUpdate, false)
 		}
@@ -326,7 +326,7 @@ func loadCfg(t *testing.T, path []string, validateAnnotations, validateExpressio
 	t.Helper()
 	var validateTplFn config.ValidateTplFn
 	if validateAnnotations {
-		validateTplFn = templates.ValidateTemplates
+		validateTplFn = notifier.ValidateTemplates
 	}
 	cfg, err := config.Parse(path, validateTplFn, validateExpressions)
 	if err != nil {

app/vmalert/notifier/alert.go

@@ -1,9 +1,15 @@
 package notifier
 
 import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+	textTpl "text/template"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -70,18 +76,118 @@ func (as AlertState) String() string {
 	return "inactive"
 }
 
-// ToTplData converts Alert to AlertTplData,
-// which only exposes necessary fields for template.
-func (a Alert) ToTplData() templates.AlertTplData {
-	return templates.AlertTplData{
+// AlertTplData is used to execute templating
+type AlertTplData struct {
+	Labels   map[string]string
+	Value    float64
+	Expr     string
+	AlertID  uint64
+	GroupID  uint64
+	ActiveAt time.Time
+	For      time.Duration
+}
+
+var tplHeaders = []string{
+	"{{ $value := .Value }}",
+	"{{ $labels := .Labels }}",
+	"{{ $expr := .Expr }}",
+	"{{ $externalLabels := .ExternalLabels }}",
+	"{{ $externalURL := .ExternalURL }}",
+	"{{ $alertID := .AlertID }}",
+	"{{ $groupID := .GroupID }}",
+	"{{ $activeAt := .ActiveAt }}",
+	"{{ $for := .For }}",
+}
+
+// ExecTemplate executes the Alert template for given
+// map of annotations.
+// Every alert could have a different datasource, so function
+// requires a queryFunction as an argument.
+func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string]string) (map[string]string, error) {
+	tplData := AlertTplData{
 		Value:    a.Value,
-		Labels:   a.Labels,
+		Labels:   labels,
 		Expr:     a.Expr,
 		AlertID:  a.ID,
 		GroupID:  a.GroupID,
 		ActiveAt: a.ActiveAt,
 		For:      a.For,
 	}
+	return ExecTemplate(q, annotations, tplData)
+}
+
+// ExecTemplate executes the given template for given annotations map.
+func ExecTemplate(q templates.QueryFn, annotations map[string]string, tplData AlertTplData) (map[string]string, error) {
+	tmpl, err := templates.GetWithFuncs(templates.FuncsWithQuery(q))
+	if err != nil {
+		return nil, fmt.Errorf("error cloning template: %w", err)
+	}
+	return templateAnnotations(annotations, tplData, tmpl, true)
+}
+
+// ValidateTemplates validate annotations for possible template error, uses empty data for template population
+func ValidateTemplates(annotations map[string]string) error {
+	tmpl, err := templates.GetWithFuncs(nil)
+	if err != nil {
+		return err
+	}
+	_, err = templateAnnotations(annotations, AlertTplData{
+		Labels: map[string]string{},
+		Value:  0,
+	}, tmpl, false)
+	return err
+}
+
+func templateAnnotations(annotations map[string]string, data AlertTplData, tmpl *textTpl.Template, execute bool) (map[string]string, error) {
+	var builder strings.Builder
+	var buf bytes.Buffer
+	eg := new(utils.ErrGroup)
+	r := make(map[string]string, len(annotations))
+	tData := tplData{data, externalLabels, externalURL}
+	header := strings.Join(tplHeaders, "")
+	for key, text := range annotations {
+		// simple check to skip text without template
+		if !strings.Contains(text, "{{") || !strings.Contains(text, "}}") {
+			r[key] = text
+			continue
+		}
+
+		buf.Reset()
+		builder.Reset()
+		builder.Grow(len(header) + len(text))
+		builder.WriteString(header)
+		builder.WriteString(text)
+		// clone a new template for each parse to avoid collision
+		ctmpl, _ := tmpl.Clone()
+		ctmpl = ctmpl.Option("missingkey=zero")
+		if err := templateAnnotation(&buf, builder.String(), tData, ctmpl, execute); err != nil {
+			r[key] = text
+			eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
+			continue
+		}
+		r[key] = buf.String()
+	}
+	return r, eg.Err()
+}
+
+type tplData struct {
+	AlertTplData
+	ExternalLabels map[string]string
+	ExternalURL    string
+}
+
+func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
+	tpl, err := tpl.Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation template: %w", err)
+	}
+	if !execute {
+		return nil
+	}
+	if err = tpl.Execute(dst, data); err != nil {
+		return fmt.Errorf("error evaluating annotation template: %w", err)
+	}
+	return nil
 }
 
 func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {

app/vmalert/notifier/alert_test.go

@@ -1,13 +1,207 @@
 package notifier
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
+func TestAlertExecTemplate(t *testing.T) {
+	extLabels := make(map[string]string)
+	const (
+		extCluster = "prod"
+		extDC      = "east"
+		extURL     = "https://foo.bar"
+	)
+	extLabels["cluster"] = extCluster
+	extLabels["dc"] = extDC
+	_, err := Init(nil, extLabels, extURL)
+	checkErr(t, err)
+
+	f := func(alert *Alert, annotations map[string]string, tplExpected map[string]string) {
+		t.Helper()
+
+		if err := ValidateTemplates(annotations); err != nil {
+			t.Fatalf("cannot validate annotations: %s", err)
+		}
+
+		qFn := func(_ string) ([]datasource.Metric, error) {
+			return []datasource.Metric{
+				{
+					Labels: []prompbmarshal.Label{
+						{Name: "foo", Value: "bar"},
+						{Name: "baz", Value: "qux"},
+					},
+					Values:     []float64{1},
+					Timestamps: []int64{1},
+				},
+				{
+					Labels: []prompbmarshal.Label{
+						{Name: "foo", Value: "garply"},
+						{Name: "baz", Value: "fred"},
+					},
+					Values:     []float64{2},
+					Timestamps: []int64{1},
+				},
+			}, nil
+		}
+
+		tpl, err := alert.ExecTemplate(qFn, alert.Labels, annotations)
+		if err != nil {
+			t.Fatalf("cannot execute template: %s", err)
+		}
+		if len(tpl) != len(tplExpected) {
+			t.Fatalf("unexpected number of elements; got %d; want %d", len(tpl), len(tplExpected))
+		}
+		for k := range tplExpected {
+			got, exp := tpl[k], tplExpected[k]
+			if got != exp {
+				t.Fatalf("unexpected template for key=%q; got %q; want %q", k, got, exp)
+			}
+		}
+	}
+
+	// empty-alert
+	f(&Alert{}, map[string]string{}, map[string]string{})
+
+	// no-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"summary":     "it's a test summary",
+		"description": "it's a test description",
+	}, map[string]string{
+		"summary":     "it's a test summary",
+		"description": "it's a test description",
+	})
+
+	// label-template
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+		For: 5 * time.Minute,
+	}, map[string]string{
+		"summary":     "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
+	}, map[string]string{
+		"summary":     "Too high connection number for localhost for job staging",
+		"description": "It is 10000 connections for localhost for more than 5m0s",
+	})
+
+	// label template override
+	f(&Alert{
+		Value: 1e4,
+	}, map[string]string{
+		"summary":     `{{- define "default.template" -}} {{ printf "summary" }} {{- end -}} {{ template "default.template" . }}`,
+		"description": `{{ template "default.template" . }}`,
+		"value":       `{{$value }}`,
+	}, map[string]string{
+		"summary":     "summary",
+		"description": "",
+		"value":       "10000",
+	})
+
+	// expression-template
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}<0`,
+	}, map[string]string{
+		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
+		"exprEscapedPath":   "{{ $expr|pathEscape }}",
+		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
+		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
+		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
+	}, map[string]string{
+		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
+		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
+		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
+		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
+		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
+	})
+
+	// query
+	f(&Alert{
+		Expr: `vm_rows{"label"="bar"}>0`,
+	}, map[string]string{
+		"summary": `{{ query "foo" | first | value }}`,
+		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
+	}, map[string]string{
+		"summary": "1",
+		"desc":    "bar 1;garply 2;",
+	})
+
+	// external
+	f(&Alert{
+		Value: 1e4,
+		Labels: map[string]string{
+			"job":      "staging",
+			"instance": "localhost",
+		},
+	}, map[string]string{
+		"url":         "{{ $externalURL }}",
+		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
+		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
+	}, map[string]string{
+		"url":         extURL,
+		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
+		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
+	})
+
+	// alert and group IDs
+	f(&Alert{
+		ID:      42,
+		GroupID: 24,
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
+	}, map[string]string{
+		"url": "/api/v1/alert?alertID=42&groupID=24",
+	})
+
+	// ActiveAt time
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
+	}, map[string]string{
+		"diagram": "![](http://example.com?render=1660941298",
+	})
+
+	// ActiveAt time is nil
+	f(&Alert{}, map[string]string{
+		"default_time": "{{$activeAt}}",
+	}, map[string]string{
+		"default_time": "0001-01-01 00:00:00 +0000 UTC",
+	})
+
+	// ActiveAt custom format
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
+	}, map[string]string{
+		"fire_time": "2022/08/19 20:34:58",
+	})
+
+	// ActiveAt query range
+	f(&Alert{
+		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
+	}, map[string]string{
+		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
+	}, map[string]string{
+		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
+	})
+}
+
 func TestAlert_toPromLabels(t *testing.T) {
 	fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
 		t.Helper()

app/vmalert/notifier/alertmanager.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -29,25 +31,34 @@ type AlertManager struct {
 	// stores already parsed RelabelConfigs object
 	relabelConfigs *promrelabel.ParsedConfigs
 
-	metrics *metrics
+	metrics *notifierMetrics
 }
 
-type metrics struct {
-	alertsSent       *utils.Counter
-	alertsSendErrors *utils.Counter
+type notifierMetrics struct {
+	set *metrics.Set
+
+	alertsSent       *metrics.Counter
+	alertsSendErrors *metrics.Counter
 }
 
-func newMetrics(addr string) *metrics {
-	return &metrics{
-		alertsSent:       utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
-		alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+func newNotifierMetrics(addr string) *notifierMetrics {
+	set := metrics.NewSet()
+	metrics.RegisterSet(set)
+
+	return &notifierMetrics{
+		set:              set,
+		alertsSent:       set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors: set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
 	}
 }
 
+func (nm *notifierMetrics) close() {
+	metrics.UnregisterSet(nm.set, true)
+}
+
 // Close is a destructor method for AlertManager
 func (am *AlertManager) Close() {
-	am.metrics.alertsSent.Unregister()
-	am.metrics.alertsSendErrors.Unregister()
+	am.metrics.close()
 }
 
 // Addr returns address where alerts are sent.
@@ -70,7 +81,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[st
 
 func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	b := &bytes.Buffer{}
-	alertsToSend := alerts[:0]
+	alertsToSend := make([]Alert, 0, len(alerts))
 	lblss := make([][]prompbmarshal.Label, 0, len(alerts))
 	for _, a := range alerts {
 		lbls := a.applyRelabelingIfNeeded(am.relabelConfigs)
@@ -180,6 +191,6 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 		relabelConfigs: relabelCfg,
 		client:         &http.Client{Transport: tr},
 		timeout:        timeout,
-		metrics:        newMetrics(alertManagerURL),
+		metrics:        newNotifierMetrics(alertManagerURL),
 	}, nil
 }

app/vmalert/notifier/init.go

@@ -3,6 +3,7 @@ package notifier
 import (
 	"flag"
 	"fmt"
+	"net/url"
 	"strings"
 	"time"
 
@@ -53,6 +54,7 @@ var (
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 	oauth2Scopes = flagutil.NewArrayString("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	sendTimeout = flagutil.NewArrayDuration("notifier.sendTimeout", 10*time.Second, "Timeout when sending alerts to the corresponding -notifier.url")
 )
 
 // cw holds a configWatcher for configPath configuration file
@@ -72,6 +74,15 @@ func Reload() error {
 
 var staticNotifiersFn func() []Notifier
 
+var (
+	// externalLabels is a global variable for holding external labels configured via flags
+	// It is supposed to be inited via Init function only.
+	externalLabels map[string]string
+	// externalURL is a global variable for holding external URL value configured via flag
+	// It is supposed to be inited via Init function only.
+	externalURL string
+)
+
 // Init returns a function for retrieving actual list of Notifier objects.
 // Init works in two mods:
 //   - configuration via flags (for backward compatibility). Is always static
@@ -79,7 +90,14 @@ var staticNotifiersFn func() []Notifier
 //   - configuration via file. Supports live reloads and service discovery.
 //
 // Init returns an error if both mods are used.
-func Init(gen AlertURLGenerator) (func() []Notifier, error) {
+func Init(gen AlertURLGenerator, extLabels map[string]string, extURL string) (func() []Notifier, error) {
+	externalURL = extURL
+	externalLabels = extLabels
+	_, err := url.Parse(externalURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse external URL: %w", err)
+	}
+
 	if *blackHole {
 		if len(*addrs) > 0 || *configPath != "" {
 			return nil, fmt.Errorf("only one of -notifier.blackhole, -notifier.url and -notifier.config flags must be specified")
@@ -109,7 +127,6 @@ func Init(gen AlertURLGenerator) (func() []Notifier, error) {
 		return staticNotifiersFn, nil
 	}
 
-	var err error
 	cw, err = newWatcher(*configPath, gen)
 	if err != nil {
 		return nil, fmt.Errorf("failed to init config watcher: %w", err)
@@ -159,7 +176,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}
 
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Second*10)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
 			return nil, err
 		}

app/vmalert/notifier/init_test.go

@@ -12,7 +12,7 @@ func TestInit(t *testing.T) {
 
 	*addrs = flagutil.ArrayString{"127.0.0.1", "127.0.0.2"}
 
-	fn, err := Init(nil)
+	fn, err := Init(nil, nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}
@@ -52,7 +52,7 @@ func TestInitNegative(t *testing.T) {
 		*configPath = path
 		*addrs = flagutil.ArrayString{addr}
 		*blackHole = bh
-		if _, err := Init(nil); err == nil {
+		if _, err := Init(nil, nil, ""); err == nil {
 			t.Fatalf("expected to get error; got nil instead")
 		}
 	}
@@ -69,7 +69,7 @@ func TestBlackHole(t *testing.T) {
 
 	*blackHole = true
 
-	fn, err := Init(nil)
+	fn, err := Init(nil, nil, "")
 	if err != nil {
 		t.Fatalf("%s", err)
 	}

app/vmalert/notifier/notifier_blackhole.go

@@ -6,7 +6,7 @@ import "context"
 // to be sent.
 type blackHoleNotifier struct {
 	addr    string
-	metrics *metrics
+	metrics *notifierMetrics
 }
 
 // Send will send no notifications, but increase the metric.
@@ -22,8 +22,7 @@ func (bh blackHoleNotifier) Addr() string {
 
 // Close unregister the metrics
 func (bh *blackHoleNotifier) Close() {
-	bh.metrics.alertsSent.Unregister()
-	bh.metrics.alertsSendErrors.Unregister()
+	bh.metrics.close()
 }
 
 // newBlackHoleNotifier creates a new blackHoleNotifier
@@ -31,6 +30,6 @@ func newBlackHoleNotifier() *blackHoleNotifier {
 	address := "blackhole"
 	return &blackHoleNotifier{
 		addr:    address,
-		metrics: newMetrics(address),
+		metrics: newNotifierMetrics(address),
 	}
 }

app/vmalert/notifier/package_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{"testdata/templates/*good.tmpl"}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{"testdata/templates/*good.tmpl"}, url.URL{}); err != nil {
 		os.Exit(1)
 	}
 	os.Exit(m.Run())

app/vmalert/remotewrite/init.go

@@ -36,7 +36,7 @@ var (
 
 	maxQueueSize  = flag.Int("remoteWrite.maxQueueSize", defaultMaxQueueSize, "Defines the max number of pending datapoints to remote write endpoint")
 	maxBatchSize  = flag.Int("remoteWrite.maxBatchSize", defaultMaxBatchSize, "Defines max number of timeseries to be flushed at once")
-	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint")
+	concurrency   = flag.Int("remoteWrite.concurrency", defaultConcurrency, "Defines number of writers for concurrent writing into remote write endpoint. Default value depends on the number of available CPU cores.")
 	flushInterval = flag.Duration("remoteWrite.flushInterval", defaultFlushInterval, "Defines interval of flushes to remote write endpoint")
 
 	tlsInsecureSkipVerify = flag.Bool("remoteWrite.tlsInsecureSkipVerify", false, "Whether to skip tls verification when connecting to -remoteWrite.url")

app/vmalert/rule/alerting.go

@@ -7,9 +7,10 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	textTpl "text/template"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -48,12 +49,6 @@ type AlertingRule struct {
 	state *ruleState
 
 	metrics *alertingRuleMetrics
-
-	// set rootTemplateName with loaded global template name,
-	// so we can check if the template has changed when evaluating.
-	rootTemplateName     string
-	LabelTemplates       map[string]*textTpl.Template
-	AnnotationsTemplates map[string]*textTpl.Template
 }
 
 type alertingRuleMetrics struct {
@@ -64,6 +59,69 @@ type alertingRuleMetrics struct {
 	seriesFetched *utils.Gauge
 }
 
+func newAlertingRuleMetrics(set *metrics.Set, ar *AlertingRule) *alertingRuleMetrics {
+	labels := fmt.Sprintf(`alertname=%q, group=%q, file=%q, id="%d"`, ar.Name, ar.GroupName, ar.File, ar.ID())
+	arm := &alertingRuleMetrics{}
+
+	arm.pending = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+		func() float64 {
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StatePending {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	arm.active = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+		func() float64 {
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StateFiring {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	arm.errors = utils.NewCounter(set, fmt.Sprintf(`vmalert_alerting_rules_errors_total{%s}`, labels))
+	arm.samples = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			return float64(e.Samples)
+		})
+	arm.seriesFetched = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_series_fetched{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			if e.SeriesFetched == nil {
+				// means seriesFetched is unsupported
+				return -1
+			}
+			seriesFetched := float64(*e.SeriesFetched)
+			if seriesFetched == 0 && e.Samples > 0 {
+				// `alert: 0.95` will fetch no series
+				// but will get one time series in response.
+				seriesFetched = float64(e.Samples)
+			}
+			return seriesFetched
+		})
+	return arm
+}
+
+func (arm *alertingRuleMetrics) close() {
+	if arm == nil {
+		return
+	}
+	arm.errors.Unregister()
+	arm.active.Unregister()
+	arm.pending.Unregister()
+	arm.samples.Unregister()
+	arm.seriesFetched.Unregister()
+}
+
 // NewAlertingRule creates a new AlertingRule
 func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
@@ -88,8 +146,7 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			Headers:                   group.Headers,
 			Debug:                     cfg.Debug,
 		}),
-		alerts:  make(map[uint64]*notifier.Alert),
-		metrics: &alertingRuleMetrics{},
+		alerts: make(map[uint64]*notifier.Alert),
 	}
 
 	entrySize := *ruleUpdateEntriesLimit
@@ -102,101 +159,17 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 	ar.state = &ruleState{
 		entries: make([]StateEntry, entrySize),
 	}
-
-	labels := fmt.Sprintf(`alertname=%q, group=%q, file=%q, id="%d"`, ar.Name, group.Name, group.File, ar.ID())
-	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
-		func() float64 {
-			ar.alertsMu.RLock()
-			defer ar.alertsMu.RUnlock()
-			var num int
-			for _, a := range ar.alerts {
-				if a.State == notifier.StatePending {
-					num++
-				}
-			}
-			return float64(num)
-		})
-	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
-		func() float64 {
-			ar.alertsMu.RLock()
-			defer ar.alertsMu.RUnlock()
-			var num int
-			for _, a := range ar.alerts {
-				if a.State == notifier.StateFiring {
-					num++
-				}
-			}
-			return float64(num)
-		})
-	ar.metrics.errors = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_alerting_rules_errors_total{%s}`, labels))
-	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
-		func() float64 {
-			e := ar.state.getLast()
-			return float64(e.Samples)
-		})
-	ar.metrics.seriesFetched = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_series_fetched{%s}`, labels),
-		func() float64 {
-			e := ar.state.getLast()
-			if e.SeriesFetched == nil {
-				// means seriesFetched is unsupported
-				return -1
-			}
-			seriesFetched := float64(*e.SeriesFetched)
-			if seriesFetched == 0 && e.Samples > 0 {
-				// `alert: 0.95` will fetch no series
-				// but will get one time series in response.
-				seriesFetched = float64(e.Samples)
-			}
-			return seriesFetched
-		})
-
-	ar.initTemplate()
-
+	ar.metrics = newAlertingRuleMetrics(group.metrics.set, ar)
 	return ar
 }
 
-// initTemplate pre-creates templates that can be reused in execution.
-func (ar *AlertingRule) initTemplate() {
-	currentTmpl := templates.GetCurrentTmpl()
-	ar.rootTemplateName = currentTmpl.Name()
-	ar.LabelTemplates = make(map[string]*textTpl.Template, len(ar.Labels))
-	ar.AnnotationsTemplates = make(map[string]*textTpl.Template, len(ar.Annotations))
-	for k, v := range ar.Labels {
-		var err error
-		tmpl, _ := currentTmpl.Clone()
-		tmpl, err = templates.ParseWithFixedHeader(v, tmpl)
-		if err != nil {
-			// parse can fail in two cases:
-			// 1. the text contains `query` function, which is not supported during rule initialization.
-			// 2. the text itself is invalid.
-			// In both case, we skip the error here, and try it again during rule execution.
-			continue
-		}
-		ar.LabelTemplates[k] = tmpl
-
-	}
-	for k, v := range ar.Annotations {
-		var err error
-		tmpl, _ := currentTmpl.Clone()
-		tmpl, err = templates.ParseWithFixedHeader(v, tmpl)
-		if err != nil {
-			// parse can fail in two cases:
-			// 1. the text contains `query` function, which is not supported during rule initialization.
-			// 2. the text itself is invalid.
-			// In both case, we skip the error here, and try it again during rule execution.
-			continue
-		}
-		ar.AnnotationsTemplates[k] = tmpl
-	}
+func (ar *AlertingRule) registerMetrics(g *Group) {
+	ar.metrics = newAlertingRuleMetrics(g.metrics.set, ar)
 }
 
 // close unregisters rule metrics
-func (ar *AlertingRule) close() {
-	ar.metrics.active.Unregister()
-	ar.metrics.pending.Unregister()
-	ar.metrics.errors.Unregister()
-	ar.metrics.samples.Unregister()
-	ar.metrics.seriesFetched.Unregister()
+func (ar *AlertingRule) unregisterMetrics() {
+	ar.metrics.close()
 }
 
 // String implements Stringer interface
@@ -270,8 +243,6 @@ func (ar *AlertingRule) updateWith(r Rule) error {
 	ar.KeepFiringFor = nr.KeepFiringFor
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
-	ar.initTemplate()
-
 	ar.EvalInterval = nr.EvalInterval
 	ar.Debug = nr.Debug
 	ar.q = nr.q
@@ -326,32 +297,15 @@ func (ar *AlertingRule) toLabels(m datasource.Metric, qFn templates.QueryFn) (*l
 		}
 		ls.processed[l.Name] = l.Value
 	}
-	extraLabels := make(map[string]string, len(ar.Labels))
-	// compare to annotation, label value can only use limited variables for now
-	labelTplData := templates.AlertTplData{
+
+	extraLabels, err := notifier.ExecTemplate(qFn, ar.Labels, notifier.AlertTplData{
 		Labels: ls.origin,
 		Value:  m.Values[0],
 		Expr:   ar.Expr,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
-	for k := range ar.Labels {
-		if ar.LabelTemplates[k] == nil {
-			// this label may contain `query` function, which requires creating new template with query function in each evaluation.
-			v, err := templates.ExecuteWithoutTemplate(qFn, ar.Labels[k], labelTplData)
-			if err != nil {
-				logger.Errorf("error templating label %q for rule %q: %w", ar.Labels[k], ar.Name, err)
-				v = ar.Labels[k]
-			}
-			extraLabels[k] = v
-			continue
-		}
-		v, err := templates.ExecuteWithTemplate(labelTplData, ar.LabelTemplates[k])
-		if err != nil {
-			logger.Errorf("error templating label %q for rule %q: %w", ar.Labels[k], ar.Name, err)
-			v = ar.Labels[k]
-		}
-		extraLabels[k] = v
-	}
-
 	for k, v := range extraLabels {
 		ls.add(k, v)
 	}
@@ -577,18 +531,12 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 }
 
 func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
-	// check if the rule template has changed during reload,
-	// if so, label&annotation templates must be re-created.
-	if ar.rootTemplateName != templates.GetCurrentTmpl().Name() {
-		ar.initTemplate()
-	}
-
 	ls, err := ar.toLabels(m, qFn)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to expand labels: %w", err)
 	}
-	extraAnnotation := make(map[string]string, len(ar.Annotations))
-	annotationTplData := templates.AlertTplData{
+
+	tplData := notifier.AlertTplData{
 		Value:    m.Values[0],
 		Labels:   ls.origin,
 		Expr:     ar.Expr,
@@ -597,26 +545,11 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 		ActiveAt: ts,
 		For:      ar.For,
 	}
-	for k := range ar.Annotations {
-		if ar.AnnotationsTemplates[k] == nil {
-			// this label may contain `query` function, which requires creating new template with query function in each evaluation.
-			v, err := templates.ExecuteWithoutTemplate(qFn, ar.Annotations[k], annotationTplData)
-			if err != nil {
-				logger.Errorf("error templating annotation %q for rule %q: %w", ar.Annotations[k], ar.Name, err)
-				v = ar.Annotations[k]
-			}
-			extraAnnotation[k] = v
-			continue
-		}
-		v, err := templates.ExecuteWithTemplate(annotationTplData, ar.AnnotationsTemplates[k])
-		if err != nil {
-			logger.Errorf("error templating annotation %q for rule %q: %w", ar.Annotations[k], ar.Name, err)
-			v = ar.Annotations[k]
-		}
-		extraAnnotation[k] = v
+	as, err := notifier.ExecTemplate(qFn, ar.Annotations, tplData)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to template annotations: %w", err)
 	}
-
-	return ls, extraAnnotation, nil
+	return ls, as, nil
 }
 
 // toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts

app/vmalert/rule/alerting_test.go

@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -1046,7 +1048,6 @@ func TestAlertingRule_Template(t *testing.T) {
 		}
 		fq.Add(metrics...)
 
-		rule.initTemplate()
 		if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -1249,13 +1250,17 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 		EvalInterval: waitFor,
 		alerts:       make(map[uint64]*notifier.Alert),
 		state:        &ruleState{entries: make([]StateEntry, 10)},
-		metrics: &alertingRuleMetrics{
-			errors: utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_alerting_rules_errors_total{alertname=%q}`, name)),
-		},
+		metrics:      getTestAlertingRuleMetrics(name),
 	}
 	return &rule
 }
 
+func getTestAlertingRuleMetrics(name string) *alertingRuleMetrics {
+	m := &alertingRuleMetrics{}
+	m.errors = utils.NewCounter(metrics.NewSet(), fmt.Sprintf(`vmalert_alerting_rules_errors_total{alertname=%q}`, name))
+	return m
+}
+
 func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, keepFiringFor time.Duration, annotation map[string]string) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	if evalInterval != 0 {
@@ -1263,7 +1268,6 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 	}
 	rule.KeepFiringFor = keepFiringFor
 	rule.Annotations = annotation
-	rule.initTemplate()
 	return rule
 }
 

app/vmalert/rule/group.go

@@ -13,6 +13,8 @@ import (
 
 	"github.com/cheggaaa/pb/v3"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -20,20 +22,20 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
 	ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
 		"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
-	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier")
+	resendDelay        = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
 	maxResolveDuration = flag.Duration("rule.maxResolveDuration", 0, "Limits the maxiMum duration for automatic alert expiration, "+
 		"which by default is 4 times evaluationInterval of the parent group")
-	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the `time` parameter for rule evaluation requests to compensate intentional data delay from the datasource."+
-		"Normally, should be equal to `-search.latencyOffset` (cmd-line flag configured for VictoriaMetrics single-node or vmselect).")
+	evalDelay = flag.Duration("rule.evalDelay", 30*time.Second, "Adjustment of the `time` parameter for rule evaluation requests to compensate intentional data delay from the datasource. "+
+		"Normally, should be equal to `-search.latencyOffset` (cmd-line flag configured for VictoriaMetrics single-node or vmselect). "+
+		"This doesn't apply to groups with eval_offset specified.")
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
-	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
-		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
+	remoteReadLookBack     = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries. "+
+		"For example, if lookback=1h then range from now() to now()-1h will be scanned.")
 )
 
 // Group is an entity for grouping rules
@@ -74,19 +76,23 @@ type Group struct {
 }
 
 type groupMetrics struct {
-	iterationTotal    *utils.Counter
-	iterationDuration *utils.Summary
-	iterationMissed   *utils.Counter
-	iterationInterval *utils.Gauge
+	set *metrics.Set
+
+	iterationTotal    *metrics.Counter
+	iterationDuration *metrics.Summary
+	iterationMissed   *metrics.Counter
+	iterationInterval *metrics.Gauge
 }
 
 func newGroupMetrics(g *Group) *groupMetrics {
 	m := &groupMetrics{}
+	m.set = metrics.NewSet()
+
 	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
-	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
-	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
-	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
-	m.iterationInterval = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+	m.iterationTotal = m.set.NewCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = m.set.NewSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = m.set.NewCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = m.set.NewGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
 		g.mu.RLock()
 		i := g.Interval.Seconds()
 		g.mu.RUnlock()
@@ -95,6 +101,14 @@ func newGroupMetrics(g *Group) *groupMetrics {
 	return m
 }
 
+func (m *groupMetrics) start() {
+	metrics.RegisterSet(m.set)
+}
+
+func (m *groupMetrics) close() {
+	metrics.UnregisterSet(m.set, true)
+}
+
 // merges group rule labels into result map
 // set2 has priority over set1.
 func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
@@ -237,7 +251,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 		if !ok {
 			// old rule is not present in the new list
 			// so we mark it for removing
-			g.Rules[i].close()
+			g.Rules[i].unregisterMetrics()
 			g.Rules[i] = nil
 			continue
 		}
@@ -257,6 +271,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	}
 	// add the rest of rules from registry
 	for _, nr := range rulesRegistry {
+		nr.registerMetrics(g)
 		newRules = append(newRules, nr)
 	}
 	// note that g.Interval is not updated here
@@ -295,13 +310,7 @@ func (g *Group) Close() {
 	g.InterruptEval()
 	<-g.finishedCh
 
-	g.metrics.iterationDuration.Unregister()
-	g.metrics.iterationTotal.Unregister()
-	g.metrics.iterationMissed.Unregister()
-	g.metrics.iterationInterval.Unregister()
-	for _, rule := range g.Rules {
-		rule.close()
-	}
+	g.metrics.close()
 }
 
 // SkipRandSleepOnGroupStart will skip random sleep delay in group first evaluation
@@ -311,6 +320,8 @@ var SkipRandSleepOnGroupStart bool
 func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
+	g.metrics.start()
+
 	evalTS := time.Now()
 	// sleep random duration to spread group rules evaluation
 	// over time in order to reduce load on datasource.
@@ -365,6 +376,7 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 		}
 
 		resolveDuration := getResolveDuration(g.Interval, *resendDelay, *maxResolveDuration)
+		// adjust request timestamp using evalDelay and evalAlignment if necessary
 		ts = g.adjustReqTimestamp(ts)
 		errs := e.execConcurrently(ctx, g.Rules, ts, g.Concurrency, resolveDuration, g.Limit)
 		for err := range errs {
@@ -443,8 +455,8 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 }
 
 // UpdateWith inserts new group to updateCh
-func (g *Group) UpdateWith(new *Group) {
-	g.updateCh <- new
+func (g *Group) UpdateWith(newGroup *Group) {
+	g.updateCh <- newGroup
 }
 
 // DeepCopy returns a deep copy of group
@@ -458,10 +470,18 @@ func (g *Group) DeepCopy() *Group {
 	return &newG
 }
 
-// delayBeforeStart returns a duration on the interval between [ts..ts+interval].
-// delayBeforeStart accounts for `offset`, so returned duration should be always
-// bigger than the `offset`.
+// if offset is specified, delayBeforeStart returns a duration to help aligning timestamp with offset;
+// otherwise, it returns a random duration between [0..interval] based on group key.
 func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *time.Duration) time.Duration {
+	if offset != nil {
+		currentOffsetPoint := ts.Truncate(interval).Add(*offset)
+		if currentOffsetPoint.Before(ts) {
+			// wait until the next offset point
+			return currentOffsetPoint.Add(interval).Sub(ts)
+		}
+		return currentOffsetPoint.Sub(ts)
+	}
+
 	var randSleep time.Duration
 	randSleep = time.Duration(float64(interval) * (float64(key) / (1 << 64)))
 	sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
@@ -469,15 +489,6 @@ func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *
 		randSleep += interval
 	}
 	randSleep -= sleepOffset
-	// check if `ts` after randSleep is before `offset`,
-	// if it is, add extra eval_offset to randSleep.
-	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3409.
-	if offset != nil {
-		tmpEvalTS := ts.Add(randSleep)
-		if tmpEvalTS.Before(tmpEvalTS.Truncate(interval).Add(*offset)) {
-			randSleep += *offset
-		}
-	}
 	return randSleep
 }
 
@@ -583,26 +594,14 @@ func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Du
 }
 
 func (g *Group) adjustReqTimestamp(timestamp time.Time) time.Time {
+	// if `eval_offset` is specified, timestamp is already aligned with offset, do nothing
 	if g.EvalOffset != nil {
-		// calculate the min timestamp on the evaluationInterval
-		intervalStart := timestamp.Truncate(g.Interval)
-		ts := intervalStart.Add(*g.EvalOffset)
-		if timestamp.Before(ts) {
-			// if passed timestamp is before the expected evaluation offset,
-			// then we should adjust it to the previous evaluation round.
-			// E.g. request with evaluationInterval=1h and evaluationOffset=30m
-			// was evaluated at 11:20. Then the timestamp should be adjusted
-			// to 10:30, to the previous evaluationInterval.
-			return ts.Add(-g.Interval)
-		}
-		// when `eval_offset` is using, ts shouldn't be effect by `eval_alignment` and `eval_delay`
-		// since it should be always aligned.
-		return ts
+		return timestamp
 	}
 
 	timestamp = timestamp.Add(-g.getEvalDelay())
 
-	// always apply the alignment as a last step
+	// apply the alignment as the last step
 	if g.evalAlignment == nil || *g.evalAlignment {
 		// align query time with interval to get similar result with grafana when plotting time series.
 		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5049

app/vmalert/rule/group_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -27,7 +28,7 @@ func init() {
 }
 
 func TestMain(m *testing.M) {
-	if err := templates.Init([]string{}, nil, url.URL{}); err != nil {
+	if err := templates.Load([]string{}, url.URL{}); err != nil {
 		fmt.Println("failed to load template for test")
 		os.Exit(1)
 	}
@@ -41,6 +42,7 @@ func TestUpdateWith(t *testing.T) {
 		g := &Group{
 			Name: "test",
 		}
+		g.metrics = newGroupMetrics(g)
 		qb := &datasource.FakeQuerier{}
 		for _, r := range currentRules {
 			r.ID = config.HashRule(r)
@@ -50,6 +52,7 @@ func TestUpdateWith(t *testing.T) {
 		ng := &Group{
 			Name: "test",
 		}
+		ng.metrics = newGroupMetrics(ng)
 		for _, r := range newRules {
 			r.ID = config.HashRule(r)
 			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
@@ -195,6 +198,7 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 		Interval: 100 * time.Hour,
 		updateCh: make(chan *Group),
 	}
+	g.metrics = newGroupMetrics(g)
 	go g.Start(context.Background(), nil, nil, nil)
 
 	rule1 := AlertingRule{
@@ -209,6 +213,7 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 			&rule1,
 		},
 	}
+	g1.metrics = newGroupMetrics(g1)
 	g.updateCh <- g1
 	time.Sleep(10 * time.Millisecond)
 	g.mu.RLock()
@@ -218,8 +223,9 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 	g.mu.RUnlock()
 
 	rule2 := AlertingRule{
-		Name: "jobDown",
-		Expr: "up{job=\"vmagent\"}==0",
+		RuleID: 1,
+		Name:   "jobDown",
+		Expr:   "up{job=\"vmagent\"}==0",
 		Labels: map[string]string{
 			"foo": "bar",
 			"baz": "qux",
@@ -227,17 +233,32 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 	}
 	g2 := &Group{
 		Rules: []Rule{
+			&rule1,
 			&rule2,
 		},
 	}
+	g2.metrics = newGroupMetrics(g2)
 	g.updateCh <- g2
 	time.Sleep(10 * time.Millisecond)
 	g.mu.RLock()
-	if len(g.Rules[0].(*AlertingRule).Labels) != 2 {
+	if len(g.Rules) != 2 {
+		t.Fatalf("expected to have updated rules")
+	}
+
+	if len(g.Rules[1].(*AlertingRule).Labels) != 2 {
 		t.Fatalf("expected to have updated labels")
 	}
 	g.mu.RUnlock()
 
+	metricsAfter := metrics.GetDefaultSet().ListMetricNames()
+	metricsRegistry := make(map[string]struct{}, len(metricsAfter))
+	for _, m := range metricsAfter {
+		if _, ok := metricsRegistry[m]; ok {
+			t.Fatalf("duplicate metric name %q", m)
+		}
+		metricsRegistry[m] = struct{}{}
+	}
+
 	g.Close()
 }
 
@@ -512,34 +533,14 @@ func TestGroupStartDelay(t *testing.T) {
 	f("2023-01-01T00:00:29.000+00:00", "2023-01-01T00:00:30.000+00:00")
 	f("2023-01-01T00:00:31.000+00:00", "2023-01-01T00:05:30.000+00:00")
 
-	// test group with offset smaller than above fixed randSleep,
-	// this way randSleep will always be enough
-	offset := 20 * time.Second
+	// test group with offset
+	offset := 3 * time.Minute
 	g.EvalOffset = &offset
 
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:00:30.000+00:00")
-	f("2023-01-01T00:00:29.000+00:00", "2023-01-01T00:00:30.000+00:00")
-	f("2023-01-01T00:00:31.000+00:00", "2023-01-01T00:05:30.000+00:00")
-
-	// test group with offset bigger than above fixed randSleep,
-	// this way offset will be added to delay
-	offset = 3 * time.Minute
-	g.EvalOffset = &offset
-
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:03:30.000+00:00")
-	f("2023-01-01T00:00:29.000+00:00", "2023-01-01T00:03:30.000+00:00")
-	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:08:30.000+00:00")
-	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:30.000+00:00")
-	f("2023-01-01T00:07:30.000+00:00", "2023-01-01T00:13:30.000+00:00")
-
-	offset = 10 * time.Minute
-	g.EvalOffset = &offset
-	// interval of 1h and key generate a static delay of 6m
-	g.Interval = time.Hour
-
-	f("2023-01-01T00:00:00.000+00:00", "2023-01-01T00:16:00.000+00:00")
-	f("2023-01-01T00:05:00.000+00:00", "2023-01-01T00:16:00.000+00:00")
-	f("2023-01-01T00:30:00.000+00:00", "2023-01-01T01:16:00.000+00:00")
+	f("2023-01-01T00:00:15.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:01:00.000+00:00", "2023-01-01T00:03:00.000+00:00")
+	f("2023-01-01T00:03:30.000+00:00", "2023-01-01T00:08:00.000+00:00")
+	f("2023-01-01T00:08:00.000+00:00", "2023-01-01T00:08:00.000+00:00")
 }
 
 func TestGetPrometheusReqTimestamp(t *testing.T) {
@@ -569,17 +570,11 @@ func TestGetPrometheusReqTimestamp(t *testing.T) {
 		evalAlignment: &disableAlign,
 	}, "2023-08-28T11:11:00+00:00", "2023-08-28T11:10:30+00:00")
 
-	// with eval_offset, find previous offset point + default evalDelay
+	// with eval_offset
 	f(&Group{
 		EvalOffset: &offset,
 		Interval:   time.Hour,
-	}, "2023-08-28T11:11:00+00:00", "2023-08-28T10:30:00+00:00")
-
-	// with eval_offset + default evalDelay
-	f(&Group{
-		EvalOffset: &offset,
-		Interval:   time.Hour,
-	}, "2023-08-28T11:41:00+00:00", "2023-08-28T11:30:00+00:00")
+	}, "2023-08-28T11:30:00+00:00", "2023-08-28T11:30:00+00:00")
 
 	// 1h interval with eval_delay
 	f(&Group{

app/vmalert/rule/recording.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
@@ -45,6 +47,28 @@ type recordingRuleMetrics struct {
 	samples *utils.Gauge
 }
 
+func newRecordingRuleMetrics(set *metrics.Set, rr *RecordingRule) *recordingRuleMetrics {
+	rmr := &recordingRuleMetrics{}
+
+	labels := fmt.Sprintf(`recording=%q, group=%q, file=%q, id="%d"`, rr.Name, rr.GroupName, rr.File, rr.ID())
+	rmr.errors = utils.NewCounter(set, fmt.Sprintf(`vmalert_recording_rules_errors_total{%s}`, labels))
+	rmr.samples = utils.NewGauge(set, fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			e := rr.state.getLast()
+			return float64(e.Samples)
+		})
+
+	return rmr
+}
+
+func (m *recordingRuleMetrics) close() {
+	if m == nil {
+		return
+	}
+	m.errors.Unregister()
+	m.samples.Unregister()
+}
+
 // String implements Stringer interface
 func (rr *RecordingRule) String() string {
 	return rr.Name
@@ -67,7 +91,6 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID:   group.ID(),
 		GroupName: group.Name,
 		File:      group.File,
-		metrics:   &recordingRuleMetrics{},
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:            group.Type.String(),
 			ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr),
@@ -87,21 +110,18 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 	rr.state = &ruleState{
 		entries: make([]StateEntry, entrySize),
 	}
+	rr.metrics = newRecordingRuleMetrics(group.metrics.set, rr)
 
-	labels := fmt.Sprintf(`recording=%q, group=%q, file=%q, id="%d"`, rr.Name, group.Name, group.File, rr.ID())
-	rr.metrics.errors = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_recording_rules_errors_total{%s}`, labels))
-	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
-		func() float64 {
-			e := rr.state.getLast()
-			return float64(e.Samples)
-		})
 	return rr
 }
 
+func (rr *RecordingRule) registerMetrics(g *Group) {
+	rr.metrics = newRecordingRuleMetrics(g.metrics.set, rr)
+}
+
 // close unregisters rule metrics
-func (rr *RecordingRule) close() {
-	rr.metrics.errors.Unregister()
-	rr.metrics.samples.Unregister()
+func (rr *RecordingRule) unregisterMetrics() {
+	rr.metrics.close()
 }
 
 // execRange executes recording rule on the given time range similarly to Exec.

app/vmalert/rule/recording_test.go

@@ -7,8 +7,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
@@ -306,15 +307,12 @@ func TestRecordingRuleLimit_Failure(t *testing.T) {
 
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-
 		rule := &RecordingRule{Name: "job:foo",
 			state: &ruleState{entries: make([]StateEntry, 10)},
 			Labels: map[string]string{
 				"source": "test_limit",
 			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
+			metrics: getTestRecordingRuleMetrics(),
 		}
 		rule.q = fq
 
@@ -344,15 +342,12 @@ func TestRecordingRuleLimit_Success(t *testing.T) {
 
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-
 		rule := &RecordingRule{Name: "job:foo",
 			state: &ruleState{entries: make([]StateEntry, 10)},
 			Labels: map[string]string{
 				"source": "test_limit",
 			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
+			metrics: getTestRecordingRuleMetrics(),
 		}
 		rule.q = fq
 
@@ -366,16 +361,19 @@ func TestRecordingRuleLimit_Success(t *testing.T) {
 	f(-1)
 }
 
+func getTestRecordingRuleMetrics() *recordingRuleMetrics {
+	m := newRecordingRuleMetrics(metrics.NewSet(), &RecordingRule{})
+	return m
+}
+
 func TestRecordingRuleExec_Negative(t *testing.T) {
 	rr := &RecordingRule{
 		Name: "job:foo",
 		Labels: map[string]string{
 			"job": "test",
 		},
-		state: &ruleState{entries: make([]StateEntry, 10)},
-		metrics: &recordingRuleMetrics{
-			errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-		},
+		state:   &ruleState{entries: make([]StateEntry, 10)},
+		metrics: getTestRecordingRuleMetrics(),
 	}
 	fq := &datasource.FakeQuerier{}
 	expErr := "connection reset by peer"

app/vmalert/rule/rule.go

@@ -27,9 +27,10 @@ type Rule interface {
 	// updateWith performs modification of current Rule
 	// with fields of the given Rule.
 	updateWith(Rule) error
-	// close performs the shutdown procedures for rule
-	// such as metrics unregister
-	close()
+	// unregister Rule metrics
+	unregisterMetrics()
+	// register Rule metrics with the given group
+	registerMetrics(g *Group)
 }
 
 var errDuplicate = errors.New("result contains metrics with the same labelset during evaluation. See https://docs.victoriametrics.com/vmalert/#series-with-the-same-labelset for details")

app/vmalert/templates/init.go

@@ -1,553 +0,0 @@
-package templates
-
-import (
-	"errors"
-	"fmt"
-	htmlTpl "html/template"
-	"io"
-	"math"
-	"net"
-	"net/url"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	textTpl "text/template"
-	"time"
-
-	"github.com/bmatcuk/doublestar/v4"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
-)
-
-var (
-	tplMu      sync.RWMutex
-	masterTmpl *textTpl.Template
-
-	// externalLabels is a global variable for holding external labels configured via flags
-	// It is supposed to be initiated via Init function only.
-	externalLabels map[string]string
-	// externalURL is a global variable for holding external URL value configured via flag
-	// It is supposed to be initiated via Init function only.
-	externalURL url.URL
-)
-
-// Init initializes global externalLabels and externalURL variables, and load templates from pathPatterns.
-func Init(pathPatterns []string, extLabels map[string]string, extURL url.URL) error {
-	externalURL = extURL
-	externalLabels = extLabels
-	return LoadTemplateFile(pathPatterns)
-}
-
-// LoadTemplateFile loads templates from multiple globs specified in pathPatterns:
-// 1. if it's the first load, sets them directly to current template;
-// 2. if it's not the first load, only update masterTmpl when the contents change.
-func LoadTemplateFile(pathPatterns []string) error {
-	templateName := fmt.Sprintf("rule-template-%d", time.Now().UnixMilli())
-	// using Load timestamp as template root name,
-	// so we can check if this global template has been reloaded when use it elsewhere, like during alerting rules execution.
-	tmpl := newTemplate(templateName)
-	tmpl = tmpl.Funcs(funcsWithExternalURL(externalURL))
-
-	for _, tp := range pathPatterns {
-		p, err := doublestar.FilepathGlob(tp)
-		if err != nil {
-			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
-		}
-		if len(p) > 0 {
-			tmpl, err = tmpl.ParseFiles(p...)
-			if err != nil {
-				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
-			}
-		}
-	}
-	err := tmpl.Execute(io.Discard, nil)
-	if err != nil {
-		return fmt.Errorf("failed to test rule template: %w", err)
-	}
-
-	tplMu.Lock()
-	defer tplMu.Unlock()
-
-	if masterTmpl == nil {
-		masterTmpl = tmpl
-	} else {
-		// only update the masterTmpl when content has changed
-		if !isTemplatesTheSame(masterTmpl, tmpl) {
-			masterTmpl = tmpl
-		}
-	}
-	return nil
-}
-
-func newTemplate(name string) *textTpl.Template {
-	tmpl := textTpl.New(name).Funcs(templateFuncs())
-	return textTpl.Must(tmpl.Parse(""))
-}
-
-// isTemplatesTheSame returns true if the content of two templates are the same,
-// the root template name difference is ignored.
-func isTemplatesTheSame(t1, t2 *textTpl.Template) bool {
-	getRootString := func(t *textTpl.Template) string {
-		if t.Tree == nil || t.Tree.Root == nil {
-			return ""
-		}
-		return t.Tree.Root.String()
-	}
-	if getRootString(t1) != getRootString(t2) {
-		return false
-	}
-	t1Templates := make(map[string]string)
-	t2Templates := make(map[string]string)
-	for _, tmpl := range t1.Templates() {
-		// skip root template, since it's null and changes every time
-		if tmpl.Name() == t1.Name() {
-			continue
-		}
-		t1Templates[tmpl.Name()] = getRootString(tmpl)
-	}
-	for _, tmpl := range t2.Templates() {
-		if tmpl.Name() == t2.Name() {
-			continue
-		}
-		t2Templates[tmpl.Name()] = getRootString(tmpl)
-	}
-	if len(t1Templates) != len(t2Templates) {
-		return false
-	}
-	for k, v := range t1Templates {
-		if t2Templates[k] != v {
-			return false
-		}
-		delete(t2Templates, k)
-	}
-	return len(t2Templates) == 0
-}
-
-// funcsWithExternalURL returns a function map that depends on externalURL value
-func funcsWithExternalURL(externalURL url.URL) textTpl.FuncMap {
-	return textTpl.FuncMap{
-		"externalURL": func() string {
-			return externalURL.String()
-		},
-
-		"pathPrefix": func() string {
-			return externalURL.Path
-		},
-	}
-}
-
-// templateFuncs initiates template helper functions
-func templateFuncs() textTpl.FuncMap {
-	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
-	// and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150
-	return textTpl.FuncMap{
-		/* Strings */
-
-		// title returns a copy of the string s with all Unicode letters
-		// that begin words mapped to their Unicode title case.
-		// alias for https://golang.org/pkg/strings/#Title
-		"title": strings.Title,
-
-		// toUpper returns s with all Unicode letters mapped to their upper case.
-		// alias for https://golang.org/pkg/strings/#ToUpper
-		"toUpper": strings.ToUpper,
-
-		// toLower returns s with all Unicode letters mapped to their lower case.
-		// alias for https://golang.org/pkg/strings/#ToLower
-		"toLower": strings.ToLower,
-
-		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
-		// This function is deprecated.
-		//
-		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
-		// these functions properly escape `\n` and `\r` chars according to their purpose.
-		"crlfEscape": func(q string) string {
-			q = strings.Replace(q, "\n", `\n`, -1)
-			return strings.Replace(q, "\r", `\r`, -1)
-		},
-
-		// quotesEscape escapes the string, so it can be safely put inside JSON string.
-		//
-		// See also jsonEscape.
-		"quotesEscape": quotesEscape,
-
-		// jsonEscape converts the string to properly encoded JSON string.
-		//
-		// See also quotesEscape.
-		"jsonEscape": jsonEscape,
-
-		// htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html.
-		//
-		// See also safeHtml.
-		"htmlEscape": htmlEscape,
-
-		// stripPort splits string into host and port, then returns only host.
-		"stripPort": func(hostPort string) string {
-			host, _, err := net.SplitHostPort(hostPort)
-			if err != nil {
-				return hostPort
-			}
-			return host
-		},
-
-		// stripDomain removes the domain part of a FQDN. Leaves port untouched.
-		"stripDomain": func(hostPort string) string {
-			host, port, err := net.SplitHostPort(hostPort)
-			if err != nil {
-				host = hostPort
-			}
-			ip := net.ParseIP(host)
-			if ip != nil {
-				return hostPort
-			}
-			host = strings.Split(host, ".")[0]
-			if port != "" {
-				return net.JoinHostPort(host, port)
-			}
-			return host
-		},
-
-		// match reports whether the string s
-		// contains any match of the regular expression pattern.
-		// alias for https://golang.org/pkg/regexp/#MatchString
-		"match": regexp.MatchString,
-
-		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
-		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
-		// so for instance $1 represents the text of the first submatch.
-		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
-		"reReplaceAll": func(pattern, repl, text string) string {
-			re := regexp.MustCompile(pattern)
-			return re.ReplaceAllString(text, repl)
-		},
-
-		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
-		"parseDuration": func(s string) (float64, error) {
-			d, err := promutils.ParseDuration(s)
-			if err != nil {
-				return 0, err
-			}
-			return d.Seconds(), nil
-		},
-
-		// same with parseDuration but returns a time.Duration
-		"parseDurationTime": func(s string) (time.Duration, error) {
-			d, err := promutils.ParseDuration(s)
-			if err != nil {
-				return 0, err
-			}
-			return d, nil
-		},
-
-		/* Numbers */
-
-		// humanize converts given number to a human readable format
-		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
-		"humanize": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			if math.Abs(v) >= 1 {
-				prefix := ""
-				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
-					if math.Abs(v) < 1000 {
-						break
-					}
-					prefix = p
-					v /= 1000
-				}
-				return fmt.Sprintf("%.4g%s", v, prefix), nil
-			}
-			prefix := ""
-			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
-				if math.Abs(v) >= 1 {
-					break
-				}
-				prefix = p
-				v *= 1000
-			}
-			return fmt.Sprintf("%.4g%s", v, prefix), nil
-		},
-
-		// humanize1024 converts given number to a human readable format with 1024 as base
-		"humanize1024": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			return formatutil.HumanizeBytes(v), nil
-		},
-
-		// humanizeDuration converts given seconds to a human-readable duration
-		"humanizeDuration": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			if v == 0 {
-				return fmt.Sprintf("%.4gs", v), nil
-			}
-			if math.Abs(v) >= 1 {
-				sign := ""
-				if v < 0 {
-					sign = "-"
-					v = -v
-				}
-				seconds := int64(v) % 60
-				minutes := (int64(v) / 60) % 60
-				hours := (int64(v) / 60 / 60) % 24
-				days := int64(v) / 60 / 60 / 24
-				// For days to minutes, we display seconds as an integer.
-				if days != 0 {
-					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
-				}
-				if hours != 0 {
-					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
-				}
-				if minutes != 0 {
-					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
-				}
-				// For seconds, we display 4 significant digits.
-				return fmt.Sprintf("%s%.4gs", sign, v), nil
-			}
-			prefix := ""
-			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
-				if math.Abs(v) >= 1 {
-					break
-				}
-				prefix = p
-				v *= 1000
-			}
-			return fmt.Sprintf("%.4g%ss", v, prefix), nil
-		},
-
-		// humanizePercentage converts given ratio value to a fraction of 100
-		"humanizePercentage": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			return fmt.Sprintf("%.4g%%", v*100), nil
-		},
-
-		// humanizeTimestamp converts given timestamp to a human readable time equivalent
-		"humanizeTimestamp": func(i any) (string, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return "", err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return fmt.Sprintf("%.4g", v), nil
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return fmt.Sprint(t), nil
-		},
-
-		// toTime converts given timestamp to a time.Time.
-		"toTime": func(i any) (time.Time, error) {
-			v, err := toFloat64(i)
-			if err != nil {
-				return time.Time{}, err
-			}
-			if math.IsNaN(v) || math.IsInf(v, 0) {
-				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
-			}
-			t := timeFromUnixTimestamp(v).Time().UTC()
-			return t, nil
-		},
-
-		/* URLs */
-
-		// externalURL returns value of `external.url` flag
-		"externalURL": func() string {
-			// externalURL function supposed to be substituted at FuncsWithExteralURL().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			return ""
-		},
-
-		// pathPrefix returns a Path segment from the URL value in `external.url` flag
-		"pathPrefix": func() string {
-			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
-			// it is present here only for validation purposes, when there is no
-			// provided datasource.
-			//
-			// return non-empty slice to pass validation with chained functions in template
-			return ""
-		},
-
-		// pathEscape escapes the string so it can be safely placed inside a URL path segment.
-		//
-		// See also queryEscape.
-		"pathEscape": url.PathEscape,
-
-		// queryEscape escapes the string so it can be safely placed inside a query arg in URL.
-		//
-		// See also queryEscape.
-		"queryEscape": url.QueryEscape,
-
-		// first returns the first by order element from the given metrics list.
-		// usually used alongside with `query` template function.
-		"first": func(metrics []metric) (metric, error) {
-			if len(metrics) > 0 {
-				return metrics[0], nil
-			}
-			return metric{}, errors.New("first() called on vector with no elements")
-		},
-
-		// label returns the value of the given label name for the given metric.
-		// usually used alongside with `query` template function.
-		"label": func(label string, m metric) string {
-			return m.Labels[label]
-		},
-
-		// value returns the value of the given metric.
-		// usually used alongside with `query` template function.
-		"value": func(m metric) float64 {
-			return m.Value
-		},
-
-		// strvalue returns metric name.
-		"strvalue": func(m metric) string {
-			return m.Labels["__name__"]
-		},
-
-		// sortByLabel sorts the given metrics by provided label key
-		"sortByLabel": func(label string, metrics []metric) []metric {
-			sort.SliceStable(metrics, func(i, j int) bool {
-				return metrics[i].Labels[label] < metrics[j].Labels[label]
-			})
-			return metrics
-		},
-
-		/* Helpers */
-
-		// Converts a list of objects to a map with keys arg0, arg1 etc.
-		// This is intended to allow multiple arguments to be passed to templates.
-		"args": func(args ...any) map[string]any {
-			result := make(map[string]any)
-			for i, a := range args {
-				result[fmt.Sprintf("arg%d", i)] = a
-			}
-			return result
-		},
-
-		// safeHtml marks string as HTML not requiring auto-escaping.
-		//
-		// See also htmlEscape.
-		"safeHtml": func(text string) htmlTpl.HTML {
-			return htmlTpl.HTML(text)
-		},
-	}
-}
-
-// metric is private copy of datasource.Metric,
-// it is used for templating annotations,
-// Labels as map simplifies templates evaluation.
-type metric struct {
-	Labels    map[string]string
-	Timestamp int64
-	Value     float64
-}
-
-// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
-func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
-	mss := make([]metric, 0, len(ms))
-	for _, m := range ms {
-		labelsMap := make(map[string]string, len(m.Labels))
-		for _, labelValue := range m.Labels {
-			labelsMap[labelValue.Name] = labelValue.Value
-		}
-		mss = append(mss, metric{
-			Labels:    labelsMap,
-			Timestamp: m.Timestamps[0],
-			Value:     m.Values[0]})
-	}
-	return mss
-}
-
-// QueryFn is used to wrap a call to datasource into simple-to-use function
-// for templating functions.
-type QueryFn func(query string) ([]datasource.Metric, error)
-
-// FuncsWithQuery returns a function map that depends on metric data
-func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
-	return textTpl.FuncMap{
-		"query": func(q string) ([]metric, error) {
-			if query == nil {
-				return nil, fmt.Errorf("cannot execute query %q: query is not available in this context", q)
-			}
-
-			result, err := query(q)
-			if err != nil {
-				return nil, err
-			}
-			return datasourceMetricsToTemplateMetrics(result), nil
-		},
-	}
-}
-
-// Time is the number of milliseconds since the epoch
-// (1970-01-01 00:00 UTC) excluding leap seconds.
-type Time int64
-
-// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
-func timeFromUnixTimestamp(t float64) Time {
-	return Time(t * 1e3)
-}
-
-// The number of nanoseconds per minimum tick.
-const nanosPerTick = int64(minimumTick / time.Nanosecond)
-
-// MinimumTick is the minimum supported time resolution. This has to be
-// at least time.Second in order for the code below to work.
-const minimumTick = time.Millisecond
-
-// second is the Time duration equivalent to one second.
-const second = int64(time.Second / minimumTick)
-
-// Time returns the time.Time representation of t.
-func (t Time) Time() time.Time {
-	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
-}
-
-func toFloat64(v any) (float64, error) {
-	switch i := v.(type) {
-	case float64:
-		return i, nil
-	case float32:
-		return float64(i), nil
-	case int64:
-		return float64(i), nil
-	case int32:
-		return float64(i), nil
-	case int:
-		return float64(i), nil
-	case uint64:
-		return float64(i), nil
-	case uint32:
-		return float64(i), nil
-	case uint:
-		return float64(i), nil
-	case string:
-		return strconv.ParseFloat(i, 64)
-	default:
-		return 0, fmt.Errorf("unexpected value type %v", i)
-	}
-}

app/vmalert/templates/init_test.go

@@ -1,261 +0,0 @@
-package templates
-
-import (
-	"fmt"
-	"math"
-	"net/url"
-	"os"
-	"strings"
-	"testing"
-	textTpl "text/template"
-)
-
-func TestMain(m *testing.M) {
-	if err := Init([]string{}, nil, url.URL{}); err != nil {
-		fmt.Println("failed to load template for test")
-		os.Exit(1)
-	}
-	os.Exit(m.Run())
-}
-
-func TestTemplateFuncs_Match(t *testing.T) {
-	funcs := templateFuncs()
-	// check "match" func
-	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
-	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
-		t.Fatalf("expecting non-nil error on invalid regexp")
-	}
-	ok, err := matchFunc("abc", "def")
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	if ok {
-		t.Fatalf("unexpected match")
-	}
-	ok, err = matchFunc("a.+b", "acsdb")
-	if err != nil {
-		t.Fatalf("unexpected error")
-	}
-	if !ok {
-		t.Fatalf("unexpected mismatch")
-	}
-}
-
-func TestTemplateFuncs_Formatting(t *testing.T) {
-	f := func(funcName string, p any, resultExpected string) {
-		t.Helper()
-
-		funcs := templateFuncs()
-		v := funcs[funcName]
-		fLocal := v.(func(s any) (string, error))
-		result, err := fLocal(p)
-		if err != nil {
-			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
-		}
-		if result != resultExpected {
-			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
-		}
-	}
-
-	f("humanize1024", float64(0), "0")
-	f("humanize1024", math.Inf(0), "+Inf")
-	f("humanize1024", math.NaN(), "NaN")
-	f("humanize1024", float64(127087), "124.1ki")
-	f("humanize1024", float64(130137088), "124.1Mi")
-	f("humanize1024", float64(133260378112), "124.1Gi")
-	f("humanize1024", float64(136458627186688), "124.1Ti")
-	f("humanize1024", float64(139733634239168512), "124.1Pi")
-	f("humanize1024", float64(143087241460908556288), "124.1Ei")
-	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
-	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
-	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
-
-	f("humanize", float64(127087), "127.1k")
-	f("humanize", float64(136458627186688), "136.5T")
-
-	f("humanizeDuration", 1, "1s")
-	f("humanizeDuration", 0.2, "200ms")
-	f("humanizeDuration", 42000, "11h 40m 0s")
-	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
-
-	f("humanizePercentage", 1, "100%")
-	f("humanizePercentage", 0.8, "80%")
-	f("humanizePercentage", 0.015, "1.5%")
-
-	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
-}
-
-func TestTemplateFuncs_StringConversion(t *testing.T) {
-	f := func(funcName, s, resultExpected string) {
-		t.Helper()
-
-		funcs := templateFuncs()
-		v := funcs[funcName]
-		fLocal := v.(func(s string) string)
-		result := fLocal(s)
-		if result != resultExpected {
-			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
-		}
-	}
-
-	f("title", "foo bar", "Foo Bar")
-	f("toUpper", "foo", "FOO")
-	f("toLower", "FOO", "foo")
-	f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz")
-	f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz")
-	f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`)
-	f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`)
-	f("htmlEscape", "foo < 10\nabc", "foo &lt; 10\nabc")
-	f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`)
-	f("stripPort", "foo", "foo")
-	f("stripPort", "foo:1234", "foo")
-	f("stripDomain", "foo.bar.baz", "foo")
-	f("stripDomain", "foo.bar:123", "foo:123")
-}
-
-func TestTemplatesLoad_Success(t *testing.T) {
-	f := func(pathPatterns []string, expectedTmpl *textTpl.Template) {
-		t.Helper()
-
-		masterTmplOrig := masterTmpl
-		defer func() {
-			masterTmpl = masterTmplOrig
-		}()
-		masterTmpl = nil
-
-		if err := LoadTemplateFile(pathPatterns); err != nil {
-			t.Fatalf("cannot load templates: %s", err)
-		}
-		if !isTemplatesTheSame(masterTmpl, expectedTmpl) {
-			t.Fatalf("unexpected template\ngot\n%+v\nwant\n%+v", masterTmpl, expectedTmpl)
-		}
-	}
-
-	// non existing path
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	expectedTmpl := textTpl.Must(newTemplate("").Parse(""))
-	f(pathPatterns, expectedTmpl)
-
-	// existing path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	expectedTmpl = textTpl.Must(newTemplate("").Parse(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.2" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`))
-	f(pathPatterns, expectedTmpl)
-
-	// template update
-	pathPatterns = []string{
-		"templates/other/nested/good0-*.tpl",
-	}
-	expectedTmpl = textTpl.Must(newTemplate("").Parse(`
-		{{- define "good0-test.tpl" -}}{{- end -}}
-		{{- define "test.0" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.1" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-		{{- define "test.3" -}}
-			{{ printf "Hello %s!" externalURL }}
-		{{- end -}}
-	`))
-	f(pathPatterns, expectedTmpl)
-}
-
-func TestTemplatesLoad_Failure(t *testing.T) {
-	f := func(pathPatterns []string, expectedErrStr string) {
-		t.Helper()
-
-		err := LoadTemplateFile(pathPatterns)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-
-		errStr := err.Error()
-		if !strings.Contains(errStr, expectedErrStr) {
-			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
-		}
-	}
-
-	// load template with syntax error
-	f([]string{
-		"templates/other/nested/bad0-*.tpl",
-		"templates/test/good0-*.tpl",
-	}, "failed to parse template glob")
-}
-
-func TestTemplatesReload(t *testing.T) {
-	masterTmplOrig := masterTmpl
-	defer func() {
-		masterTmpl = masterTmplOrig
-	}()
-	masterTmpl = nil
-
-	// load with non existing path
-	pathPatterns := []string{
-		"templates/non-existing/good-*.tpl",
-		"templates/absent/good-*.tpl",
-	}
-	if err := LoadTemplateFile(pathPatterns); err != nil {
-		t.Fatalf("cannot load templates: %s", err)
-	}
-	tpl1 := GetCurrentTmpl()
-
-	// reload with existing path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	if err := LoadTemplateFile(pathPatterns); err != nil {
-		t.Fatalf("cannot load templates: %s", err)
-	}
-	tpl2 := GetCurrentTmpl()
-	if isTemplatesTheSame(tpl1, tpl2) {
-		t.Fatalf("tpl1 should be different from tpl2")
-	}
-
-	// reload the same path
-	pathPatterns = []string{
-		"templates/test/good0-*.tpl",
-	}
-	if err := LoadTemplateFile(pathPatterns); err != nil {
-		t.Fatalf("cannot load templates: %s", err)
-	}
-	tpl3 := GetCurrentTmpl()
-	if !isTemplatesTheSame(tpl2, tpl3) || tpl2.Name() != tpl3.Name() {
-		t.Fatalf("tpl3 should be the same as tpl2")
-	}
-}
-
-func TestIsTemplateTheSame(t *testing.T) {
-	f := func(tmpl1, tmpl2 *textTpl.Template, isTheSame bool) {
-		t.Helper()
-
-		if isTemplatesTheSame(tmpl1, tmpl2) != isTheSame {
-			t.Fatalf("unexpected result for isTemplatesTheSame")
-		}
-	}
-
-	tmpl1 := textTpl.Must(newTemplate("t1").Parse("{{- define \"test\" -}}{{- end -}}"))
-	tmpl2 := textTpl.Must(newTemplate("t2").Parse("{{- define \"test\" -}}{{- end -}}"))
-	f(tmpl1, tmpl2, true)
-
-	tmpl1, _ = tmpl1.Parse("{{- define \"test2\" -}}{{- end -}}")
-	f(tmpl1, tmpl2, false)
-
-	tmpl2, _ = tmpl2.Parse("{{- define \"test3\" -}}{{- end -}}")
-	f(tmpl1, tmpl2, false)
-}

app/vmalert/templates/template.go

@@ -1,95 +1,554 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package templates
 
 import (
-	"bytes"
+	"errors"
 	"fmt"
+	htmlTpl "html/template"
+	"io"
+	"math"
+	"net"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	textTpl "text/template"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/bmatcuk/doublestar/v4"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
-// supported variables are list in https://docs.victoriametrics.com/vmalert/#templating.
-const tplHeaders = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}{{ $externalLabels := .ExternalLabels }}{{ $externalURL := .ExternalURL }}{{ $alertID := .AlertID }}{{ $groupID := .GroupID }}{{ $activeAt := .ActiveAt }}{{ $for := .For }}`
+// go template execution fails when it's tree is empty
+const defaultTemplate = `{{- define "default.template" -}}{{- end -}}`
 
-// AlertTplData is used to execute templating
-type AlertTplData struct {
-	Labels   map[string]string
-	Value    float64
-	Expr     string
-	AlertID  uint64
-	GroupID  uint64
-	ActiveAt time.Time
-	For      time.Duration
+var tplMu sync.RWMutex
+
+type textTemplate struct {
+	current     *textTpl.Template
+	replacement *textTpl.Template
 }
 
-// ValidateTemplates validates the given annotations,
-// mock the `query` function during validation.
-func ValidateTemplates(annotations map[string]string) error {
-	// it's ok to reuse one template for multiple text validations.
-	tmpl := GetCurrentTmpl()
-	tmpl = tmpl.Funcs(FuncsWithQuery(nil))
-	for _, v := range annotations {
-		_, err := tmpl.Parse(tplHeaders + v)
+var masterTmpl textTemplate
+
+func newTemplate() *textTpl.Template {
+	tmpl := textTpl.New("").Option("missingkey=zero").Funcs(templateFuncs())
+	return textTpl.Must(tmpl.Parse(defaultTemplate))
+}
+
+// Load func loads templates from multiple globs specified in pathPatterns and either
+// sets them directly to current template if it's the first init;
+// or sets replacement templates and wait for Reload() to replace current template with replacement.
+func Load(pathPatterns []string, externalURL url.URL) error {
+	tmpl := newTemplate()
+	for _, tp := range pathPatterns {
+		p, err := doublestar.FilepathGlob(tp)
 		if err != nil {
-			return fmt.Errorf("failed to parse text %q into template: %w", v, err)
+			return fmt.Errorf("failed to retrieve a template glob %q: %w", tp, err)
 		}
+		if len(p) > 0 {
+			tmpl, err = tmpl.ParseFiles(p...)
+			if err != nil {
+				return fmt.Errorf("failed to parse template glob %q: %w", tp, err)
+			}
+		}
+	}
+	if len(tmpl.Templates()) > 0 {
+		err := tmpl.Execute(io.Discard, nil)
+		if err != nil {
+			return fmt.Errorf("failed to execute template: %w", err)
+		}
+	}
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	tmpl = tmpl.Funcs(funcsWithExternalURL(externalURL))
+
+	if masterTmpl.current == nil {
+		masterTmpl.current = tmpl
+	} else {
+		masterTmpl.replacement = tmpl
 	}
 	return nil
 }
 
-// GetCurrentTmpl returns a copy of the current global template
-func GetCurrentTmpl() *textTpl.Template {
+// Reload func replaces current template with a replacement template
+// which was set by Load with override=false
+func Reload() {
+	tplMu.Lock()
+	defer tplMu.Unlock()
+	if masterTmpl.replacement != nil {
+		masterTmpl.current = masterTmpl.replacement
+		masterTmpl.replacement = nil
+	}
+}
+
+// metric is private copy of datasource.Metric,
+// it is used for templating annotations,
+// Labels as map simplifies templates evaluation.
+type metric struct {
+	Labels    map[string]string
+	Timestamp int64
+	Value     float64
+}
+
+// datasourceMetricsToTemplateMetrics converts Metrics from datasource package to private copy for templating.
+func datasourceMetricsToTemplateMetrics(ms []datasource.Metric) []metric {
+	mss := make([]metric, 0, len(ms))
+	for _, m := range ms {
+		labelsMap := make(map[string]string, len(m.Labels))
+		for _, labelValue := range m.Labels {
+			labelsMap[labelValue.Name] = labelValue.Value
+		}
+		mss = append(mss, metric{
+			Labels:    labelsMap,
+			Timestamp: m.Timestamps[0],
+			Value:     m.Values[0]})
+	}
+	return mss
+}
+
+// QueryFn is used to wrap a call to datasource into simple-to-use function
+// for templating functions.
+type QueryFn func(query string) ([]datasource.Metric, error)
+
+// GetWithFuncs returns a copy of current template with additional FuncMap
+// provided with funcs argument
+func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
 	tplMu.RLock()
 	defer tplMu.RUnlock()
-	tmpl, err := masterTmpl.Clone()
+	tmpl, err := masterTmpl.current.Clone()
 	if err != nil {
-		logger.Panicf("failed to clone current rule template: %w", err)
+		return nil, err
 	}
-	return tmpl
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tmpl = tmpl.Option("missingkey=zero")
+	return tmpl.Funcs(funcs), nil
 }
 
-type tplData struct {
-	AlertTplData
-	ExternalLabels map[string]string
-	ExternalURL    string
+// FuncsWithQuery returns a function map that depends on metric data
+func FuncsWithQuery(query QueryFn) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"query": func(q string) ([]metric, error) {
+			if query == nil {
+				return nil, fmt.Errorf("cannot execute query %q: query is not available in this context", q)
+			}
+
+			result, err := query(q)
+			if err != nil {
+				return nil, err
+			}
+			return datasourceMetricsToTemplateMetrics(result), nil
+		},
+	}
 }
 
-// ParseWithFixedHeader parses the text with the fixed tplHeaders into the given template
-func ParseWithFixedHeader(text string, tpl *textTpl.Template) (*textTpl.Template, error) {
-	return tpl.Parse(tplHeaders + text)
+// funcsWithExternalURL returns a function map that depends on externalURL value
+func funcsWithExternalURL(externalURL url.URL) textTpl.FuncMap {
+	return textTpl.FuncMap{
+		"externalURL": func() string {
+			return externalURL.String()
+		},
+
+		"pathPrefix": func() string {
+			return externalURL.Path
+		},
+	}
 }
 
-// ExecuteWithoutTemplate retrieves the current global templates, parses the text and executes with the given data
-func ExecuteWithoutTemplate(q QueryFn, text string, data AlertTplData) (string, error) {
-	if !strings.Contains(text, "{{") || !strings.Contains(text, "}}") {
-		return text, nil
-	}
+// templateFuncs initiates template helper functions
+func templateFuncs() textTpl.FuncMap {
+	// See https://prometheus.io/docs/prometheus/latest/configuration/template_reference/
+	// and https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/template/template.go#L150
+	return textTpl.FuncMap{
+		/* Strings */
 
-	var err error
-	tmpl := GetCurrentTmpl()
-	tmpl = tmpl.Funcs(FuncsWithQuery(q))
-	tmpl, err = tmpl.Parse(tplHeaders + text)
-	if err != nil {
-		return "", fmt.Errorf("failed to parse text %q into template: %w", text, err)
+		// title returns a copy of the string s with all Unicode letters
+		// that begin words mapped to their Unicode title case.
+		// alias for https://golang.org/pkg/strings/#Title
+		"title": strings.Title,
 
+		// toUpper returns s with all Unicode letters mapped to their upper case.
+		// alias for https://golang.org/pkg/strings/#ToUpper
+		"toUpper": strings.ToUpper,
+
+		// toLower returns s with all Unicode letters mapped to their lower case.
+		// alias for https://golang.org/pkg/strings/#ToLower
+		"toLower": strings.ToLower,
+
+		// crlfEscape replaces '\n' and '\r' chars with `\\n` and `\\r`.
+		// This function is deprecated.
+		//
+		// It is better to use quotesEscape, jsonEscape, queryEscape or pathEscape instead -
+		// these functions properly escape `\n` and `\r` chars according to their purpose.
+		"crlfEscape": func(q string) string {
+			q = strings.Replace(q, "\n", `\n`, -1)
+			return strings.Replace(q, "\r", `\r`, -1)
+		},
+
+		// quotesEscape escapes the string, so it can be safely put inside JSON string.
+		//
+		// See also jsonEscape.
+		"quotesEscape": quotesEscape,
+
+		// jsonEscape converts the string to properly encoded JSON string.
+		//
+		// See also quotesEscape.
+		"jsonEscape": jsonEscape,
+
+		// htmlEscape applies html-escaping to q, so it can be safely embedded as plaintext into html.
+		//
+		// See also safeHtml.
+		"htmlEscape": htmlEscape,
+
+		// stripPort splits string into host and port, then returns only host.
+		"stripPort": func(hostPort string) string {
+			host, _, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				return hostPort
+			}
+			return host
+		},
+
+		// stripDomain removes the domain part of a FQDN. Leaves port untouched.
+		"stripDomain": func(hostPort string) string {
+			host, port, err := net.SplitHostPort(hostPort)
+			if err != nil {
+				host = hostPort
+			}
+			ip := net.ParseIP(host)
+			if ip != nil {
+				return hostPort
+			}
+			host = strings.Split(host, ".")[0]
+			if port != "" {
+				return net.JoinHostPort(host, port)
+			}
+			return host
+		},
+
+		// match reports whether the string s
+		// contains any match of the regular expression pattern.
+		// alias for https://golang.org/pkg/regexp/#MatchString
+		"match": regexp.MatchString,
+
+		// reReplaceAll ReplaceAllString returns a copy of src, replacing matches of the Regexp with
+		// the replacement string repl. Inside repl, $ signs are interpreted as in Expand,
+		// so for instance $1 represents the text of the first submatch.
+		// alias for https://golang.org/pkg/regexp/#Regexp.ReplaceAllString
+		"reReplaceAll": func(pattern, repl, text string) string {
+			re := regexp.MustCompile(pattern)
+			return re.ReplaceAllString(text, repl)
+		},
+
+		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
+		"parseDuration": func(s string) (float64, error) {
+			d, err := timeutil.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d.Seconds(), nil
+		},
+
+		// same with parseDuration but returns a time.Duration
+		"parseDurationTime": func(s string) (time.Duration, error) {
+			d, err := timeutil.ParseDuration(s)
+			if err != nil {
+				return 0, err
+			}
+			return d, nil
+		},
+
+		/* Numbers */
+
+		// humanize converts given number to a human readable format
+		// by adding metric prefixes https://en.wikipedia.org/wiki/Metric_prefix
+		"humanize": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if v == 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			if math.Abs(v) >= 1 {
+				prefix := ""
+				for _, p := range []string{"k", "M", "G", "T", "P", "E", "Z", "Y"} {
+					if math.Abs(v) < 1000 {
+						break
+					}
+					prefix = p
+					v /= 1000
+				}
+				return fmt.Sprintf("%.4g%s", v, prefix), nil
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%s", v, prefix), nil
+		},
+
+		// humanize1024 converts given number to a human readable format with 1024 as base
+		"humanize1024": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.Abs(v) <= 1 || math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			return formatutil.HumanizeBytes(v), nil
+		},
+
+		// humanizeDuration converts given seconds to a human-readable duration
+		"humanizeDuration": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			if v == 0 {
+				return fmt.Sprintf("%.4gs", v), nil
+			}
+			if math.Abs(v) >= 1 {
+				sign := ""
+				if v < 0 {
+					sign = "-"
+					v = -v
+				}
+				seconds := int64(v) % 60
+				minutes := (int64(v) / 60) % 60
+				hours := (int64(v) / 60 / 60) % 24
+				days := int64(v) / 60 / 60 / 24
+				// For days to minutes, we display seconds as an integer.
+				if days != 0 {
+					return fmt.Sprintf("%s%dd %dh %dm %ds", sign, days, hours, minutes, seconds), nil
+				}
+				if hours != 0 {
+					return fmt.Sprintf("%s%dh %dm %ds", sign, hours, minutes, seconds), nil
+				}
+				if minutes != 0 {
+					return fmt.Sprintf("%s%dm %ds", sign, minutes, seconds), nil
+				}
+				// For seconds, we display 4 significant digits.
+				return fmt.Sprintf("%s%.4gs", sign, v), nil
+			}
+			prefix := ""
+			for _, p := range []string{"m", "u", "n", "p", "f", "a", "z", "y"} {
+				if math.Abs(v) >= 1 {
+					break
+				}
+				prefix = p
+				v *= 1000
+			}
+			return fmt.Sprintf("%.4g%ss", v, prefix), nil
+		},
+
+		// humanizePercentage converts given ratio value to a fraction of 100
+		"humanizePercentage": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			return fmt.Sprintf("%.4g%%", v*100), nil
+		},
+
+		// humanizeTimestamp converts given timestamp to a human readable time equivalent
+		"humanizeTimestamp": func(i any) (string, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return "", err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return fmt.Sprintf("%.4g", v), nil
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return fmt.Sprint(t), nil
+		},
+
+		// toTime converts given timestamp to a time.Time.
+		"toTime": func(i any) (time.Time, error) {
+			v, err := toFloat64(i)
+			if err != nil {
+				return time.Time{}, err
+			}
+			if math.IsNaN(v) || math.IsInf(v, 0) {
+				return time.Time{}, fmt.Errorf("cannot convert %v to time.Time", v)
+			}
+			t := timeFromUnixTimestamp(v).Time().UTC()
+			return t, nil
+		},
+
+		/* URLs */
+
+		// externalURL returns value of `external.url` flag
+		"externalURL": func() string {
+			// externalURL function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
+		},
+
+		// pathPrefix returns a Path segment from the URL value in `external.url` flag
+		"pathPrefix": func() string {
+			// pathPrefix function supposed to be substituted at FuncsWithExteralURL().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			return ""
+		},
+
+		// pathEscape escapes the string so it can be safely placed inside a URL path segment.
+		//
+		// See also queryEscape.
+		"pathEscape": url.PathEscape,
+
+		// queryEscape escapes the string so it can be safely placed inside a query arg in URL.
+		//
+		// See also queryEscape.
+		"queryEscape": url.QueryEscape,
+
+		// query executes the MetricsQL/PromQL query against
+		// configured `datasource.url` address.
+		// For example, {{ query "foo" | first | value }} will
+		// execute "/api/v1/query?query=foo" request and will return
+		// the first value in response.
+		"query": func(_ string) ([]metric, error) {
+			// query function supposed to be substituted at FuncsWithQuery().
+			// it is present here only for validation purposes, when there is no
+			// provided datasource.
+			//
+			// return non-empty slice to pass validation with chained functions in template
+			// see issue #989 for details
+			return []metric{{}}, nil
+		},
+
+		// first returns the first by order element from the given metrics list.
+		// usually used alongside with `query` template function.
+		"first": func(metrics []metric) (metric, error) {
+			if len(metrics) > 0 {
+				return metrics[0], nil
+			}
+			return metric{}, errors.New("first() called on vector with no elements")
+		},
+
+		// label returns the value of the given label name for the given metric.
+		// usually used alongside with `query` template function.
+		"label": func(label string, m metric) string {
+			return m.Labels[label]
+		},
+
+		// value returns the value of the given metric.
+		// usually used alongside with `query` template function.
+		"value": func(m metric) float64 {
+			return m.Value
+		},
+
+		// strvalue returns metric name.
+		"strvalue": func(m metric) string {
+			return m.Labels["__name__"]
+		},
+
+		// sortByLabel sorts the given metrics by provided label key
+		"sortByLabel": func(label string, metrics []metric) []metric {
+			sort.SliceStable(metrics, func(i, j int) bool {
+				return metrics[i].Labels[label] < metrics[j].Labels[label]
+			})
+			return metrics
+		},
+
+		/* Helpers */
+
+		// Converts a list of objects to a map with keys arg0, arg1 etc.
+		// This is intended to allow multiple arguments to be passed to templates.
+		"args": func(args ...any) map[string]any {
+			result := make(map[string]any)
+			for i, a := range args {
+				result[fmt.Sprintf("arg%d", i)] = a
+			}
+			return result
+		},
+
+		// safeHtml marks string as HTML not requiring auto-escaping.
+		//
+		// See also htmlEscape.
+		"safeHtml": func(text string) htmlTpl.HTML {
+			return htmlTpl.HTML(text)
+		},
 	}
-	return ExecuteWithTemplate(data, tmpl)
 }
 
-// ExecuteWithTemplate executes with the given template and data
-func ExecuteWithTemplate(data AlertTplData, tpl *textTpl.Template) (string, error) {
-	fullData := tplData{
-		data,
-		externalLabels,
-		externalURL.String(),
-	}
-	var buf bytes.Buffer
-	// returns the zero value for the map type's element
-	tpl.Option("missingkey=zero")
-	if err := tpl.Execute(&buf, fullData); err != nil {
-		return "", fmt.Errorf("failed to execute template: %w", err)
-	}
-	return buf.String(), nil
+// Time is the number of milliseconds since the epoch
+// (1970-01-01 00:00 UTC) excluding leap seconds.
+type Time int64
+
+// timeFromUnixTimestamp returns the Time equivalent to t in unix timestamp.
+func timeFromUnixTimestamp(t float64) Time {
+	return Time(t * 1e3)
+}
+
+// The number of nanoseconds per minimum tick.
+const nanosPerTick = int64(minimumTick / time.Nanosecond)
+
+// MinimumTick is the minimum supported time resolution. This has to be
+// at least time.Second in order for the code below to work.
+const minimumTick = time.Millisecond
+
+// second is the Time duration equivalent to one second.
+const second = int64(time.Second / minimumTick)
+
+// Time returns the time.Time representation of t.
+func (t Time) Time() time.Time {
+	return time.Unix(int64(t)/second, (int64(t)%second)*nanosPerTick)
+}
+
+func toFloat64(v any) (float64, error) {
+	switch i := v.(type) {
+	case float64:
+		return i, nil
+	case float32:
+		return float64(i), nil
+	case int64:
+		return float64(i), nil
+	case int32:
+		return float64(i), nil
+	case int:
+		return float64(i), nil
+	case uint64:
+		return float64(i), nil
+	case uint32:
+		return float64(i), nil
+	case uint:
+		return float64(i), nil
+	case string:
+		return strconv.ParseFloat(i, 64)
+	default:
+		return 0, fmt.Errorf("unexpected value type %v", i)
+	}
 }

app/vmalert/templates/template_test.go

@@ -1,226 +1,239 @@
 package templates
 
 import (
-	"fmt"
+	"math"
 	"net/url"
+	"strings"
 	"testing"
-	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	textTpl "text/template"
 )
 
-func TestValidateTemplates(t *testing.T) {
-	f := func(annotations map[string]string, isValid bool) {
+func TestTemplateFuncs_StringConversion(t *testing.T) {
+	f := func(funcName, s, resultExpected string) {
 		t.Helper()
 
-		err := ValidateTemplates(annotations)
-		if (err == nil) != isValid {
-			t.Fatalf("failed to validate template, got %t; want %t", (err == nil), isValid)
+		funcs := templateFuncs()
+		v := funcs[funcName]
+		fLocal := v.(func(s string) string)
+		result := fLocal(s)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%q); got\n%s\nwant\n%s", funcName, s, result, resultExpected)
 		}
 	}
 
-	// empty
-	f(map[string]string{}, true)
-
-	// wrong text
-	f(map[string]string{
-		"summary": "{{",
-	}, false)
-
-	// valid
-	f(map[string]string{
-		"value":   "{{$value}}",
-		"summary": "it's a test summary",
-	}, true)
-
-	// invalid variable
-	f(map[string]string{
-		"value":   "{{$invalidValue}}",
-		"summary": "it's a test summary",
-	}, false)
+	f("title", "foo bar", "Foo Bar")
+	f("toUpper", "foo", "FOO")
+	f("toLower", "FOO", "foo")
+	f("pathEscape", "foo/bar\n+baz", "foo%2Fbar%0A+baz")
+	f("queryEscape", "foo+bar\n+baz", "foo%2Bbar%0A%2Bbaz")
+	f("jsonEscape", `foo{bar="baz"}`+"\n + 1", `"foo{bar=\"baz\"}\n + 1"`)
+	f("quotesEscape", `foo{bar="baz"}`+"\n + 1", `foo{bar=\"baz\"}\n + 1`)
+	f("htmlEscape", "foo < 10\nabc", "foo &lt; 10\nabc")
+	f("crlfEscape", "foo\nbar\rx", `foo\nbar\rx`)
+	f("stripPort", "foo", "foo")
+	f("stripPort", "foo:1234", "foo")
+	f("stripDomain", "foo.bar.baz", "foo")
+	f("stripDomain", "foo.bar:123", "foo:123")
 }
 
-func TestExecuteWithoutTemplate(t *testing.T) {
-	extLabels := make(map[string]string)
-	const (
-		extCluster = "prod"
-		extDC      = "east"
-		extURL     = "https://foo.bar"
-	)
-	url, _ := url.Parse(extURL)
-	extLabels["cluster"] = extCluster
-	extLabels["dc"] = extDC
-
-	err := Init(nil, extLabels, *url)
+func TestTemplateFuncs_Match(t *testing.T) {
+	funcs := templateFuncs()
+	// check "match" func
+	matchFunc := funcs["match"].(func(pattern, s string) (bool, error))
+	if _, err := matchFunc("invalid[regexp", "abc"); err == nil {
+		t.Fatalf("expecting non-nil error on invalid regexp")
+	}
+	ok, err := matchFunc("abc", "def")
 	if err != nil {
-		t.Fatalf("cannot init templates: %s", err)
+		t.Fatalf("unexpected error")
 	}
+	if ok {
+		t.Fatalf("unexpected match")
+	}
+	ok, err = matchFunc("a.+b", "acsdb")
+	if err != nil {
+		t.Fatalf("unexpected error")
+	}
+	if !ok {
+		t.Fatalf("unexpected mismatch")
+	}
+}
 
-	f := func(data AlertTplData, annotations, expResults map[string]string) {
+func TestTemplateFuncs_Formatting(t *testing.T) {
+	f := func(funcName string, p any, resultExpected string) {
 		t.Helper()
 
-		qFn := func(_ string) ([]datasource.Metric, error) {
-			return []datasource.Metric{
-				{
-					Labels: []prompbmarshal.Label{
-						{Name: "foo", Value: "bar"},
-						{Name: "baz", Value: "qux"},
-					},
-					Values:     []float64{1},
-					Timestamps: []int64{1},
-				},
-				{
-					Labels: []prompbmarshal.Label{
-						{Name: "foo", Value: "garply"},
-						{Name: "baz", Value: "fred"},
-					},
-					Values:     []float64{2},
-					Timestamps: []int64{1},
-				},
-			}, nil
+		funcs := templateFuncs()
+		v := funcs[funcName]
+		fLocal := v.(func(s any) (string, error))
+		result, err := fLocal(p)
+		if err != nil {
+			t.Fatalf("unexpected error for %s(%f): %s", funcName, p, err)
 		}
-
-		for k := range annotations {
-			v, err := ExecuteWithoutTemplate(qFn, annotations[k], data)
-			if err != nil {
-				t.Fatalf("cannot execute template: %s", err)
-			}
-			if v != expResults[k] {
-				t.Fatalf("unexpected result; got %s; want %s", v, expResults[k])
-			}
+		if result != resultExpected {
+			t.Fatalf("unexpected result for %s(%f); got\n%s\nwant\n%s", funcName, p, result, resultExpected)
 		}
-
 	}
 
-	// empty-alert
-	f(AlertTplData{}, map[string]string{}, map[string]string{})
+	f("humanize1024", float64(0), "0")
+	f("humanize1024", math.Inf(0), "+Inf")
+	f("humanize1024", math.NaN(), "NaN")
+	f("humanize1024", float64(127087), "124.1ki")
+	f("humanize1024", float64(130137088), "124.1Mi")
+	f("humanize1024", float64(133260378112), "124.1Gi")
+	f("humanize1024", float64(136458627186688), "124.1Ti")
+	f("humanize1024", float64(139733634239168512), "124.1Pi")
+	f("humanize1024", float64(143087241460908556288), "124.1Ei")
+	f("humanize1024", float64(146521335255970361638912), "124.1Zi")
+	f("humanize1024", float64(150037847302113650318245888), "124.1Yi")
+	f("humanize1024", float64(153638755637364377925883789312), "1.271e+05Yi")
 
-	// no-template
-	f(AlertTplData{
-		Value: 1e4,
-		Labels: map[string]string{
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"summary":     "it's a test summary",
-		"description": "it's a test description",
-	}, map[string]string{
-		"summary":     "it's a test summary",
-		"description": "it's a test description",
-	})
+	f("humanize", float64(127087), "127.1k")
+	f("humanize", float64(136458627186688), "136.5T")
 
-	// label-template
-	f(AlertTplData{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-		For: 5 * time.Minute,
-	}, map[string]string{
-		"summary":            "Too high connection number for {{$labels.instance}} for job {{$labels.job}}",
-		"description":        "It is {{ $value }} connections for {{$labels.instance}} for more than {{ .For }}",
-		"non-existing-label": "{{$labels.nonexisting}}",
-	}, map[string]string{
-		"summary":            "Too high connection number for localhost for job staging",
-		"description":        "It is 10000 connections for localhost for more than 5m0s",
-		"non-existing-label": "",
-	})
+	f("humanizeDuration", 1, "1s")
+	f("humanizeDuration", 0.2, "200ms")
+	f("humanizeDuration", 42000, "11h 40m 0s")
+	f("humanizeDuration", 16790555, "194d 8h 2m 35s")
 
-	// label template override
-	f(AlertTplData{
-		Value: 1e4,
-	}, map[string]string{
-		"summary":     `{{- define "default.template" -}} {{ printf "summary" }} {{- end -}} {{ template "default.template" . }}`,
-		"description": `{{- define "default.template" -}} {{ printf "description" }} {{- end -}} {{ template "default.template" . }}`,
-		"value":       `{{$value }}`,
-	}, map[string]string{
-		"summary":     "summary",
-		"description": "description",
-		"value":       "10000",
-	})
+	f("humanizePercentage", 1, "100%")
+	f("humanizePercentage", 0.8, "80%")
+	f("humanizePercentage", 0.015, "1.5%")
 
-	// expression-template
-	f(AlertTplData{
-		Expr: `vm_rows{"label"="bar"}<0`,
-	}, map[string]string{
-		"exprEscapedQuery":  "{{ $expr|queryEscape }}",
-		"exprEscapedPath":   "{{ $expr|pathEscape }}",
-		"exprEscapedJSON":   "{{ $expr|jsonEscape }}",
-		"exprEscapedQuotes": "{{ $expr|quotesEscape }}",
-		"exprEscapedHTML":   "{{ $expr|htmlEscape }}",
-	}, map[string]string{
-		"exprEscapedQuery":  "vm_rows%7B%22label%22%3D%22bar%22%7D%3C0",
-		"exprEscapedPath":   "vm_rows%7B%22label%22=%22bar%22%7D%3C0",
-		"exprEscapedJSON":   `"vm_rows{\"label\"=\"bar\"}\u003c0"`,
-		"exprEscapedQuotes": `vm_rows{\"label\"=\"bar\"}\u003c0`,
-		"exprEscapedHTML":   "vm_rows{&quot;label&quot;=&quot;bar&quot;}&lt;0",
-	})
-
-	// query function
-	f(AlertTplData{
-		Expr: `vm_rows{"label"="bar"}>0`,
-	}, map[string]string{
-		"summary": `{{ query "foo" | first | value }}`,
-		"desc":    `{{ range query "bar" }}{{ . | label "foo" }} {{ . | value }};{{ end }}`,
-	}, map[string]string{
-		"summary": "1",
-		"desc":    "bar 1;garply 2;",
-	})
-
-	// external
-	f(AlertTplData{
-		Value: 1e4,
-		Labels: map[string]string{
-			"job":      "staging",
-			"instance": "localhost",
-		},
-	}, map[string]string{
-		"url":         "{{ $externalURL }}",
-		"summary":     "Issues with {{$labels.instance}} (dc-{{$externalLabels.dc}}) for job {{$labels.job}}",
-		"description": "It is {{ $value }} connections for {{$labels.instance}} (cluster-{{$externalLabels.cluster}})",
-	}, map[string]string{
-		"url":         extURL,
-		"summary":     fmt.Sprintf("Issues with localhost (dc-%s) for job staging", extDC),
-		"description": fmt.Sprintf("It is 10000 connections for localhost (cluster-%s)", extCluster),
-	})
-
-	// alert, group IDs & ActiveAt time
-	f(AlertTplData{
-		AlertID:  42,
-		GroupID:  24,
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"url":     "/api/v1/alert?alertID={{$alertID}}&groupID={{$groupID}}",
-		"diagram": "![](http://example.com?render={{$activeAt.Unix}}",
-	}, map[string]string{
-		"url":     "/api/v1/alert?alertID=42&groupID=24",
-		"diagram": "![](http://example.com?render=1660941298",
-	})
-
-	// ActiveAt time is nil
-	f(AlertTplData{}, map[string]string{
-		"default_time": "{{$activeAt}}",
-	}, map[string]string{
-		"default_time": "0001-01-01 00:00:00 +0000 UTC",
-	})
-
-	// ActiveAt custom format
-	f(AlertTplData{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"fire_time": `{{$activeAt.Format "2006/01/02 15:04:05"}}`,
-	}, map[string]string{
-		"fire_time": "2022/08/19 20:34:58",
-	})
-
-	// ActiveAt query range
-	f(AlertTplData{
-		ActiveAt: time.Date(2022, 8, 19, 20, 34, 58, 651387237, time.UTC),
-	}, map[string]string{
-		"grafana_url": `vm-grafana.com?from={{($activeAt.Add (parseDurationTime "1h")).Unix}}&to={{($activeAt.Add (parseDurationTime "-1h")).Unix}}`,
-	}, map[string]string{
-		"grafana_url": "vm-grafana.com?from=1660944898&to=1660937698",
-	})
+	f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
+}
+
+func mkTemplate(current, replacement any) textTemplate {
+	tmpl := textTemplate{}
+	if current != nil {
+		switch val := current.(type) {
+		case string:
+			tmpl.current = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	if replacement != nil {
+		switch val := replacement.(type) {
+		case string:
+			tmpl.replacement = textTpl.Must(newTemplate().Parse(val))
+		}
+	}
+	return tmpl
+}
+
+func equalTemplates(tmpls ...*textTpl.Template) bool {
+	var cmp *textTpl.Template
+	for i, tmpl := range tmpls {
+		if i == 0 {
+			cmp = tmpl
+		} else {
+			if cmp == nil || tmpl == nil {
+				if cmp != tmpl {
+					return false
+				}
+				continue
+			}
+			if len(tmpl.Templates()) != len(cmp.Templates()) {
+				return false
+			}
+			for _, t := range tmpl.Templates() {
+				tp := cmp.Lookup(t.Name())
+				if tp == nil {
+					return false
+				}
+				if tp.Root.String() != t.Root.String() {
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
+func TestTemplatesLoad_Failure(t *testing.T) {
+	f := func(pathPatterns []string, expectedErrStr string) {
+		t.Helper()
+
+		err := Load(pathPatterns, url.URL{})
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+
+		errStr := err.Error()
+		if !strings.Contains(errStr, expectedErrStr) {
+			t.Fatalf("the returned error %q doesn't contain %q", errStr, expectedErrStr)
+		}
+	}
+
+	// load template with syntax error
+	f([]string{
+		"templates/other/nested/bad0-*.tpl",
+		"templates/test/good0-*.tpl",
+	}, "failed to parse template glob")
+}
+
+func TestTemplatesLoad_Success(t *testing.T) {
+	f := func(pathPatterns []string, expectedTmpl textTemplate) {
+		t.Helper()
+
+		masterTmplOrig := masterTmpl
+		defer func() {
+			masterTmpl = masterTmplOrig
+		}()
+
+		if err := Load(pathPatterns, url.URL{}); err != nil {
+			t.Fatalf("cannot load templates: %s", err)
+		}
+		Reload()
+
+		if !equalTemplates(masterTmpl.replacement, expectedTmpl.replacement) {
+			t.Fatalf("unexpected replacement template\ngot\n%+v\nwant\n%+v", masterTmpl.replacement, expectedTmpl.replacement)
+		}
+		if !equalTemplates(masterTmpl.current, expectedTmpl.current) {
+			t.Fatalf("unexpected current template\ngot\n%+v\nwant\n%+v", masterTmpl.current, expectedTmpl.current)
+		}
+	}
+
+	// non existing path
+	pathPatterns := []string{
+		"templates/non-existing/good-*.tpl",
+		"templates/absent/good-*.tpl",
+	}
+	expectedTmpl := mkTemplate(``, nil)
+	f(pathPatterns, expectedTmpl)
+
+	// existing path
+	pathPatterns = []string{
+		"templates/test/good0-*.tpl",
+	}
+	expectedTmpl = mkTemplate(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.2" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`, nil)
+	f(pathPatterns, expectedTmpl)
+
+	// existing path defined template override
+	pathPatterns = []string{
+		"templates/other/nested/good0-*.tpl",
+	}
+	expectedTmpl = mkTemplate(`
+		{{- define "good0-test.tpl" -}}{{- end -}}
+		{{- define "test.0" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.1" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+		{{- define "test.3" -}}
+			{{ printf "Hello %s!" externalURL }}
+		{{- end -}}
+	`, nil)
+	f(pathPatterns, expectedTmpl)
 }

app/vmalert/utils/metrics.go

@@ -4,11 +4,13 @@ import "github.com/VictoriaMetrics/metrics"
 
 type namedMetric struct {
 	Name string
+
+	set *metrics.Set
 }
 
 // Unregister removes the metric by name from default registry
 func (nm namedMetric) Unregister() {
-	metrics.UnregisterMetric(nm.Name)
+	nm.set.UnregisterMetric(nm.Name)
 }
 
 // Gauge is a metrics.Gauge with Name
@@ -17,11 +19,11 @@ type Gauge struct {
 	*metrics.Gauge
 }
 
-// GetOrCreateGauge creates a new Gauge with the given name
-func GetOrCreateGauge(name string, f func() float64) *Gauge {
+// NewGauge creates a new Gauge with the given name
+func NewGauge(set *metrics.Set, name string, f func() float64) *Gauge {
 	return &Gauge{
-		namedMetric: namedMetric{Name: name},
-		Gauge:       metrics.GetOrCreateGauge(name, f),
+		namedMetric: namedMetric{Name: name, set: set},
+		Gauge:       set.NewGauge(name, f),
 	}
 }
 
@@ -31,11 +33,11 @@ type Counter struct {
 	*metrics.Counter
 }
 
-// GetOrCreateCounter creates a new Counter with the given name
-func GetOrCreateCounter(name string) *Counter {
+// NewCounter creates a new Counter with the given name
+func NewCounter(set *metrics.Set, name string) *Counter {
 	return &Counter{
-		namedMetric: namedMetric{Name: name},
-		Counter:     metrics.GetOrCreateCounter(name),
+		namedMetric: namedMetric{Name: name, set: set},
+		Counter:     set.NewCounter(name),
 	}
 }
 
@@ -45,10 +47,10 @@ type Summary struct {
 	*metrics.Summary
 }
 
-// GetOrCreateSummary creates a new Summary with the given name
-func GetOrCreateSummary(name string) *Summary {
+// NewSummary creates a new Summary with the given name
+func NewSummary(set *metrics.Set, name string) *Summary {
 	return &Summary{
-		namedMetric: namedMetric{Name: name},
-		Summary:     metrics.GetOrCreateSummary(name),
+		namedMetric: namedMetric{Name: name, set: set},
+		Summary:     set.NewSummary(name),
 	}
 }

app/vmalert/web_test.go

@@ -21,14 +21,18 @@ func TestHandler(t *testing.T) {
 	fq.Add(datasource.Metric{
 		Values: []float64{1}, Timestamps: []int64{0},
 	})
-	g := &rule.Group{
+	g := rule.NewGroup(config.Group{
 		Name:        "group",
 		File:        "rules.yaml",
 		Concurrency: 1,
-	}
-	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
-	rr := rule.NewRecordingRule(fq, g, config.Rule{ID: 1, Record: "record"})
-	g.Rules = []rule.Rule{ar, rr}
+		Rules: []config.Rule{
+			{ID: 0, Alert: "alert"},
+			{ID: 1, Record: "record"},
+		},
+	}, fq, 1*time.Minute, nil)
+	ar := g.Rules[0].(*rule.AlertingRule)
+	rr := g.Rules[1].(*rule.RecordingRule)
+
 	g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
 
 	m := &manager{groups: map[uint64]*rule.Group{
@@ -162,7 +166,7 @@ func TestHandler(t *testing.T) {
 
 		gotRuleWithUpdates := apiRuleWithUpdates{}
 		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
-		if gotRuleWithUpdates.StateUpdates == nil || len(gotRuleWithUpdates.StateUpdates) < 1 {
+		if len(gotRuleWithUpdates.StateUpdates) < 1 {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})

app/vmalert/web_types_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -15,20 +16,22 @@ func TestRecordingToApi(t *testing.T) {
 	fq.Add(datasource.Metric{
 		Values: []float64{1}, Timestamps: []int64{0},
 	})
-	g := &rule.Group{
+	entriesLimit := 44
+	g := rule.NewGroup(config.Group{
 		Name:        "group",
 		File:        "rules.yaml",
 		Concurrency: 1,
-	}
-
-	entriesLimit := 44
-	rr := rule.NewRecordingRule(fq, g, config.Rule{
-		ID:                 1248,
-		Record:             "record_name",
-		Expr:               "up",
-		Labels:             map[string]string{"label": "value"},
-		UpdateEntriesLimit: &entriesLimit,
-	})
+		Rules: []config.Rule{
+			{
+				ID:                 1248,
+				Record:             "record_name",
+				Expr:               "up",
+				Labels:             map[string]string{"label": "value"},
+				UpdateEntriesLimit: &entriesLimit,
+			},
+		},
+	}, fq, 1*time.Minute, nil)
+	rr := g.Rules[0].(*rule.RecordingRule)
 
 	expectedRes := apiRule{
 		Name:           "record_name",

app/vmauth/main.go

@@ -31,7 +31,11 @@ import (
 )
 
 var (
-	httpListenAddrs  = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. See also -tls and -httpListenAddr.useProxyProtocol")
+	httpListenAddrs = flagutil.NewArrayString("httpListenAddr", "TCP address to listen for incoming http requests. "+
+		"By default, serves internal API and proxy requests. "+
+		" See also -tls, -httpListenAddr.useProxyProtocol and -httpInternalListenAddr.")
+	httpInternalListenAddr = flagutil.NewArrayString("httpInternalListenAddr", "TCP address to listen for incoming internal API http requests. Such as /health, /-/reload, /debug/pprof, etc. "+
+		"If flag is set, vmauth no longer serves internal API at -httpListenAddr.")
 	useProxyProtocol = flagutil.NewArrayBool("httpListenAddr.useProxyProtocol", "Whether to use proxy protocol for connections accepted at the corresponding -httpListenAddr . "+
 		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
 		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
@@ -91,7 +95,21 @@ func main() {
 	logger.Infof("starting vmauth at %q...", listenAddrs)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(listenAddrs, useProxyProtocol, requestHandler)
+	disableInternalRoutes := len(*httpInternalListenAddr) > 0
+	rh := requestHandlerWithInternalRoutes
+	if disableInternalRoutes {
+		rh = requestHandler
+	}
+
+	serveOpts := httpserver.ServeOptions{
+		UseProxyProtocol:     useProxyProtocol,
+		DisableBuiltinRoutes: disableInternalRoutes,
+	}
+	go httpserver.ServeWithOpts(listenAddrs, rh, serveOpts)
+
+	if len(*httpInternalListenAddr) > 0 {
+		go httpserver.Serve(*httpInternalListenAddr, nil, internalRequestHandler)
+	}
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	pushmetrics.Init()
@@ -109,7 +127,7 @@ func main() {
 	logger.Infof("successfully stopped vmauth in %.3f seconds", time.Since(startTime).Seconds())
 }
 
-func requestHandler(w http.ResponseWriter, r *http.Request) bool {
+func internalRequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
 		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey) {
@@ -120,6 +138,17 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusOK)
 		return true
 	}
+	return false
+}
+
+func requestHandlerWithInternalRoutes(w http.ResponseWriter, r *http.Request) bool {
+	if internalRequestHandler(w, r) {
+		return true
+	}
+	return requestHandler(w, r)
+}
+
+func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	ats := getAuthTokensFromRequest(r)
 	if len(ats) == 0 {
@@ -222,8 +251,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		isDefault = true
 	}
 
-	rtb := getReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
-	defer putReadTrackingBody(rtb)
+	rtb := newReadTrackingBody(r.Body, maxRequestBodySizeToRetry.IntN())
 	r.Body = rtb
 
 	maxAttempts := up.getBackendsCount()
@@ -559,22 +587,11 @@ type readTrackingBody struct {
 	bufComplete bool
 }
 
-func (rtb *readTrackingBody) reset() {
-	rtb.maxBodySize = 0
-	rtb.r = nil
-	rtb.buf = rtb.buf[:0]
-	rtb.readBuf = nil
-	rtb.cannotRetry = false
-	rtb.bufComplete = false
-}
-
-func getReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
-	v := readTrackingBodyPool.Get()
-	if v == nil {
-		v = &readTrackingBody{}
-	}
-	rtb := v.(*readTrackingBody)
-
+func newReadTrackingBody(r io.ReadCloser, maxBodySize int) *readTrackingBody {
+	// do not use sync.Pool there
+	// since http.RoundTrip may still use request body after return
+	// See this issue for details https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8051
+	rtb := &readTrackingBody{}
 	if maxBodySize < 0 {
 		maxBodySize = 0
 	}
@@ -597,13 +614,6 @@ func (r *zeroReader) Close() error {
 	return nil
 }
 
-func putReadTrackingBody(rtb *readTrackingBody) {
-	rtb.reset()
-	readTrackingBodyPool.Put(rtb)
-}
-
-var readTrackingBodyPool sync.Pool
-
 // Read implements io.Reader interface.
 func (rtb *readTrackingBody) Read(p []byte) (int, error) {
 	if len(rtb.readBuf) > 0 {

app/vmauth/main_test.go

@@ -52,7 +52,7 @@ func TestRequestHandler(t *testing.T) {
 		r.Header.Set("Pass-Header", "abc")
 
 		w := &fakeResponseWriter{}
-		if !requestHandler(w, r) {
+		if !requestHandlerWithInternalRoutes(w, r) {
 			t.Fatalf("unexpected false is returned from requestHandler")
 		}
 
@@ -195,7 +195,7 @@ unauthorized_user:
 	}
 	responseExpected = `
 statusCode=401
-The provided authKey doesn't match -reloadAuthKey`
+Expected to receive non-empty authKey when -reloadAuthKey is set`
 	f(cfgStr, requestURL, backendHandler, responseExpected)
 	if err := reloadAuthKey.Set(origAuthKey); err != nil {
 		t.Fatalf("unexpected error: %s", err)
@@ -545,8 +545,7 @@ func TestReadTrackingBody_RetrySuccess(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")
@@ -581,8 +580,7 @@ func TestReadTrackingBody_RetrySuccessPartialRead(t *testing.T) {
 		t.Helper()
 
 		// Check the case with partial read
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		for i := 0; i < len(s); i++ {
 			buf := make([]byte, i)
@@ -631,8 +629,7 @@ func TestReadTrackingBody_RetryFailureTooBigBody(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")
@@ -681,8 +678,7 @@ func TestReadTrackingBody_RetryFailureZeroOrNegativeMaxBodySize(t *testing.T) {
 	f := func(s string, maxBodySize int) {
 		t.Helper()
 
-		rtb := getReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
-		defer putReadTrackingBody(rtb)
+		rtb := newReadTrackingBody(io.NopCloser(bytes.NewBufferString(s)), maxBodySize)
 
 		if !rtb.canRetry() {
 			t.Fatalf("canRetry() must return true before reading anything")

app/vmauth/target_url_test.go

@@ -98,6 +98,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		up, hc := ui.getURLPrefixAndHeaders(u, u.Host, nil)
 		if up == nil {
 			t.Fatalf("cannot match available backend: %s", err)
+			return
 		}
 		bu := up.getBackendURL()
 		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
@@ -309,6 +310,7 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 		up, _ := ui.getURLPrefixAndHeaders(u, u.Host, nil)
 		if up == nil {
 			t.Fatalf("cannot match available backend: %s", err)
+			return
 		}
 		bu := up.getBackendURL()
 		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)

app/vmctl/flags.go

@@ -344,7 +344,7 @@ var (
 		},
 		&cli.BoolFlag{
 			Name:  influxSkipDatabaseLabel,
-			Usage: "Wether to skip adding the label 'db' to timeseries.",
+			Usage: "Whether to skip adding the label 'db' to timeseries.",
 			Value: false,
 		},
 		&cli.BoolFlag{
@@ -596,7 +596,8 @@ var (
 		&cli.Int64Flag{
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
-				"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases.",
+				"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases. \n" +
+				"Rate limit is applied per worker, see `--vm-concurrency`.",
 		},
 		&cli.BoolFlag{
 			Name: vmInterCluster,

app/vmctl/influx/influx.go

@@ -144,9 +144,9 @@ func timeFilter(start, end string) string {
 // by checking available (non-empty) tags, fields and measurements
 // which unique combination represents all possible
 // time series existing in database.
-// The explore required to reduce the load on influx
+// Explore is required to reduce the load on influx
 // by querying field of the exact time series at once,
-// instead of fetching all of the values over and over.
+// instead of fetching all the values over and over.
 //
 // May contain non-existing time series.
 func (c *Client) Explore() ([]*Series, error) {
@@ -340,10 +340,7 @@ func (c *Client) fieldsByMeasurement() (map[string][]string, error) {
 }
 
 func (c *Client) getSeries() ([]*Series, error) {
-	com := "show series"
-	if c.filterSeries != "" {
-		com = fmt.Sprintf("%s %s", com, c.filterSeries)
-	}
+	com := c.getSeriesCommand()
 	q := influx.Query{
 		Command:         com,
 		Database:        c.database,
@@ -389,6 +386,21 @@ func (c *Client) getSeries() ([]*Series, error) {
 	return result, nil
 }
 
+func (c *Client) getSeriesCommand() string {
+	com := "show series"
+	if c.filterSeries != "" {
+		com = fmt.Sprintf("%s %s", com, c.filterSeries)
+	}
+	if c.filterTime != "" {
+		joinStatement := " where "
+		if strings.Contains(strings.ToLower(com), joinStatement) {
+			joinStatement = " AND "
+		}
+		com = fmt.Sprintf("%s%s%s", com, joinStatement, c.filterTime)
+	}
+	return com
+}
+
 // getMeasurementTags get the tags for each measurement.
 // tags are placed in a map without values (similar to a set) for quick lookups:
 // {"measurement1": {"tag1", "tag2"}, "measurement2": {"tag3", "tag4"}}

app/vmctl/influx/influx_test.go

@@ -103,3 +103,26 @@ func TestTimeFilter(t *testing.T) {
 	// both start and end filters
 	f("2020-01-01T20:07:00Z", "2020-01-01T21:07:00Z", "time >= '2020-01-01T20:07:00Z' and time <= '2020-01-01T21:07:00Z'")
 }
+
+func TestGetSeriesCommand(t *testing.T) {
+	f := func(filterSeries, filterTime, expCommand string) {
+		t.Helper()
+
+		c := &Client{
+			filterTime:   filterTime,
+			filterSeries: filterSeries,
+		}
+		gotCommand := c.getSeriesCommand()
+		if gotCommand != expCommand {
+			t.Fatalf("unexpected command\ngot\n%s\nwant\n%s", gotCommand, expCommand)
+		}
+	}
+
+	f("", "", "show series")
+	f("from cpu", "", "show series from cpu")
+	f("from cpu where arch='x86'", "", "show series from cpu where arch='x86'")
+	f("", "time >= '2020-01-01T20:07:00Z'", "show series where time >= '2020-01-01T20:07:00Z'")
+	f("from cpu", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where time >= '2020-01-01T20:07:00Z'")
+	f("from cpu where arch='x86'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND time >= '2020-01-01T20:07:00Z'")
+	f("from cpu where arch='x86' AND hostname='host_2753'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND hostname='host_2753' AND time >= '2020-01-01T20:07:00Z'")
+}

app/vmctl/prometheus/prometheus.go

@@ -40,15 +40,15 @@ type filter struct {
 	labelValue string
 }
 
-func (f filter) inRange(min, max int64) bool {
+func (f filter) inRange(minV, maxV int64) bool {
 	fmin, fmax := f.min, f.max
-	if min == 0 {
-		fmin = min
+	if minV == 0 {
+		fmin = minV
 	}
 	if fmax == 0 {
-		fmax = max
+		fmax = maxV
 	}
-	return min <= fmax && fmin <= max
+	return minV <= fmax && fmin <= maxV
 }
 
 // NewClient creates and validates new Client
@@ -59,13 +59,13 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
 	}
 	c := &Client{DBReadOnly: db}
-	min, max, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
+	minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse time in filter: %s", err)
 	}
 	c.filter = filter{
-		min:        min,
-		max:        max,
+		min:        minTime,
+		max:        maxTime,
 		label:      cfg.Filter.Label,
 		labelValue: cfg.Filter.LabelValue,
 	}

app/vmctl/utils/time.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 const (
@@ -16,7 +16,7 @@ const (
 // ParseTime parses time in s string and returns time.Time object
 // if parse correctly or error if not
 func ParseTime(s string) (time.Time, error) {
-	msecs, err := promutils.ParseTimeMsec(s)
+	msecs, err := timeutil.ParseTimeMsec(s)
 	if err != nil {
 		return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err)
 	}

app/vminsert/common/streamaggr.go

@@ -36,6 +36,9 @@ var (
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
 	streamAggrIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after restarts. It could be caused by receiving unordered delayed data from clients pushing data into the database. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
+	streamAggrEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
 )
 
 var (
@@ -62,6 +65,7 @@ func CheckStreamAggrConfig() error {
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     *streamAggrIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		EnableWindows:        *streamAggrEnableWindows,
 	}
 	sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushNoop, opts, "global")
 	if err != nil {
@@ -78,7 +82,7 @@ func InitStreamAggr() {
 	saCfgReloaderStopCh = make(chan struct{})
 	if *streamAggrConfig == "" {
 		if *streamAggrDedupInterval > 0 {
-			deduplicator = streamaggr.NewDeduplicator(pushAggregateSeries, *streamAggrDedupInterval, *streamAggrDropInputLabels, "global")
+			deduplicator = streamaggr.NewDeduplicator(pushAggregateSeries, *streamAggrEnableWindows, *streamAggrDedupInterval, *streamAggrDropInputLabels, "global")
 		}
 		return
 	}

app/vminsert/prompush/push.go

@@ -2,6 +2,7 @@ package prompush
 
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
@@ -49,6 +50,7 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 	}
 	ctx.Reset(rowsLen)
 	rowsTotal := 0
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range tss {
 		ts := &tss[i]
 		rowsTotal += len(ts.Samples)
@@ -57,7 +59,7 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 			label := &ts.Labels[j]
 			ctx.AddLabel(label.Name, label.Value)
 		}
-		if !ctx.TryPrepareLabels(false) {
+		if !ctx.TryPrepareLabels(hasRelabeling) {
 			continue
 		}
 		var metricNameRaw []byte

app/vmselect/graphite/aggr.go

@@ -98,13 +98,13 @@ func aggrMin(values []float64) float64 {
 	if pos < 0 {
 		return nan
 	}
-	min := values[pos]
+	minV := values[pos]
 	for _, v := range values[pos+1:] {
-		if !math.IsNaN(v) && v < min {
-			min = v
+		if !math.IsNaN(v) && v < minV {
+			minV = v
 		}
 	}
-	return min
+	return minV
 }
 
 func aggrMax(values []float64) float64 {
@@ -112,13 +112,13 @@ func aggrMax(values []float64) float64 {
 	if pos < 0 {
 		return nan
 	}
-	max := values[pos]
+	maxV := values[pos]
 	for _, v := range values[pos+1:] {
-		if !math.IsNaN(v) && v > max {
-			max = v
+		if !math.IsNaN(v) && v > maxV {
+			maxV = v
 		}
 	}
-	return max
+	return maxV
 }
 
 func aggrDiff(values []float64) float64 {
@@ -177,12 +177,12 @@ func aggrCount(values []float64) float64 {
 }
 
 func aggrRange(values []float64) float64 {
-	min := aggrMin(values)
-	if math.IsNaN(min) {
+	minV := aggrMin(values)
+	if math.IsNaN(minV) {
 		return nan
 	}
-	max := aggrMax(values)
-	return max - min
+	maxV := aggrMax(values)
+	return maxV - minV
 }
 
 func aggrMultiply(values []float64) float64 {

app/vmselect/graphite/transform.go

@@ -2594,17 +2594,17 @@ func transformMinMax(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, e
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		min := aggrMin(values)
-		if math.IsNaN(min) {
-			min = 0
+		minV := aggrMin(values)
+		if math.IsNaN(minV) {
+			minV = 0
 		}
-		max := aggrMax(values)
-		if math.IsNaN(max) {
-			max = 0
+		maxV := aggrMax(values)
+		if math.IsNaN(maxV) {
+			maxV = 0
 		}
-		vRange := max - min
+		vRange := maxV - minV
 		for i, v := range values {
-			v = (v - min) / vRange
+			v = (v - minV) / vRange
 			if math.IsInf(v, 0) {
 				v = 0
 			}
@@ -2975,9 +2975,9 @@ func transformRemoveAbovePercentile(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		max := aggrFunc(values)
+		maxV := aggrFunc(values)
 		for i, v := range values {
-			if v > max {
+			if v > maxV {
 				values[i] = nan
 			}
 		}
@@ -3035,9 +3035,9 @@ func transformRemoveBelowPercentile(ec *evalConfig, fe *graphiteql.FuncExpr) (ne
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		min := aggrFunc(values)
+		minV := aggrFunc(values)
 		for i, v := range values {
-			if v < min {
+			if v < minV {
 				values[i] = nan
 			}
 		}
@@ -4514,11 +4514,11 @@ func transformOffsetToZero(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesF
 	}
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		values := s.Values
-		min := aggrMin(values)
+		minV := aggrMin(values)
 		for i, v := range values {
-			values[i] = v - min
+			values[i] = v - minV
 		}
-		s.Tags["offsetToZero"] = fmt.Sprintf("%g", min)
+		s.Tags["offsetToZero"] = fmt.Sprintf("%g", minV)
 		s.Name = fmt.Sprintf("offsetToZero(%s)", s.Name)
 		s.expr = fe
 		s.pathExpression = s.Name
@@ -4567,29 +4567,29 @@ func transformPerSecond(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc
 	return f, nil
 }
 
-func nonNegativeDelta(curr, prev, max, min float64) (float64, float64) {
-	if !math.IsNaN(max) && curr > max {
+func nonNegativeDelta(currV, prevV, maxV, minV float64) (float64, float64) {
+	if !math.IsNaN(maxV) && currV > maxV {
 		return nan, nan
 	}
-	if !math.IsNaN(min) && curr < min {
+	if !math.IsNaN(minV) && currV < minV {
 		return nan, nan
 	}
-	if math.IsNaN(curr) || math.IsNaN(prev) {
-		return nan, curr
+	if math.IsNaN(currV) || math.IsNaN(prevV) {
+		return nan, currV
 	}
-	if curr >= prev {
-		return curr - prev, curr
+	if currV >= prevV {
+		return currV - prevV, currV
 	}
-	if !math.IsNaN(max) {
-		if math.IsNaN(min) {
-			min = float64(0)
+	if !math.IsNaN(maxV) {
+		if math.IsNaN(minV) {
+			minV = float64(0)
 		}
-		return max + 1 + curr - prev - min, curr
+		return maxV + 1 + currV - prevV - minV, currV
 	}
-	if !math.IsNaN(min) {
-		return curr - min, curr
+	if !math.IsNaN(minV) {
+		return currV - minV, currV
 	}
-	return nan, curr
+	return nan, currV
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.threshold
@@ -4941,8 +4941,8 @@ func transformSortByMinima(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesF
 	}
 	// Filter out series with all the values smaller than 0
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
-		max := aggrMax(s.Values)
-		if math.IsNaN(max) || max <= 0 {
+		maxV := aggrMax(s.Values)
+		if math.IsNaN(maxV) || maxV <= 0 {
 			return nil, nil
 		}
 		return s, nil

app/vmselect/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/stats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -29,13 +30,16 @@ import (
 )
 
 var (
-	deleteAuthKey         = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It overrides -httpAuth.*")
+	deleteAuthKey                = flagutil.NewPassword("deleteAuthKey", "authKey for metrics' deletion via /api/v1/admin/tsdb/delete_series and /tags/delSeries. It could be passed via authKey query arg. It overrides -httpAuth.*")
+	metricNamesStatsResetAuthKey = flagutil.NewPassword("metricNamesStatsResetAuthKey", "authKey for reseting metric names usage cache via /api/v1/admin/status/metric_names_stats/reset. It overrides -httpAuth.*. "+
+		"See https://docs.victoriametrics.com/#track-ingested-metrics-usage")
+
 	maxConcurrentRequests = flag.Int("search.maxConcurrentRequests", getDefaultMaxConcurrentRequests(), "The maximum number of concurrent search requests. "+
 		"It shouldn't be high, since a single request can saturate all the CPU cores, while many concurrently executed requests may require high amounts of memory. "+
 		"See also -search.maxQueueDuration and -search.maxMemoryPerQuery")
 	maxQueueDuration = flag.Duration("search.maxQueueDuration", 10*time.Second, "The maximum time the request waits for execution when -search.maxConcurrentRequests "+
 		"limit is reached; see also -search.maxQueryDuration")
-	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It overrides -httpAuth.*")
+	resetCacheAuthKey    = flagutil.NewPassword("search.resetCacheAuthKey", "Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call. It could be passed via authKey query arg. It overrides -httpAuth.*")
 	logSlowQueryDuration = flag.Duration("search.logSlowQueryDuration", 5*time.Second, "Log queries with execution time exceeding this value. Zero disables slow query logging. "+
 		"See also -search.logQueryMemoryUsage")
 	vmalertProxyURL = flag.String("vmalert.proxyURL", "", "Optional URL for proxying requests to vmalert. For example, if -vmalert.proxyURL=http://vmalert:8880 , then alerting API requests such as /api/v1/rules from Grafana will be proxied to http://vmalert:8880/api/v1/rules")
@@ -178,7 +182,6 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		promql.ResetRollupResultCache()
 		return true
 	}
-
 	if strings.HasPrefix(path, "/api/v1/label/") {
 		s := path[len("/api/v1/label/"):]
 		if strings.HasSuffix(s, "/values") {
@@ -399,6 +402,26 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		w.WriteHeader(http.StatusNoContent)
 		return true
+	case "/api/v1/status/metric_names_stats":
+		metricNamesStatsRequests.Inc()
+		if err := stats.MetricNamesStatsHandler(qt, w, r); err != nil {
+			metricNamesStatsErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		return true
+	case "/api/v1/admin/status/metric_names_stats/reset":
+		metricNamesStatsResetRequests.Inc()
+		if !httpserver.CheckAuthFlag(w, r, metricNamesStatsResetAuthKey) {
+			return true
+		}
+		if err := stats.ResetMetricNamesStatsHandler(qt); err != nil {
+			metricNamesStatsResetErrors.Inc()
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.WriteHeader(http.StatusNoContent)
+		return true
 	default:
 		return false
 	}
@@ -674,6 +697,12 @@ var (
 	metadataRequests       = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`)
 	buildInfoRequests      = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/buildinfo"}`)
 	queryExemplarsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/query_exemplars"}`)
+
+	metricNamesStatsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/status/metric_names_stats"}`)
+	metricNamesStatsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/status/metric_names_stats"}`)
+
+	metricNamesStatsResetRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
+	metricNamesStatsResetErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/api/v1/admin/status/metric_names_stats/reset"}`)
 )
 
 func proxyVMAlertRequests(w http.ResponseWriter, r *http.Request) {

app/vmselect/netstorage/netstorage.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"sync"
 	"sync/atomic"
-	"time"
 	"unsafe"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -792,7 +791,7 @@ func LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames i
 	if err != nil {
 		return nil, err
 	}
-	labels, err := vmstorage.SearchLabelNamesWithFiltersOnTimeRange(qt, tfss, tr, maxLabelNames, sq.MaxMetrics, deadline.Deadline())
+	labels, err := vmstorage.SearchLabelNames(qt, tfss, tr, maxLabelNames, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search on time range: %w", err)
 	}
@@ -865,7 +864,7 @@ func LabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQue
 	if err != nil {
 		return nil, err
 	}
-	labelValues, err := vmstorage.SearchLabelValuesWithFiltersOnTimeRange(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
+	labelValues, err := vmstorage.SearchLabelValues(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label values search on time range for labelName=%q: %w", labelName, err)
 	}
@@ -1002,9 +1001,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 
 	sr := getStorageSearch()
 	defer putStorageSearch(sr)
-	startTime := time.Now()
 	sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
-	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
 	workCh := make(chan *exportWork, gomaxprocs*8)
@@ -1142,9 +1139,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
-	startTime := time.Now()
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
-	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
 		brs []blockRef
 	}
@@ -1296,8 +1291,6 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	return &rss, nil
 }
 
-var indexSearchDuration = metrics.NewHistogram(`vm_index_search_duration_seconds`)
-
 type blockRef struct {
 	partRef storage.PartRef
 	addr    tmpBlockAddr
@@ -1374,3 +1367,18 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 //
 // See https://github.com/golang/go/blob/704401ffa06c60e059c9e6e4048045b4ff42530a/src/runtime/malloc.go#L11
 const maxFastAllocBlockSize = 32 * 1024
+
+// GetMetricNamesStats returns statistic for timeseries metric names usage.
+func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
+	qt = qt.NewChild("get metric names usage statistics with limit: %d, less or equal to: %d, match pattern=%q", limit, le, matchPattern)
+	defer qt.Done()
+	return vmstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+}
+
+// ResetMetricNamesStats resets state of metric names usage
+func ResetMetricNamesStats(qt *querytracer.Tracer) error {
+	qt = qt.NewChild("reset metric names usage stats")
+	defer qt.Done()
+	vmstorage.ResetMetricNamesStats(qt)
+	return nil
+}

app/vmselect/prometheus/prometheus.go

@@ -29,6 +29,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -38,7 +39,7 @@ var (
 		"It can be overridden on per-query basis via latency_offset arg. "+
 		"Too small value can result in incomplete last points for query results")
 	maxQueryLen = flagutil.NewBytes("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
-	maxLookback = flag.Duration("search.maxLookback", 0, "Synonym to -search.lookback-delta from Prometheus. "+
+	maxLookback = flag.Duration("search.maxLookback", 0, "Synonym to -query.lookback-delta from Prometheus. "+
 		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+
 		"See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons")
 	maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+
@@ -142,10 +143,13 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
 		WriteFederate(bb, rs)
 		return sw.maybeFlushBuffer(bb)
 	})
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("error during sending data to remote client: %w", err)
 	}
-	return sw.flush()
+	return nil
 }
 
 var federateDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/federate"}`)
@@ -226,10 +230,13 @@ func ExportCSVHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 		}()
 	}
 	err = <-doneCh
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("error during sending the exported csv data to remote client: %w", err)
 	}
-	return sw.flush()
+	return nil
 }
 
 var exportCSVDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/csv"}`)
@@ -281,10 +288,13 @@ func ExportNativeHandler(startTime time.Time, w http.ResponseWriter, r *http.Req
 		bb.B = dst
 		return sw.maybeFlushBuffer(bb)
 	})
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("error during sending native data to remote client: %w", err)
 	}
-	return sw.flush()
+	return nil
 }
 
 var exportNativeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/export/native"}`)
@@ -441,16 +451,19 @@ func exportHandler(qt *querytracer.Tracer, w http.ResponseWriter, cp *commonPara
 		}()
 	}
 	err := <-doneCh
-	if err != nil {
+	if err == nil {
+		err = sw.flush()
+	}
+	if err == nil {
+		if format == "promapi" {
+			WriteExportPromAPIFooter(bw, qt)
+		}
+		err = bw.Flush()
+	}
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		return fmt.Errorf("cannot send data to remote client: %w", err)
 	}
-	if err := sw.flush(); err != nil {
-		return fmt.Errorf("cannot send data to remote client: %w", err)
-	}
-	if format == "promapi" {
-		WriteExportPromAPIFooter(bw, qt)
-	}
-	return bw.Flush()
+	return nil
 }
 
 type exportBlock struct {
@@ -481,6 +494,8 @@ func DeleteHandler(startTime time.Time, r *http.Request) error {
 	if err != nil {
 		return err
 	}
+	cp.deadline = searchutils.GetDeadlineForDelete(r, startTime)
+
 	if !cp.IsDefaultTimeRange() {
 		return fmt.Errorf("start=%d and end=%d args aren't supported. Remove these args from the query in order to delete all the matching metrics", cp.start, cp.end)
 	}

app/vmselect/promql/aggr.go

@@ -295,13 +295,13 @@ func aggrFuncMin(tss []*timeseries) []*timeseries {
 	}
 	dst := tss[0]
 	for i := range dst.Values {
-		min := dst.Values[i]
+		minV := dst.Values[i]
 		for _, ts := range tss {
-			if math.IsNaN(min) || ts.Values[i] < min {
-				min = ts.Values[i]
+			if math.IsNaN(minV) || ts.Values[i] < minV {
+				minV = ts.Values[i]
 			}
 		}
-		dst.Values[i] = min
+		dst.Values[i] = minV
 	}
 	return tss[:1]
 }
@@ -313,13 +313,13 @@ func aggrFuncMax(tss []*timeseries) []*timeseries {
 	}
 	dst := tss[0]
 	for i := range dst.Values {
-		max := dst.Values[i]
+		maxV := dst.Values[i]
 		for _, ts := range tss {
-			if math.IsNaN(max) || ts.Values[i] > max {
-				max = ts.Values[i]
+			if math.IsNaN(maxV) || ts.Values[i] > maxV {
+				maxV = ts.Values[i]
 			}
 		}
-		dst.Values[i] = max
+		dst.Values[i] = maxV
 	}
 	return tss[:1]
 }
@@ -793,7 +793,7 @@ func fillNaNsAtIdx(idx int, k float64, tss []*timeseries) {
 	}
 }
 
-func getIntK(k float64, max int) int {
+func getIntK(k float64, maxV int) int {
 	if math.IsNaN(k) {
 		return 0
 	}
@@ -801,38 +801,38 @@ func getIntK(k float64, max int) int {
 	if kn < 0 {
 		return 0
 	}
-	if kn > max {
-		return max
+	if kn > maxV {
+		return maxV
 	}
 	return kn
 }
 
 func minValue(values []float64) float64 {
-	min := nan
-	for len(values) > 0 && math.IsNaN(min) {
-		min = values[0]
+	minV := nan
+	for len(values) > 0 && math.IsNaN(minV) {
+		minV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if !math.IsNaN(v) && v < min {
-			min = v
+		if !math.IsNaN(v) && v < minV {
+			minV = v
 		}
 	}
-	return min
+	return minV
 }
 
 func maxValue(values []float64) float64 {
-	max := nan
-	for len(values) > 0 && math.IsNaN(max) {
-		max = values[0]
+	maxV := nan
+	for len(values) > 0 && math.IsNaN(maxV) {
+		maxV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if !math.IsNaN(v) && v > max {
-			max = v
+		if !math.IsNaN(v) && v > maxV {
+			maxV = v
 		}
 	}
-	return max
+	return maxV
 }
 
 func avgValue(values []float64) float64 {

app/vmselect/promql/binary_op.go

@@ -483,8 +483,11 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries
 
-	for _, tss := range mLeft {
-		rvs = append(rvs, tss...)
+	for k, tss := range mLeft {
+		tssLeft := removeEmptySeries(tss)
+		// re-assign modified slice to map, since it can be referred later
+		mLeft[k] = tssLeft
+		rvs = append(rvs, tssLeft...)
 	}
 	// Sort left-hand-side series by metric name as Prometheus does.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5393
@@ -497,7 +500,10 @@ func binaryOpOr(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 			rvs = append(rvs, tssRight...)
 			continue
 		}
-		fillLeftNaNsWithRightValues(tssLeft, tssRight)
+		fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight)
+		// tssRight might be filled with NaNs after merge
+		tssRight = removeEmptySeries(tssRight)
+		rvs = append(rvs, tssRight...)
 	}
 	// Sort the added right-hand-side series by metric name as Prometheus does.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5393
@@ -526,6 +532,35 @@ func fillLeftNaNsWithRightValues(tssLeft, tssRight []*timeseries) {
 	}
 }
 
+// fill gaps in tssLeft with values from tssRight when labels match
+// Set NaNs to tssRight when tssLeft has corresponding values
+// or if tssLeft and tssRight can be merged.
+//
+// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759
+// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640
+func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
+	for _, tsLeft := range tssLeft {
+		valuesLeft := tsLeft.Values
+		nameLeft := tsLeft.MetricName.String()
+		for i, v := range valuesLeft {
+			leftIsNaN := math.IsNaN(v)
+			for _, tsRight := range tssRight {
+				canBeMerged := nameLeft == tsRight.MetricName.String()
+				valueRight := tsRight.Values[i]
+				if leftIsNaN && canBeMerged {
+					// fill NaNs with valueRight if labels match
+					valuesLeft[i] = valueRight
+				}
+				if !leftIsNaN || canBeMerged {
+					// set NaN to valueRight if valueLeft is not NaN
+					// or if left and right can be merged
+					tsRight.Values[i] = nan
+				}
+			}
+		}
+	}
+}
+
 func binaryOpIfnot(bfa *binaryOpFuncArg) ([]*timeseries, error) {
 	mLeft, mRight := createTimeseriesMapByTagSet(bfa.be, bfa.left, bfa.right)
 	var rvs []*timeseries

app/vmselect/promql/eval.go

@@ -12,6 +12,9 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -25,8 +28,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
 var (
@@ -814,7 +815,19 @@ func evalRollupFunc(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf
 			Err: fmt.Errorf("`@` modifier must return a single series; it returns %d series instead", len(tssAt)),
 		}
 	}
-	atTimestamp := int64(tssAt[0].Values[0] * 1000)
+	atValue := math.NaN()
+	for _, v := range tssAt[0].Values {
+		if !math.IsNaN(v) {
+			atValue = v
+			break
+		}
+	}
+	if math.IsNaN(atValue) {
+		return nil, &httpserver.UserReadableError{
+			Err: fmt.Errorf("`@` modifier must return a non-NaN value"),
+		}
+	}
+	atTimestamp := int64(atValue * 1000)
 	ecNew := copyEvalConfig(ec)
 	ecNew.Start = atTimestamp
 	ecNew.End = atTimestamp

app/vmselect/promql/exec_test.go

@@ -4461,9 +4461,9 @@ func TestExecSuccess(t *testing.T) {
 	t.Run(`histogram_quantile(nan-bucket-count-some)`, func(t *testing.T) {
 		t.Parallel()
 		q := `round(histogram_quantile(0.6,
-			label_set(90, "foo", "bar", "le", "10")
-			or label_set(NaN, "foo", "bar", "le", "30")
-			or label_set(300, "foo", "bar", "le", "+Inf")
+			union(label_set(90, "foo", "bar", "le", "10"),
+			label_set(NaN, "foo", "bar", "le", "30"),
+			label_set(300, "foo", "bar", "le", "+Inf"))
 		),0.01)`
 		r := netstorage.Result{
 			MetricName: metricNameExpected,
@@ -9409,7 +9409,384 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run(`nan or on() series`, func(t *testing.T) {
+		t.Parallel()
+		// left side returns NaNs only, so the right side should replace its values and labels
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759
+		q := `(label_set(1, "a", "a", "b", "b1") == 0) or on(a) label_set(2, "a", "a", "b", "b2")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b2"),
+		}}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`series with NaNs or scalar`, func(t *testing.T) {
+		t.Parallel()
+		q := `(label_set(time() >= 1600, "a", "a", "b", "b1")) or 1`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
 
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() scalar`, func(t *testing.T) {
+		t.Parallel()
+		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640
+		q := `(label_set(time() > 1200, "a", "a", "b", "b1")) or on() vector(0)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() series`, func(t *testing.T) {
+		t.Parallel()
+		// left side + right side
+		q := `(label_set(time() <= 1200, "a", "a", "b", "b1")) or on(a) label_set(time() > 1200, "a", "a", "b", "b2")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b2"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series with no NaNs or on() series`, func(t *testing.T) {
+		t.Parallel()
+		// left side contains all needed values, so the right side should be dropped
+		q := `(label_set(time() < 3000, "a", "a", "b", "b1")) or on(a) label_set(time() > 3000, "a", "a", "b", "b2")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() series with overlap`, func(t *testing.T) {
+		t.Parallel()
+		// left overlap with right
+		q := `(label_set(time() <= 1500, "a", "a", "b", "b1")) or on(a) label_set(time() > 1100, "a", "a", "b", "b2")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b2"),
+		}}
+
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or on() series merge`, func(t *testing.T) {
+		t.Parallel()
+		// left + right for same series
+		q := `(label_set(time() <= 1200, "a", "a", "b", "b1")) or on(a) label_set(time() > 1400, "a", "a", "b", "b1")`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("a"),
+			Value: []byte("a"),
+		}, {
+			Key:   []byte("b"),
+			Value: []byte("b1"),
+		}}
+
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run(`scalar or timeseries`, func(t *testing.T) {
+		t.Parallel()
+		q := `time() > 1400 or label_set(123, "foo", "bar")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{123, 123, 123, 123, 123, 123},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{{
+			Key:   []byte("foo"),
+			Value: []byte("bar"),
+		}}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`series or many series`, func(t *testing.T) {
+		//load 1m
+		//    foo{a="a", b="1"} 1 0 1 1 1
+		//    bar{a="a", b="2"} 2 2 2 2 2
+		//    bar{a="a", b="3"} 3 3 3 3 3
+		//
+		//eval range from 0 to 4m step 1m foo!=0 or on (a) bar
+		//    foo{a="a", b="1"} 1 _ 1 1 1
+		//    bar{a="a", b="2"} _ 2 _ _ _
+		//    bar{a="a", b="3"} _ 3 _ _ _
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1200, "x", "foo"),
+		) or on(x) (
+			label_set(time()+1, "x", "foo", "y", "bar"),
+			label_set(time()+2, "y", "baz", "x", "foo"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1201, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+			{Key: []byte("y"), Value: []byte("bar")},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1202, nan, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+			{Key: []byte("y"), Value: []byte("baz")},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3}
+		f(q, resultExpected)
+	})
+	t.Run(`many series or series`, func(t *testing.T) {
+		//load 1m
+		//    foo{a="a", b="1"} 1 0 1 1 1
+		//    foo{a="a", b="2"} 2 2 2 2 2
+		//    bar{a="a", b="3"} 3 3 3 3 3
+		//
+		//eval range from 0 to 4m step 1m foo!=0 or on (a) bar
+		//    foo{a="a", b="1"} 1 _ 1 1 1
+		//    foo{a="a", b="2"} 2 2 2 2 2
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1200, "x", "foo"),
+			label_set(time()+1, "x", "foo", "y","baz"),
+		) or on(x) (
+			label_set(time()+2, "x", "foo", "y", "bar"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1001, 1201, 1401, 1601, 1801, 2001},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("x"), Value: []byte("foo")},
+			{Key: []byte("y"), Value: []byte("baz")},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`many series or series with no merge`, func(t *testing.T) {
+		//	load 1m
+		//    foo{job="a1", a="a"} 0 0 1 1 0
+		//    foo{job="a2", a="a"} 1 1 0 0 0
+		//    foo{job="a3", a="a"} 1 1 1 1 1
+		//    foo{job="a4", a="a"} 1 1 1 1 1
+		//
+		//eval range from 0 to 4m step 1m (foo{job=~"a1|a2"} == 0) or on (a) (foo{job=~"a3|a4"} == 1)
+		//    foo{job="a1", a="a"} 0 0 _ _ 0
+		//    foo{job="a2", a="a"} _ _ 0 0 0
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1400, "job", "a1", "a", "a"),
+			label_set(time()>=1400, "job", "a2", "a", "a"),
+		) or on(a) (
+			label_set(time(), "job", "a3", "a", "a"),
+			label_set(time(), "job", "a4", "a", "a"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a1")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a2")},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
+	t.Run(`many series or series with merge`, func(t *testing.T) {
+		//	load 1m
+		//    foo{job="a1", a="a"} 0 0 1 1 0
+		//    foo{job="a2", a="a"} 1 1 1 0 0
+		//    foo{job="a3", a="a"} 1 1 1 1 1
+		//    foo{job="a4", a="a"} 1 1 1 1 1
+		//
+		//eval range from 0 to 4m step 1m (foo{job=~"a1|a2"} == 0) or on (a) (foo{job=~"a3|a4"} == 1)
+		//    foo{job="a1", a="a"} 0 0 _ _ 0
+		//    foo{job="a2", a="a"} _ _ _ 0 0
+		//    foo{job="a3", a="a"} _ _ 1 _ _
+		//    foo{job="a4", a="a"} _ _ 1 _ _
+		// https://github.com/prometheus/prometheus/tree/main/promql/promqltest
+		t.Parallel()
+		q := `(
+			label_set(time()!=1400, "job", "a1", "a", "a"),
+			label_set(time()>=1600, "job", "a2", "a", "a"),
+		) or on(a) (
+			label_set(time(), "job", "a3", "a", "a"),
+			label_set(time(), "job", "a4", "a", "a"),
+		)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1000, 1200, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a1")},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a2")},
+		}
+		r3 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r3.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a3")},
+		}
+		r4 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, nan, 1400, nan, nan, nan},
+			Timestamps: timestampsExpected,
+		}
+		r4.MetricName.Tags = []storage.Tag{
+			{Key: []byte("a"), Value: []byte("a")},
+			{Key: []byte("job"), Value: []byte("a4")},
+		}
+		resultExpected := []netstorage.Result{r1, r2, r3, r4}
+		f(q, resultExpected)
+	})
 }
 
 func TestExecError(t *testing.T) {

app/vmselect/promql/rollup.go

@@ -373,9 +373,17 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 	func(values []float64, timestamps []int64), []*rollupConfig, error) {
 	preFunc := func(_ []float64, _ []int64) {}
 	funcName = strings.ToLower(funcName)
+
+	// window > lookbackDelta could result in negative delta.
+	// See issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8342
+	stalenessInterval := lookbackDelta
+	if stalenessInterval != 0 && stalenessInterval < window {
+		stalenessInterval = window
+	}
+
 	if rollupFuncsRemoveCounterResets[funcName] {
-		preFunc = func(values []float64, _ []int64) {
-			removeCounterResets(values)
+		preFunc = func(values []float64, timestamps []int64) {
+			removeCounterResets(values, timestamps, stalenessInterval)
 		}
 	}
 	samplesScannedPerCall := rollupFuncsSamplesScannedPerCall[funcName]
@@ -486,8 +494,8 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 		for _, aggrFuncName := range aggrFuncNames {
 			if rollupFuncsRemoveCounterResets[aggrFuncName] {
 				// There is no need to save the previous preFunc, since it is either empty or the same.
-				preFunc = func(values []float64, _ []int64) {
-					removeCounterResets(values)
+				preFunc = func(values []float64, timestamps []int64) {
+					removeCounterResets(values, timestamps, stalenessInterval)
 				}
 			}
 			rf := rollupAggrFuncs[aggrFuncName]
@@ -521,7 +529,7 @@ type rollupFuncArg struct {
 	timestamps []int64
 
 	// Real value preceding values.
-	// Is populated if preceding value is within the staleness interval.
+	// Is populated if preceding value is within the -search.maxStalenessInterval (rc.LookbackDelta).
 	realPrevValue float64
 
 	// Real value which goes after values.
@@ -768,7 +776,13 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 		rfa.realPrevValue = nan
 		if i > 0 {
 			prevValue, prevTimestamp := values[i-1], timestamps[i-1]
-			if (tEnd - prevTimestamp) < maxPrevInterval {
+			// set realPrevValue if rc.LookbackDelta == 0
+			// or if distance between datapoint in prev interval and beginning of this interval
+			// doesn't exceed LookbackDelta.
+			// https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1381
+			// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/894
+			// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8045
+			if rc.LookbackDelta == 0 || (tStart-prevTimestamp) < rc.LookbackDelta {
 				rfa.realPrevValue = prevValue
 			}
 		}
@@ -894,7 +908,7 @@ func getMaxPrevInterval(scrapeInterval int64) int64 {
 	return scrapeInterval + scrapeInterval/8
 }
 
-func removeCounterResets(values []float64) {
+func removeCounterResets(values []float64, timestamps []int64, maxStalenessInterval int64) {
 	// There is no need in handling NaNs here, since they are impossible
 	// on values from vmstorage.
 	if len(values) == 0 {
@@ -913,6 +927,16 @@ func removeCounterResets(values []float64) {
 				correction += prevValue
 			}
 		}
+		if i > 0 && maxStalenessInterval > 0 {
+			gap := timestamps[i] - timestamps[i-1]
+			if gap > maxStalenessInterval {
+				// reset correction if gap between samples exceeds staleness interval
+				// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072
+				correction = 0
+				prevValue = v
+				continue
+			}
+		}
 		prevValue = v
 		values[i] = v + correction
 		// Check again, there could be precision error in float operations,
@@ -1685,9 +1709,9 @@ func rollupRateOverSum(rfa *rollupFuncArg) float64 {
 }
 
 func rollupRange(rfa *rollupFuncArg) float64 {
-	max := rollupMax(rfa)
-	min := rollupMin(rfa)
-	return max - min
+	maxV := rollupMax(rfa)
+	minV := rollupMin(rfa)
+	return maxV - minV
 }
 
 func rollupSum2(rfa *rollupFuncArg) float64 {
@@ -2195,38 +2219,38 @@ func rollupClose(rfa *rollupFuncArg) float64 {
 
 func rollupHigh(rfa *rollupFuncArg) float64 {
 	values := getCandlestickValues(rfa)
-	max := getFirstValueForCandlestick(rfa)
-	if math.IsNaN(max) {
+	maxV := getFirstValueForCandlestick(rfa)
+	if math.IsNaN(maxV) {
 		if len(values) == 0 {
 			return nan
 		}
-		max = values[0]
+		maxV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if v > max {
-			max = v
+		if v > maxV {
+			maxV = v
 		}
 	}
-	return max
+	return maxV
 }
 
 func rollupLow(rfa *rollupFuncArg) float64 {
 	values := getCandlestickValues(rfa)
-	min := getFirstValueForCandlestick(rfa)
-	if math.IsNaN(min) {
+	minV := getFirstValueForCandlestick(rfa)
+	if math.IsNaN(minV) {
 		if len(values) == 0 {
 			return nan
 		}
-		min = values[0]
+		minV = values[0]
 		values = values[1:]
 	}
 	for _, v := range values {
-		if v < min {
-			min = v
+		if v < minV {
+			minV = v
 		}
 	}
-	return min
+	return minV
 }
 
 func rollupModeOverTime(rfa *rollupFuncArg) float64 {

app/vmselect/promql/rollup_test.go

@@ -117,31 +117,49 @@ func TestRollupIderivDuplicateTimestamps(t *testing.T) {
 }
 
 func TestRemoveCounterResets(t *testing.T) {
-	removeCounterResets(nil)
+	removeCounterResets(nil, nil, 0)
 
 	values := append([]float64{}, testValues...)
-	removeCounterResets(values)
+	timestamps := append([]int64{}, testTimestamps...)
+	removeCounterResets(values, timestamps, 0)
 	valuesExpected := []float64{123, 157, 167, 188, 221, 255, 320, 332, 364, 396, 398, 398}
 	testRowsEqual(t, values, testTimestamps, valuesExpected, testTimestamps)
 
 	// removeCounterResets doesn't expect negative values, so it doesn't work properly with them.
 	values = []float64{-100, -200, -300, -400}
-	removeCounterResets(values)
-	valuesExpected = []float64{-100, -100, -100, -100}
 	timestampsExpected := []int64{0, 1, 2, 3}
+	removeCounterResets(values, timestampsExpected, 0)
+	valuesExpected = []float64{-100, -100, -100, -100}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
 	// verify how partial counter reset is handled.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2787
 	values = []float64{100, 95, 120, 119, 139, 50}
-	removeCounterResets(values)
-	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	timestampsExpected = []int64{0, 1, 2, 3, 4, 5}
+	removeCounterResets(values, timestampsExpected, 0)
+	valuesExpected = []float64{100, 100, 125, 125, 145, 195}
 	testRowsEqual(t, values, timestampsExpected, valuesExpected, timestampsExpected)
 
+	// verify that staleness interval is respected during resets
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8072
+	values = []float64{10, 12, 14, 4, 6, 8, 6, 8, 4, 6}
+	timestamps = []int64{10, 20, 30, 60, 70, 80, 90, 100, 120, 130}
+	valuesExpected = []float64{10, 12, 14, 4, 6, 8, 14, 16, 4, 6}
+	removeCounterResets(values, timestamps, 10)
+	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
+
+	// verify that staleness is respected if there was no counter reset
+	// but correction was made previously
+	values = []float64{10, 12, 2, 4}
+	timestamps = []int64{10, 20, 30, 60}
+	valuesExpected = []float64{10, 12, 14, 4}
+	removeCounterResets(values, timestamps, 10)
+	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
+
 	// verify results always increase monotonically with possible float operations precision error
 	values = []float64{34.094223, 2.7518, 2.140669, 0.044878, 1.887095, 2.546569, 2.490149, 0.045, 0.035684, 0.062454, 0.058296}
-	removeCounterResets(values)
+	timestampsExpected = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	removeCounterResets(values, timestampsExpected, 0)
 	var prev float64
 	for i, v := range values {
 		if v < prev {
@@ -166,7 +184,7 @@ func TestDeltaValues(t *testing.T) {
 
 	// remove counter resets
 	values = append([]float64{}, testValues...)
-	removeCounterResets(values)
+	removeCounterResets(values, testTimestamps, 0)
 	deltaValues(values)
 	valuesExpected = []float64{34, 10, 21, 33, 34, 65, 12, 32, 32, 2, 0, 0}
 	testRowsEqual(t, values, testTimestamps, valuesExpected, testTimestamps)
@@ -188,7 +206,7 @@ func TestDerivValues(t *testing.T) {
 
 	// remove counter resets
 	values = append([]float64{}, testValues...)
-	removeCounterResets(values)
+	removeCounterResets(values, testTimestamps, 0)
 	derivValues(values, testTimestamps)
 	valuesExpected = []float64{3400, 1111.111111111111, 1750, 2538.4615384615386, 3090.909090909091, 3611.1111111111113,
 		6000, 1882.3529411764705, 1777.7777777777778, 400, 0, 0}
@@ -219,7 +237,7 @@ func testRollupFunc(t *testing.T, funcName string, args []any, vExpected float64
 	rfa.timestamps = append(rfa.timestamps, testTimestamps...)
 	rfa.window = rfa.timestamps[len(rfa.timestamps)-1] - rfa.timestamps[0]
 	if rollupFuncsRemoveCounterResets[funcName] {
-		removeCounterResets(rfa.values)
+		removeCounterResets(rfa.values, rfa.timestamps, 0)
 	}
 	for i := 0; i < 5; i++ {
 		v := rf(&rfa)
@@ -1590,17 +1608,60 @@ func TestRollupDelta(t *testing.T) {
 	f(100, nan, nan, nil, 0)
 }
 
-func TestRollupIncreaseWithStaleness(t *testing.T) {
+func TestRollupDeltaWithStaleness(t *testing.T) {
 	// there is a gap between samples in the dataset below
 	timestamps := []int64{0, 15000, 30000, 70000}
 	values := []float64{1, 1, 1, 1}
 
-	t.Run("step > gap", func(t *testing.T) {
+	// if step > gap, then delta will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:               rollupDelta,
 			Start:              0,
 			End:                70000,
-			Step:               35000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDelta,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta==0 then delta will always respect value before gap
+	// as LookbackDelta=0 ignores staleness
+	t.Run("step<gap;LookbackDelta=0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupDelta,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			LookbackDelta:      0,
 			Window:             0,
 			MaxPointsPerSeries: 1e4,
 		}
@@ -1609,12 +1670,14 @@ func TestRollupIncreaseWithStaleness(t *testing.T) {
 		if samplesScanned != 8 {
 			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
 		}
-		valuesExpected := []float64{1, 0, 0}
-		timestampsExpected := []int64{0, 35e3, 70e3}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 
-	t.Run("step < gap", func(t *testing.T) {
+	// if step < gap and LookbackDelta>0 then delta will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
 		rc := rollupConfig{
 			Func:               rollupDelta,
 			Start:              0,
@@ -1622,6 +1685,7 @@ func TestRollupIncreaseWithStaleness(t *testing.T) {
 			Step:               10000,
 			Window:             0,
 			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
 		}
 		rc.Timestamps = rc.getTimestamps()
 		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
@@ -1656,3 +1720,116 @@ func TestRollupIncreaseWithStaleness(t *testing.T) {
 		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
 	})
 }
+
+func TestRollupIncreasePureWithStaleness(t *testing.T) {
+	// there is a gap between samples in the dataset below
+	timestamps := []int64{0, 15000, 30000, 70000}
+	values := []float64{1, 1, 1, 1}
+
+	// if step > gap, then delta will always respect value before gap
+	t.Run("step>gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+	// even if LookbackDelta < gap
+	t.Run("step>gap;LookbackDelta<gap", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               45000,
+			LookbackDelta:      10e3,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 7 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0}
+		timestampsExpected := []int64{0, 45e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta==0 then delta will always respect value before gap
+	// as LookbackDelta=0 ignores staleness
+	t.Run("step<gap;LookbackDelta=0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			LookbackDelta:      0,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 0}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// if step < gap and LookbackDelta>0 then delta will respect value before gap
+	// only if it is not stale according to LookbackDelta
+	t.Run("step<gap;LookbackDelta>0", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                70000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+			LookbackDelta:      30e3,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 8 {
+			t.Fatalf("expecting 8 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, 0, 0, 0, 0, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3, 50e3, 60e3, 70e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+
+	// there is a staleness marker between samples in the dataset below
+	timestamps = []int64{0, 10000, 20000, 30000, 40000}
+	values = []float64{1, 1, 1, decimal.StaleNaN, 1}
+
+	t.Run("staleness marker", func(t *testing.T) {
+		rc := rollupConfig{
+			Func:               rollupIncreasePure,
+			Start:              0,
+			End:                40000,
+			Step:               10000,
+			Window:             0,
+			MaxPointsPerSeries: 1e4,
+		}
+		rc.Timestamps = rc.getTimestamps()
+		gotValues, samplesScanned := rc.Do(nil, values, timestamps)
+		if samplesScanned != 10 {
+			t.Fatalf("expecting 10 samplesScanned from rollupConfig.Do; got %d", samplesScanned)
+		}
+		valuesExpected := []float64{1, 0, 0, nan, 1}
+		timestampsExpected := []int64{0, 10e3, 20e3, 30e3, 40e3}
+		testRowsEqual(t, gotValues, rc.Timestamps, valuesExpected, timestampsExpected)
+	})
+}

app/vmselect/searchutils/searchutils.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	maxExportDuration        = flag.Duration("search.maxExportDuration", time.Hour*24*30, "The maximum duration for /api/v1/export call")
+	maxDeleteDuration        = flag.Duration("search.maxDeleteDuration", time.Minute*5, "The maximum duration for /api/v1/admin/tsdb/delete_series call")
 	maxQueryDuration         = flag.Duration("search.maxQueryDuration", time.Second*30, "The maximum duration for query execution. It can be overridden to a smaller value on a per-query basis via 'timeout' query arg")
 	maxStatusRequestDuration = flag.Duration("search.maxStatusRequestDuration", time.Minute*5, "The maximum duration for /api/v1/status/* requests")
 	maxLabelsAPIDuration     = flag.Duration("search.maxLabelsAPIDuration", time.Second*5, "The maximum duration for /api/v1/labels, /api/v1/label/.../values and /api/v1/series requests. "+
@@ -58,6 +59,12 @@ func GetDeadlineForLabelsAPI(r *http.Request, startTime time.Time) Deadline {
 	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxLabelsAPIDuration")
 }
 
+// GetDeadlineForDelete returns deadline for the given request to /api/v1/admin/tsdb/delete_series.
+func GetDeadlineForDelete(r *http.Request, startTime time.Time) Deadline {
+	dMax := maxDeleteDuration.Milliseconds()
+	return getDeadlineWithMaxDuration(r, startTime, dMax, "-search.maxDeleteDuration")
+}
+
 func getDeadlineWithMaxDuration(r *http.Request, startTime time.Time, dMax int64, flagHint string) Deadline {
 	d, err := httputils.GetDuration(r, "timeout", 0)
 	if err != nil {

app/vmselect/searchutils/searchutils_test.go

@@ -99,7 +99,8 @@ func TestParseMetricSelectorSuccess(t *testing.T) {
 	f(`{foo="bar"}`)
 	f(`{:f:oo=~"bar.+"}`)
 	f(`foo {bar != "baz"}`)
-	f(` foo { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"}  `)
+	f(` { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"}  `)
+	f(` { bar !~ "^ddd(x+)$", a="ss", "foo"}  `)
 	f(`(foo)`)
 	f(`\п\р\и\в\е\т{\ы="111"}`)
 }

app/vmselect/stats/metric_names_usage_response.qtpl

@@ -0,0 +1,33 @@
+{% import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+) %}
+
+{% stripspace %}
+MetricNamesStatsResponse generates response for /api/v1/status/metric_names_stats .
+{% func MetricNamesStatsResponse(stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) %}
+{
+	"status":"success",
+  "statsCollectedSince": {%dul= stats.CollectedSinceTs %},
+  "statsCollectedRecordsTotal": {%dul= stats.TotalRecords %},
+  "trackerMemoryMaxSizeBytes": {%dul= stats.MaxSizeBytes %},
+  "trackerCurrentMemoryUsageBytes": {%dul= stats.CurrentSizeBytes %},
+  "records":
+  [
+    {% for i, r := range stats.Records %}
+      {
+        "metricName":{%q= r.MetricName %},
+        "queryRequestsCount":{%dul= r.RequestsCount %},
+        "lastQueryRequestTimestamp":{%dul= r.LastRequestTs %}
+      }
+      {% if i+1 < len(stats.Records) %},{% endif %}
+    {% endfor %}
+  ]
+	{% code	qt.Done() %}
+	{% code	traceJSON := qt.ToJSON() %}
+	{% if traceJSON != "" %},"trace":{%s= traceJSON %}{% endif %}
+
+}
+{% endfunc %}
+
+{% endstripspace %}

app/vmselect/stats/metric_names_usage_response.qtpl.go

@@ -0,0 +1,117 @@
+// Code generated by qtc from "metric_names_usage_response.qtpl". DO NOT EDIT.
+// See https://github.com/valyala/quicktemplate for details.
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:1
+package stats
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:1
+import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+)
+
+// MetricNamesStatsResponse generates response for /api/v1/status/metric_names_stats .
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+import (
+	qtio422016 "io"
+
+	qt422016 "github.com/valyala/quicktemplate"
+)
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+var (
+	_ = qtio422016.Copy
+	_ = qt422016.AcquireByteBuffer
+)
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+func StreamMetricNamesStatsResponse(qw422016 *qt422016.Writer, stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:8
+	qw422016.N().S(`{"status":"success","statsCollectedSince":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:11
+	qw422016.N().DUL(stats.CollectedSinceTs)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:11
+	qw422016.N().S(`,"statsCollectedRecordsTotal":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:12
+	qw422016.N().DUL(stats.TotalRecords)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:12
+	qw422016.N().S(`,"trackerMemoryMaxSizeBytes":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:13
+	qw422016.N().DUL(stats.MaxSizeBytes)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:13
+	qw422016.N().S(`,"trackerCurrentMemoryUsageBytes":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:14
+	qw422016.N().DUL(stats.CurrentSizeBytes)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:14
+	qw422016.N().S(`,"records":[`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:17
+	for i, r := range stats.Records {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:17
+		qw422016.N().S(`{"metricName":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:19
+		qw422016.N().Q(r.MetricName)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:19
+		qw422016.N().S(`,"queryRequestsCount":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:20
+		qw422016.N().DUL(r.RequestsCount)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:20
+		qw422016.N().S(`,"lastQueryRequestTimestamp":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:21
+		qw422016.N().DUL(r.LastRequestTs)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:21
+		qw422016.N().S(`}`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:23
+		if i+1 < len(stats.Records) {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:23
+			qw422016.N().S(`,`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:23
+		}
+//line app/vmselect/stats/metric_names_usage_response.qtpl:24
+	}
+//line app/vmselect/stats/metric_names_usage_response.qtpl:24
+	qw422016.N().S(`]`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:26
+	qt.Done()
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:27
+	traceJSON := qt.ToJSON()
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+	if traceJSON != "" {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+		qw422016.N().S(`,"trace":`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+		qw422016.N().S(traceJSON)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+	}
+//line app/vmselect/stats/metric_names_usage_response.qtpl:28
+	qw422016.N().S(`}`)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+}
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+func WriteMetricNamesStatsResponse(qq422016 qtio422016.Writer, stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	StreamMetricNamesStatsResponse(qw422016, stats, qt)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qt422016.ReleaseWriter(qw422016)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+}
+
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+func MetricNamesStatsResponse(stats *storage.MetricNamesStatsResponse, qt *querytracer.Tracer) string {
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qb422016 := qt422016.AcquireByteBuffer()
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	WriteMetricNamesStatsResponse(qb422016, stats, qt)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qs422016 := string(qb422016.B)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	qt422016.ReleaseByteBuffer(qb422016)
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+	return qs422016
+//line app/vmselect/stats/metric_names_usage_response.qtpl:31
+}

app/vmselect/stats/stats.go

@@ -0,0 +1,50 @@
+package stats
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+)
+
+// MetricNamesStatsHandler returns timeseries metric names usage statistics
+func MetricNamesStatsHandler(qt *querytracer.Tracer, w http.ResponseWriter, r *http.Request) error {
+	limit := 1000
+	limitStr := r.FormValue("limit")
+	if len(limitStr) > 0 {
+		n, err := strconv.Atoi(limitStr)
+		if err != nil {
+			return fmt.Errorf("cannot parse `limit` arg %q: %w", limitStr, err)
+		}
+		if n > 0 {
+			limit = n
+		}
+	}
+	// by default display all values
+	le := -1
+	leStr := r.FormValue("le")
+	if len(leStr) > 0 {
+		n, err := strconv.Atoi(leStr)
+		if err != nil {
+			return fmt.Errorf("cannot parse `le` arg %q: %w", leStr, err)
+		}
+		le = n
+	}
+	matchPattern := r.FormValue("match_pattern")
+	stats, err := netstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	if err != nil {
+		return err
+	}
+	WriteMetricNamesStatsResponse(w, &stats, qt)
+	return nil
+}
+
+// ResetMetricNamesStatsHandler resets metric names usage state
+func ResetMetricNamesStatsHandler(qt *querytracer.Tracer) error {
+	if err := netstorage.ResetMetricNamesStats(qt); err != nil {
+		return err
+	}
+	return nil
+}

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.63479b72.css",
-    "main.js": "./static/js/main.256ee243.js",
-    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.63479b72.css",
-    "static/js/main.256ee243.js"
-  ]
-}