app/vmctl: remove unused channel

Signed-off-by: hagen1778 <roman@victoriametrics.com>

.github/workflows/docs.yaml

@@ -5,6 +5,7 @@ on:
       - 'master'
     paths:
       - 'docs/**'
+      - '.github/workflows/docs.yaml'
   workflow_dispatch: {}
 permissions:
   contents: read  # This is required for actions/checkout and to commit back image update
@@ -17,35 +18,41 @@ jobs:
       - name: Code checkout
         uses: actions/checkout@v4
         with:
-          path: main
+          path: __vm
+
       - name: Checkout private code
         uses: actions/checkout@v4
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          path: docs
+          path: __vm-docs
+
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@v6
+        id: import-gpg
         with:
           gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
           git_user_signingkey: true
           git_commit_gpgsign: true
-          workdir: docs
-      - name: Set short git commit SHA
-        id: vars
+          git_config_global: true
+
+      - name: Copy docs
+        id: update
         run: |
-          calculatedSha=$(git rev-parse --short ${{ github.sha }})
-          echo "short_sha=$calculatedSha" >> $GITHUB_OUTPUT
-        working-directory: main
-      - name: update code and commit
+          rsync -zarv \
+            --exclude="Makefile" \
+            docs/ ../__vm-docs/content/victoriametrics
+          echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_OUTPUT
+        working-directory: __vm
+
+      - name: Push to vmdocs
         run: |
-          rm -rf content
-          cp -r ../main/docs content
-          make clean-after-copy
           git config --global user.name "${{ steps.import-gpg.outputs.email }}"
           git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          git add .
-          git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.vars.outputs.short_sha }}"
-          git push
-        working-directory: docs
+          if [[ -n $(git status --porcelain) ]]; then
+            git add .
+            git commit -S -m "sync docs with VictoriaMetrics/VictoriaMetrics commit: ${{ steps.update.outputs.SHORT_SHA }}"
+            git push
+          fi
+        working-directory: __vm-docs

.github/workflows/main.yml

@@ -90,7 +90,7 @@ jobs:
       - name: Publish coverage
         uses: codecov/codecov-action@v5
         with:
-          file: ./coverage.txt
+          files: ./coverage.txt
 
   integration-test:
     name: integration-test

Makefile

@@ -567,7 +567,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.63.4
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.64.5
 
 remove-golangci-lint:
 	rm -rf `which golangci-lint`

README.md

@@ -3,7 +3,7 @@
 ![Latest Release](https://img.shields.io/github/v/release/VictoriaMetrics/VictoriaMetrics?sort=semver&label=&filter=!*-victorialogs&logo=github&labelColor=gray&color=gray&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Freleases%2Flatest)
 ![Docker Pulls](https://img.shields.io/docker/pulls/victoriametrics/victoria-metrics?label=&logo=docker&logoColor=white&labelColor=2496ED&color=2496ED&link=https%3A%2F%2Fhub.docker.com%2Fr%2Fvictoriametrics%2Fvictoria-metrics)
 ![Go Report](https://goreportcard.com/badge/github.com/VictoriaMetrics/VictoriaMetrics?link=https%3A%2F%2Fgoreportcard.com%2Freport%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics)
-![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/workflows/main/badge.svg?link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
+![Build Status](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/main.yml/badge.svg?branch=master&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Factions)
 ![codecov](https://codecov.io/gh/VictoriaMetrics/VictoriaMetrics/branch/master/graph/badge.svg?link=https%3A%2F%2Fcodecov.io%2Fgh%2FVictoriaMetrics%2FVictoriaMetrics)
 ![License](https://img.shields.io/github/license/VictoriaMetrics/VictoriaMetrics?labelColor=green&label=&link=https%3A%2F%2Fgithub.com%2FVictoriaMetrics%2FVictoriaMetrics%2Fblob%2Fmaster%2FLICENSE)
 ![Slack](https://img.shields.io/badge/Join-4A154B?logo=slack&link=https%3A%2F%2Fslack.victoriametrics.com)
@@ -22,7 +22,7 @@ Here are some resources and information about VictoriaMetrics:
 
 - Documentation: [docs.victoriametrics.com](https://docs.victoriametrics.com)
 - Case studies: [Grammarly, Roblox, Wix,...](https://docs.victoriametrics.com/casestudies/).
-- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), [Docker images](https://hub.docker.com/r/victoriametrics/victoria-metrics/), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
+- Available: [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest), docker images [Docker Hub](https://hub.docker.com/r/victoriametrics/victoria-metrics/) and [Quay](https://quay.io/repository/victoriametrics/victoria-metrics), [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics)
 - Deployment types: [Single-node version](https://docs.victoriametrics.com/), [Cluster version](https://docs.victoriametrics.com/cluster-victoriametrics/), and [Enterprise version](https://docs.victoriametrics.com/enterprise/)
 - Changelog: [CHANGELOG](https://docs.victoriametrics.com/changelog/), and [How to upgrade](https://docs.victoriametrics.com/#how-to-upgrade-victoriametrics)
 - Community: [Slack](https://slack.victoriametrics.com/), [X (Twitter)](https://x.com/VictoriaMetrics), [LinkedIn](https://www.linkedin.com/company/victoriametrics/), [YouTube](https://www.youtube.com/@VictoriaMetrics)

SECURITY.md

@@ -8,7 +8,7 @@ The following versions of VictoriaMetrics receive regular security fixes:
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/changelog/) | :white_check_mark: |
 | v1.102.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
-| v1.97.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
+| v1.110.x [LTS line](https://docs.victoriametrics.com/lts-releases/) | :white_check_mark: |
 | other releases  | :x:                |
 
 See [this page](https://victoriametrics.com/security/) for more details.

app/vlinsert/elasticsearch/elasticsearch.go

@@ -60,7 +60,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	switch path {
-	case "/":
+	case "/", "":
 		switch r.Method {
 		case http.MethodGet:
 			// Return fake response for Elasticsearch ping request.

app/vlinsert/insertutils/timestamp.go

@@ -2,20 +2,19 @@ package insertutils
 
 import (
 	"fmt"
-	"math"
 	"strconv"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-// ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.
+// ExtractTimestampFromFields extracts timestamp in nanoseconds from the field with the name timeField at fields.
 //
 // The value for the timeField is set to empty string after returning from the function,
 // so it could be ignored during data ingestion.
 //
 // The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.
-func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error) {
+func ExtractTimestampFromFields(timeField string, fields []logstorage.Field) (int64, error) {
 	for i := range fields {
 		f := &fields[i]
 		if f.Name != timeField {
@@ -48,22 +47,24 @@ func parseTimestamp(s string) (int64, error) {
 	return nsecs, nil
 }
 
-// ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.
+// ParseUnixTimestamp parses s as unix timestamp in seconds, milliseconds, microseconds or nanoseconds and returns the parsed timestamp in nanoseconds.
 func ParseUnixTimestamp(s string) (int64, error) {
 	n, err := strconv.ParseInt(s, 10, 64)
 	if err != nil {
 		return 0, fmt.Errorf("cannot parse unix timestamp from %q: %w", s, err)
 	}
 	if n < (1<<31) && n >= (-1<<31) {
-		// The timestamp is in seconds. Convert it to milliseconds
-		n *= 1e3
+		// The timestamp is in seconds.
+		return n * 1e9, nil
 	}
-	if n > int64(math.MaxInt64)/1e6 {
-		return 0, fmt.Errorf("too big timestamp in milliseconds: %d; mustn't exceed %d", n, int64(math.MaxInt64)/1e6)
+	if n < 1e3*(1<<31) && n >= 1e3*(-1<<31) {
+		// The timestamp is in milliseconds.
+		return n * 1e6, nil
 	}
-	if n < int64(math.MinInt64)/1e6 {
-		return 0, fmt.Errorf("too small timestamp in milliseconds: %d; must be bigger than %d", n, int64(math.MinInt64)/1e6)
+	if n < 1e6*(1<<31) && n >= 1e6*(-1<<31) {
+		// The timestamp is in microseconds.
+		return n * 1e3, nil
 	}
-	n *= 1e6
+	// The timestamp is in nanoseconds
 	return n, nil
 }

app/vlinsert/insertutils/timestamp_test.go

@@ -6,11 +6,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
-func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
+func TestExtractTimestampFromFields_Success(t *testing.T) {
 	f := func(timeField string, fields []logstorage.Field, nsecsExpected int64) {
 		t.Helper()
 
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields(timeField, fields)
+		nsecs, err := ExtractTimestampFromFields(timeField, fields)
 		if err != nil {
 			t.Fatalf("unexpected error: %s", err)
 		}
@@ -51,6 +51,18 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 		{Name: "foo", Value: "bar"},
 	}, 1718773640123456789)
 
+	// Unix timestamp in nanoseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123456789"},
+	}, 1718773640123456789)
+
+	// Unix timestamp in microseconds
+	f("time", []logstorage.Field{
+		{Name: "foo", Value: "bar"},
+		{Name: "time", Value: "1718773640123456"},
+	}, 1718773640123456000)
+
 	// Unix timestamp in milliseconds
 	f("time", []logstorage.Field{
 		{Name: "foo", Value: "bar"},
@@ -64,14 +76,14 @@ func TestExtractTimestampRFC3339NanoFromFields_Success(t *testing.T) {
 	}, 1718773640000000000)
 }
 
-func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
+func TestExtractTimestampFromFields_Error(t *testing.T) {
 	f := func(s string) {
 		t.Helper()
 
 		fields := []logstorage.Field{
 			{Name: "time", Value: s},
 		}
-		nsecs, err := ExtractTimestampRFC3339NanoFromFields("time", fields)
+		nsecs, err := ExtractTimestampFromFields("time", fields)
 		if err == nil {
 			t.Fatalf("expecting non-nil error")
 		}
@@ -80,6 +92,7 @@ func TestExtractTimestampRFC3339NanoFromFields_Error(t *testing.T) {
 		}
 	}
 
+	// invalid time
 	f("foobar")
 
 	// incomplete time

app/vlinsert/journald/journald.go

@@ -30,7 +30,7 @@ const (
 
 var (
 	bodyBufferPool                bytesutil.ByteBufferPool
-	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]+`)
+	allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
 )
 
 var (
@@ -129,6 +129,11 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
 		return
 	}
 
+	// systemd starting release v258 will support compression, which starts working after negotiation: it expects supported compression
+	// algorithms list in Accept-Encoding response header in a format "<algorithm_1>[:<priority_1>][;<algorithm_2>:<priority_2>]"
+	// See https://github.com/systemd/systemd/pull/34822
+	w.Header().Set("Accept-Encoding", "zstd")
+
 	// update requestJournaldDuration only for successfully parsed requests
 	// There is no need in updating requestJournaldDuration for request errors,
 	// since their timings are usually much smaller than the timing for successful request parsing.

app/vlinsert/journald/journald_test.go

@@ -35,9 +35,9 @@ func TestPushJournaldOk(t *testing.T) {
 	)
 
 	// Parse binary data
-	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
+	f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\nE=JobStateChanged\n__REALTIME_TIMESTAMP=1729698775704404\n__MONOTONIC_TIMESTAMP=206357648416\n__SEQNUM=7942\n__SEQNUM_ID=e0afe8412a6a49d2bfcf66aa7927b588\n_BOOT_ID=f778b6e2f7584a77b991a2366612a7b5\n_UID=0\n_GID=0\n_MACHINE_ID=a4a970370c30a925df02a13c67167847\n_HOSTNAME=ecd5e4555787\n_RUNTIME_SCOPE=system\n_TRANSPORT=journal\n_CAP_EFFECTIVE=1ffffffffff\n_SYSTEMD_CGROUP=/init.scope\n_SYSTEMD_UNIT=init.scope\n_SYSTEMD_SLICE=-.slice\nCODE_FILE=<stdin>\nCODE_LINE=1\nCODE_FUNC=<module>\nSYSLOG_IDENTIFIER=python3\n_COMM=python3\n_EXE=/usr/bin/python3.12\n_CMDLINE=python3\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n_PID=2763\n_SOURCE_REALTIME_TIMESTAMP=1729698775704375\n\n",
 		[]int64{1729698775704404000},
-		"{\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
+		"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
 	)
 }
 

app/vlinsert/jsonline/jsonline.go

@@ -51,20 +51,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
 
 	lmp := cp.NewLogMessageProcessor("jsonline")
 	streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
-	err = processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
+	processStreamInternal(streamName, reader, cp.TimeField, cp.MsgFields, lmp)
 	lmp.MustClose()
 
-	if err != nil {
-		logger.Errorf("jsonline: %s", err)
-	} else {
-		// update requestDuration only for successfully parsed requests.
-		// There is no need in updating requestDuration for request errors,
-		// since their timings are usually much smaller than the timing for successful request parsing.
-		requestDuration.UpdateDuration(startTime)
-	}
+	requestDuration.UpdateDuration(startTime)
 }
 
-func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) error {
+func processStreamInternal(streamName string, r io.Reader, timeField string, msgFields []string, lmp insertutils.LogMessageProcessor) {
 	wcr := writeconcurrencylimiter.GetReader(r)
 	defer writeconcurrencylimiter.PutReader(wcr)
 
@@ -76,10 +69,10 @@ func processStreamInternal(streamName string, r io.Reader, timeField string, msg
 		wcr.DecConcurrency()
 		if err != nil {
 			errorsTotal.Inc()
-			return fmt.Errorf("cannot read line #%d in /jsonline request: %s", n, err)
+			logger.Warnf("jsonline: cannot read line #%d in /jsonline request: %s", n, err)
 		}
 		if !ok {
-			return nil
+			return
 		}
 		n++
 	}
@@ -96,16 +89,17 @@ func readLine(lr *insertutils.LineReader, timeField string, msgFields []string,
 	}
 
 	p := logstorage.GetJSONParser()
+	defer logstorage.PutJSONParser(p)
+
 	if err := p.ParseLogMessage(line); err != nil {
-		return false, fmt.Errorf("cannot parse json-encoded log entry: %w", err)
+		return true, fmt.Errorf("cannot parse json-encoded line: %w; line contents: %q", err, line)
 	}
-	ts, err := insertutils.ExtractTimestampRFC3339NanoFromFields(timeField, p.Fields)
+	ts, err := insertutils.ExtractTimestampFromFields(timeField, p.Fields)
 	if err != nil {
-		return false, fmt.Errorf("cannot get timestamp: %w", err)
+		return true, fmt.Errorf("cannot get timestamp from json-encoded line: %w; line contents: %q", err, line)
 	}
 	logstorage.RenameField(p.Fields, msgFields, "_msg")
 	lmp.AddRow(ts, p.Fields, nil)
-	logstorage.PutJSONParser(p)
 
 	return true, nil
 }

app/vlinsert/jsonline/jsonline_test.go

@@ -7,16 +7,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 )
 
-func TestProcessStreamInternal_Success(t *testing.T) {
+func TestProcessStreamInternal(t *testing.T) {
 	f := func(data, timeField, msgField string, timestampsExpected []int64, resultExpected string) {
 		t.Helper()
 
 		msgFields := []string{msgField}
 		tlp := &insertutils.TestLogMessageProcessor{}
 		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, timeField, msgFields, tlp); err != nil {
-			t.Fatalf("unexpected error: %s", err)
-		}
+		processStreamInternal("test", r, timeField, msgFields, tlp)
 
 		if err := tlp.Verify(timestampsExpected, resultExpected); err != nil {
 			t.Fatal(err)
@@ -45,22 +43,37 @@ func TestProcessStreamInternal_Success(t *testing.T) {
 	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","message":"foobar"}
 {"message":"baz"}`
 	f(data, timeField, msgField, timestampsExpected, resultExpected)
-}
-
-func TestProcessStreamInternal_Failure(t *testing.T) {
-	f := func(data string) {
-		t.Helper()
-
-		tlp := &insertutils.TestLogMessageProcessor{}
-		r := bytes.NewBufferString(data)
-		if err := processStreamInternal("test", r, "time", nil, tlp); err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
-	}
 
 	// invalid json
-	f("foobar")
+	data = "foobar"
+	timeField = "@timestamp"
+	msgField = "aaa"
+	timestampsExpected = nil
+	resultExpected = ``
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
 
 	// invalid timestamp field
-	f(`{"time":"foobar"}`)
+	data = `{"time":"foobar"}`
+	timeField = "time"
+	msgField = "abc"
+	timestampsExpected = nil
+	resultExpected = ``
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
+
+	// invalid lines among valid lines
+	data = `
+dsfodmasd
+
+{"time":"2023-06-06T04:48:11.735Z","log":{"offset":71770,"file":{"path":"/var/log/auth.log"}},"message":"foobar"}
+invalid line
+{"time":"2023-06-06T04:48:12.735+01:00","message":"baz"}
+asbsdf
+
+`
+	timeField = "time"
+	msgField = "message"
+	timestampsExpected = []int64{1686026891735000000, 1686023292735000000}
+	resultExpected = `{"log.offset":"71770","log.file.path":"/var/log/auth.log","_msg":"foobar"}
+{"_msg":"baz"}`
+	f(data, timeField, msgField, timestampsExpected, resultExpected)
 }

app/vlinsert/main.go

@@ -46,7 +46,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	switch {
-	case strings.HasPrefix(path, "/elasticsearch/"):
+	case strings.HasPrefix(path, "/elasticsearch"):
+		// some clients may omit trailing slash
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
 		path = strings.TrimPrefix(path, "/elasticsearch")
 		return elasticsearch.RequestHandler(path, w, r)
 	case strings.HasPrefix(path, "/loki/"):

app/vlinsert/opentelemetry/opentelemetry.go

@@ -101,7 +101,7 @@ func pushProtobufRequest(data []byte, lmp insertutils.LogMessageProcessor, useDe
 		commonFields = slicesutil.SetLength(commonFields, len(attributes))
 		for i, attr := range attributes {
 			commonFields[i].Name = attr.Key
-			commonFields[i].Value = attr.Value.FormatString()
+			commonFields[i].Value = attr.Value.FormatString(true)
 		}
 		commonFieldsLen := len(commonFields)
 		for _, sc := range rl.ScopeLogs {
@@ -118,12 +118,24 @@ func pushFieldsFromScopeLogs(sc *pb.ScopeLogs, commonFields []logstorage.Field,
 		fields = fields[:len(commonFields)]
 		fields = append(fields, logstorage.Field{
 			Name:  "_msg",
-			Value: lr.Body.FormatString(),
+			Value: lr.Body.FormatString(true),
 		})
 		for _, attr := range lr.Attributes {
 			fields = append(fields, logstorage.Field{
 				Name:  attr.Key,
-				Value: attr.Value.FormatString(),
+				Value: attr.Value.FormatString(true),
+			})
+		}
+		if len(lr.TraceID) > 0 {
+			fields = append(fields, logstorage.Field{
+				Name:  "trace_id",
+				Value: lr.TraceID,
+			})
+		}
+		if len(lr.SpanID) > 0 {
+			fields = append(fields, logstorage.Field{
+				Name:  "span_id",
+				Value: lr.SpanID,
 			})
 		}
 		fields = append(fields, logstorage.Field{

app/vlinsert/opentelemetry/opentelemetry_test.go

@@ -66,9 +66,9 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 1236},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
+		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Unspecified"}
+{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Unspecified"}`,
 	)
 
 	// multi-scope with resource attributes and multi-line
@@ -113,8 +113,8 @@ func TestPushProtoOk(t *testing.T) {
 		},
 	},
 		[]int64{1234, 1235, 2345, 2346, 2347, 2348},
-		`{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message","severity":"Trace"}
-{"logger":"context","instance_id":"10","node_taints":"[{\"Key\":\"role\",\"Value\":{\"StringValue\":\"dev\",\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":null,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}},{\"Key\":\"cluster_load_percent\",\"Value\":{\"StringValue\":null,\"BoolValue\":null,\"IntValue\":null,\"DoubleValue\":0.55,\"ArrayValue\":null,\"KeyValueList\":null,\"BytesValue\":null}}]","_msg":"log-line-message-msg-2","severity":"Debug"}
+		`{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message","severity":"Trace"}
+{"logger":"context","instance_id":"10","node_taints":"{\"role\":\"dev\",\"cluster_load_percent\":0.55}","_msg":"log-line-message-msg-2","severity":"Debug"}
 {"_msg":"log-line-resource-scope-1-0-0","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-0-1","severity":"Info2"}
 {"_msg":"log-line-resource-scope-1-1-0","severity":"Info4"}

app/vlinsert/syslog/syslog.go

@@ -560,7 +560,7 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal
 	if useLocalTimestamp {
 		ts = time.Now().UnixNano()
 	} else {
-		nsecs, err := insertutils.ExtractTimestampRFC3339NanoFromFields("timestamp", p.Fields)
+		nsecs, err := insertutils.ExtractTimestampFromFields("timestamp", p.Fields)
 		if err != nil {
 			return fmt.Errorf("cannot get timestamp from syslog line %q: %w", line, err)
 		}

app/vlogsgenerator/main.go

@@ -18,7 +18,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 var (
@@ -306,7 +306,7 @@ type timeFlag struct {
 }
 
 func (tf *timeFlag) Set(s string) error {
-	msec, err := promutils.ParseTimeMsec(s)
+	msec, err := timeutil.ParseTimeMsec(s)
 	if err != nil {
 		return fmt.Errorf("cannot parse time from %q: %w", s, err)
 	}

app/vlselect/logsql/logsql.go

@@ -22,7 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 // ProcessFacetsRequest handles /select/logsql/facets request.
@@ -116,7 +116,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	if stepStr == "" {
 		stepStr = "1d"
 	}
-	step, err := promutils.ParseDuration(stepStr)
+	step, err := timeutil.ParseDuration(stepStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse 'step' arg: %s", err)
 		return
@@ -131,7 +131,7 @@ func ProcessHitsRequest(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	if offsetStr == "" {
 		offsetStr = "0s"
 	}
-	offset, err := promutils.ParseDuration(offsetStr)
+	offset, err := timeutil.ParseDuration(offsetStr)
 	if err != nil {
 		httpserver.Errorf(w, r, "cannot parse 'offset' arg: %s", err)
 		return
@@ -665,7 +665,7 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	if stepStr == "" {
 		stepStr = "1d"
 	}
-	step, err := promutils.ParseDuration(stepStr)
+	step, err := timeutil.ParseDuration(stepStr)
 	if err != nil {
 		err = fmt.Errorf("cannot parse 'step' arg: %s", err)
 		httpserver.SendPrometheusError(w, r, err)
@@ -688,19 +688,22 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 	m := make(map[string]*statsSeries)
 	var mLock sync.Mutex
 
-	timestamp := q.GetTimestamp()
 	writeBlock := func(_ uint, timestamps []int64, columns []logstorage.BlockColumn) {
 		clonedColumnNames := make([]string, len(columns))
 		for i, c := range columns {
 			clonedColumnNames[i] = strings.Clone(c.Name)
 		}
 		for i := range timestamps {
+			// Do not move q.GetTimestamp() outside writeBlock, since ts
+			// must be initialized to query timestamp for every processed log row.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8312
+			ts := q.GetTimestamp()
 			labels := make([]logstorage.Field, 0, len(byFields))
 			for j, c := range columns {
 				if c.Name == "_time" {
 					nsec, ok := logstorage.TryParseTimestampRFC3339Nano(c.Values[i])
 					if ok {
-						timestamp = nsec
+						ts = nsec
 						continue
 					}
 				}
@@ -721,7 +724,7 @@ func ProcessStatsQueryRangeRequest(ctx context.Context, w http.ResponseWriter, r
 					dst = logstorage.MarshalFieldsToJSON(dst, labels)
 					key := string(dst)
 					p := statsPoint{
-						Timestamp: timestamp,
+						Timestamp: ts,
 						Value:     strings.Clone(c.Values[i]),
 					}
 
@@ -1119,7 +1122,7 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
 		return 0, false, nil
 	}
 	currentTimestamp := time.Now().UnixNano()
-	nsecs, err := promutils.ParseTimeAt(s, currentTimestamp)
+	nsecs, err := timeutil.ParseTimeAt(s, currentTimestamp)
 	if err != nil {
 		return 0, false, fmt.Errorf("cannot parse %s=%s: %w", argName, s, err)
 	}

app/vlselect/main.go

@@ -176,50 +176,62 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 func processSelectRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, path string) bool {
 	httpserver.EnableCORS(w, r)
+	startTime := time.Now()
 	switch path {
 	case "/select/logsql/facets":
 		logsqlFacetsRequests.Inc()
 		logsql.ProcessFacetsRequest(ctx, w, r)
+		logsqlFacetsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/field_names":
 		logsqlFieldNamesRequests.Inc()
 		logsql.ProcessFieldNamesRequest(ctx, w, r)
+		logsqlFieldNamesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/field_values":
 		logsqlFieldValuesRequests.Inc()
 		logsql.ProcessFieldValuesRequest(ctx, w, r)
+		logsqlFieldValuesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/hits":
 		logsqlHitsRequests.Inc()
 		logsql.ProcessHitsRequest(ctx, w, r)
+		logsqlHitsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/query":
 		logsqlQueryRequests.Inc()
 		logsql.ProcessQueryRequest(ctx, w, r)
+		logsqlQueryDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stats_query":
 		logsqlStatsQueryRequests.Inc()
 		logsql.ProcessStatsQueryRequest(ctx, w, r)
+		logsqlStatsQueryDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stats_query_range":
 		logsqlStatsQueryRangeRequests.Inc()
 		logsql.ProcessStatsQueryRangeRequest(ctx, w, r)
+		logsqlStatsQueryRangeDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_field_names":
 		logsqlStreamFieldNamesRequests.Inc()
 		logsql.ProcessStreamFieldNamesRequest(ctx, w, r)
+		logsqlStreamFieldNamesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_field_values":
 		logsqlStreamFieldValuesRequests.Inc()
 		logsql.ProcessStreamFieldValuesRequest(ctx, w, r)
+		logsqlStreamFieldValuesDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/stream_ids":
 		logsqlStreamIDsRequests.Inc()
 		logsql.ProcessStreamIDsRequest(ctx, w, r)
+		logsqlStreamIDsDuration.UpdateDuration(startTime)
 		return true
 	case "/select/logsql/streams":
 		logsqlStreamsRequests.Inc()
 		logsql.ProcessStreamsRequest(ctx, w, r)
+		logsqlStreamsDuration.UpdateDuration(startTime)
 		return true
 	default:
 		return false
@@ -240,16 +252,39 @@ func getMaxQueryDuration(r *http.Request) time.Duration {
 }
 
 var (
-	logsqlFacetsRequests            = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
-	logsqlFieldNamesRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
-	logsqlFieldValuesRequests       = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
-	logsqlHitsRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
-	logsqlQueryRequests             = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
-	logsqlStatsQueryRequests        = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
-	logsqlStatsQueryRangeRequests   = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
-	logsqlStreamFieldNamesRequests  = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlFacetsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/facets"}`)
+	logsqlFacetsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/facets"}`)
+
+	logsqlFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_names"}`)
+	logsqlFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_names"}`)
+
+	logsqlFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/field_values"}`)
+	logsqlFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/field_values"}`)
+
+	logsqlHitsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/hits"}`)
+	logsqlHitsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/hits"}`)
+
+	logsqlQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/query"}`)
+	logsqlQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/query"}`)
+
+	logsqlStatsQueryRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query"}`)
+	logsqlStatsQueryDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query"}`)
+
+	logsqlStatsQueryRangeRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stats_query_range"}`)
+	logsqlStatsQueryRangeDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stats_query_range"}`)
+
+	logsqlStreamFieldNamesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_names"}`)
+	logsqlStreamFieldNamesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_names"}`)
+
 	logsqlStreamFieldValuesRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_field_values"}`)
-	logsqlStreamIDsRequests         = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
-	logsqlStreamsRequests           = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
-	logsqlTailRequests              = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
+	logsqlStreamFieldValuesDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_field_values"}`)
+
+	logsqlStreamIDsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/stream_ids"}`)
+	logsqlStreamIDsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/stream_ids"}`)
+
+	logsqlStreamsRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/streams"}`)
+	logsqlStreamsDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/select/logsql/streams"}`)
+
+	// no need to track duration for tail requests, as they usually take long time
+	logsqlTailRequests = metrics.NewCounter(`vl_http_requests_total{path="/select/logsql/tail"}`)
 )

app/vlselect/vmui/asset-manifest.json

@@ -1,12 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.02a1c6cb.css",
-    "main.js": "./static/js/main.55c8060b.js",
-    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.02a1c6cb.css",
-    "static/js/main.55c8060b.js"
-  ]
-}

app/vlselect/vmui/assets/vendor-D1GxaB_c.css

@@ -0,0 +1 @@
+.uplot,.uplot *,.uplot *:before,.uplot *:after{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol,"Noto Color Emoji";line-height:1.5;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:#00000012;position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{position:absolute;left:0;top:0;pointer-events:none;will-change:transform}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607D8B}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607D8B}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;background-clip:padding-box!important}.u-axis.u-off,.u-select.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-cursor-pt.u-off{display:none}

app/vlselect/vmui/index.html

@@ -1 +1,57 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.55c8060b.js"></script><link href="./static/css/main.02a1c6cb.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="./favicon.svg" />
+  <link rel="apple-touch-icon" href="./favicon.svg" />
+  <link rel="mask-icon" href="./favicon.svg" color="#000000">
+
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta name="description" content="Explore your log data with VictoriaLogs UI"/>
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="./manifest.json"/>
+  <!--
+    Notice the use of in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>UI for VictoriaLogs</title>
+
+  <meta name="twitter:card" content="summary">
+  <meta name="twitter:title" content="UI for VictoriaLogs">
+  <meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/">
+  <meta name="twitter:description" content="Explore your log data with VictoriaLogs UI">
+  <meta name="twitter:image" content="./preview.jpg">
+
+  <meta property="og:type" content="website">
+  <meta property="og:title" content="UI for VictoriaLogs">
+  <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
+  <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
+  <script type="module" crossorigin src="./assets/index-DuTUAk-m.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
+  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-CEiptoJw.css">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vlselect/vmui/static/js/main.55c8060b.js.LICENSE.txt

@@ -1,38 +0,0 @@
-/*!
-	Copyright (c) 2018 Jed Watson.
-	Licensed under the MIT License (MIT), see
-	http://jedwatson.github.io/classnames
-*/
-
-/**
- * @remix-run/router v1.19.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router DOM v6.26.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router v6.26.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */

app/vmagent/remotewrite/remotewrite_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/metrics"
 )
@@ -53,7 +54,7 @@ func TestGetLabelsHash_Distribution(t *testing.T) {
 }
 
 func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
-	f := func(streamAggrConfig, relabelConfig string, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
+	f := func(streamAggrConfig, relabelConfig string, enableWindows bool, dedupInterval time.Duration, keepInput, dropInput bool, input string) {
 		t.Helper()
 		perURLRelabel, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfig))
 		if err != nil {
@@ -77,12 +78,15 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 			rowsDroppedByRelabel:   metrics.GetOrCreateCounter(`bar`),
 		}
 		if dedupInterval > 0 {
-			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, dedupInterval, nil, "dedup-global")
+			rwctx.deduplicator = streamaggr.NewDeduplicator(nil, enableWindows, dedupInterval, nil, "dedup-global")
 		}
 
 		if streamAggrConfig != "" {
 			pushNoop := func(_ []prompbmarshal.TimeSeries) {}
-			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, nil, "global")
+			opts := streamaggr.Options{
+				EnableWindows: enableWindows,
+			}
+			sas, err := streamaggr.LoadFromData([]byte(streamAggrConfig), pushNoop, &opts, "global")
 			if err != nil {
 				t.Fatalf("cannot load streamaggr configs: %s", err)
 			}
@@ -91,7 +95,7 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 		}
 
 		offsetMsecs := time.Now().UnixMilli()
-		inputTss := prompbmarshal.MustParsePromMetrics(input, offsetMsecs)
+		inputTss := prometheus.MustParsePromMetrics(input, offsetMsecs)
 		expectedTss := make([]prompbmarshal.TimeSeries, len(inputTss))
 
 		// copy inputTss to make sure it is not mutated during TryPush call
@@ -114,13 +118,13 @@ func TestRemoteWriteContext_TryPush_ImmutableTimeseries(t *testing.T) {
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, 0, false, false, `
+`, false, 0, false, false, `
 metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
 metric{env="bar"} 25
 `)
-	f(``, ``, time.Hour, false, false, `
+	f(``, ``, true, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="foo"} 20
 metric{env="dev"} 15
@@ -130,7 +134,7 @@ metric{env="foo"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, false, false, `
+`, true, time.Hour, false, false, `
 metric{env="dev"} 10
 metric{env="bar"} 20
 metric{env="dev"} 15
@@ -140,7 +144,7 @@ metric{env="bar"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, true, false, `
+`, true, time.Hour, true, false, `
 metric{env="test"} 10
 metric{env="dev"} 20
 metric{env="foo"} 15
@@ -150,7 +154,7 @@ metric{env="dev"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, false, true, `
+`, true, time.Hour, false, true, `
 metric{env="foo"} 10
 metric{env="dev"} 20
 metric{env="foo"} 15
@@ -160,7 +164,7 @@ metric{env="dev"} 25
 - action: keep
   source_labels: [env]
   regex: "dev"
-`, time.Hour, true, true, `
+`, true, time.Hour, true, true, `
 metric{env="dev"} 10
 metric{env="test"} 20
 metric{env="dev"} 15

app/vmagent/remotewrite/streamaggr.go

@@ -35,6 +35,9 @@ var (
 		"clients pushing data into the vmagent. See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
 	streamAggrGlobalDropInputLabels = flagutil.NewArrayString("streamAggr.dropInputLabels", "An optional list of labels to drop from samples for aggregator "+
 		"before stream de-duplication and aggregation . See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+	streamAggrGlobalEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all global aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
 
 	// Per URL config
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config for the corresponding -remoteWrite.url. "+
@@ -59,6 +62,9 @@ var (
 		"before stream de-duplication and aggregation with -remoteWrite.streamAggr.config and -remoteWrite.streamAggr.dedupInterval at the corresponding -remoteWrite.url. "+
 		"Multiple labels per remoteWrite.url must be delimited by '^^': -remoteWrite.streamAggr.dropInputLabels='replica^^az,replica'. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#dropping-unneeded-labels")
+	streamAggrEnableWindows = flagutil.NewArrayBool("remoteWrite.streamAggr.enableWindows", "Enables aggregation within fixed windows for all remote write's aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
 )
 
 // CheckStreamAggrConfigs checks -remoteWrite.streamAggr.config and -streamAggr.config.
@@ -135,7 +141,7 @@ func initStreamAggrConfigGlobal() {
 	}
 	dedupInterval := *streamAggrGlobalDedupInterval
 	if dedupInterval > 0 {
-		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
+		deduplicatorGlobal = streamaggr.NewDeduplicator(pushToRemoteStoragesTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, *streamAggrGlobalDropInputLabels, "dedup-global")
 	}
 }
 
@@ -161,7 +167,7 @@ func (rwctx *remoteWriteCtx) initStreamAggrConfig() {
 		if streamAggrDropInputLabels.GetOptionalArg(idx) != "" {
 			dropLabels = strings.Split(streamAggrDropInputLabels.GetOptionalArg(idx), "^^")
 		}
-		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, dedupInterval, dropLabels, alias)
+		rwctx.deduplicator = streamaggr.NewDeduplicator(rwctx.pushInternalTrackDropped, *streamAggrGlobalEnableWindows, dedupInterval, dropLabels, alias)
 	}
 }
 
@@ -207,6 +213,7 @@ func newStreamAggrConfigGlobal() (*streamaggr.Aggregators, error) {
 		IgnoreOldSamples:     *streamAggrGlobalIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrGlobalIgnoreFirstIntervals,
 		KeepInput:            *streamAggrGlobalKeepInput,
+		EnableWindows:        *streamAggrGlobalEnableWindows,
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushToRemoteStoragesTrackDropped, opts, "global")
@@ -240,6 +247,7 @@ func newStreamAggrConfigPerURL(idx int, pushFunc streamaggr.PushFunc) (*streamag
 		IgnoreOldSamples:     streamAggrIgnoreOldSamples.GetOptionalArg(idx),
 		IgnoreFirstIntervals: streamAggrIgnoreFirstIntervals.GetOptionalArg(idx),
 		KeepInput:            streamAggrKeepInput.GetOptionalArg(idx),
+		EnableWindows:        streamAggrEnableWindows.GetOptionalArg(idx),
 	}
 
 	sas, err := streamaggr.LoadFromFile(path, pushFunc, opts, alias)

app/vmalert-tool/main.go

@@ -51,6 +51,11 @@ Examples:
 						Usage:    `Optional external URL to template in rule's labels or annotations.`,
 						Required: false,
 					},
+					&cli.StringFlag{
+						Name:     "httpListenPort",
+						Usage:    `Optional local port for incoming HTTP requests. If not specified, a random unoccupied port will be used.`,
+						Required: false,
+					},
 					&cli.StringFlag{
 						Name:     "loggerLevel",
 						Usage:    `Minimum level of errors to log. Possible values: INFO, WARN, ERROR, FATAL, PANIC (default "ERROR").`,
@@ -58,7 +63,7 @@ Examples:
 					},
 				},
 				Action: func(c *cli.Context) error {
-					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url"), c.String("loggerLevel")); failed {
+					if failed := unittest.UnitTest(c.StringSlice("files"), c.Bool("disableAlertgroupLabel"), c.StringSlice("external.label"), c.String("external.url"), c.String("httpListenPort"), c.String("loggerLevel")); failed {
 						return fmt.Errorf("unittest failed")
 					}
 					return nil

app/vmalert-tool/unittest/unittest.go

@@ -4,13 +4,18 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"net/url"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"gopkg.in/yaml.v2"
@@ -28,7 +33,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/prometheus"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -38,15 +42,9 @@ import (
 
 var (
 	storagePath    string
-	httpListenAddr = ":8880"
+	httpListenAddr string
 	// insert series from 1970-01-01T00:00:00
-	testStartTime = time.Unix(0, 0).UTC()
-
-	testPromWriteHTTPPath = "http://127.0.0.1" + httpListenAddr + "/api/v1/write"
-	testDataSourcePath    = "http://127.0.0.1" + httpListenAddr + "/prometheus"
-	testRemoteWritePath   = "http://127.0.0.1" + httpListenAddr
-	testHealthHTTPPath    = "http://127.0.0.1" + httpListenAddr + "/health"
-
+	testStartTime          = time.Unix(0, 0).UTC()
 	testLogLevel           = "ERROR"
 	disableAlertgroupLabel bool
 )
@@ -56,7 +54,7 @@ const (
 )
 
 // UnitTest runs unittest for files
-func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL, logLevel string) bool {
+func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, externalURL, httpListenPort, logLevel string) bool {
 	if logLevel != "" {
 		testLogLevel = logLevel
 	}
@@ -67,7 +65,42 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)
 	}
-	storagePath = filepath.Join(os.TempDir(), testStoragePath)
+
+	// set up http server
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/prometheus/api/v1/query":
+			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+			}
+		case "/prometheus/api/v1/write", "/api/v1/write":
+			if err := promremotewrite.InsertHandler(r); err != nil {
+				httpserver.Errorf(w, r, "%s", err)
+			}
+		default:
+		}
+	})
+	if httpListenPort == "" {
+		server := httptest.NewServer(handler)
+		httpListenAddr = strings.Split(server.URL, ":")[2]
+		defer server.Close()
+	} else {
+		httpListenAddr = httpListenPort
+		ln, err := net.Listen("tcp", fmt.Sprintf(":%s", httpListenPort))
+		if err != nil {
+			logger.Fatalf("cannot listen on port %s: %v", httpListenPort, err)
+		}
+		go func() {
+			err = http.Serve(ln, handler)
+			if err != nil {
+				logger.Fatalf("cannot start http server: %v", err)
+			}
+			defer ln.Close()
+		}()
+	}
+
+	// adding time.Now().UnixNano() to avoid possible file conflict when multiple processes run on a single host
+	storagePath = filepath.Join(os.TempDir(), testStoragePath, strconv.FormatInt(time.Now().UnixNano(), 10))
 	processFlags()
 	vminsert.Init()
 	vmselect.Init()
@@ -102,16 +135,34 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	}
 
 	var failed bool
-	for fileName, file := range testfiles {
-		if err := ruleUnitTest(fileName, file, labels); err != nil {
-			fmt.Println("FAILED")
-			fmt.Printf("failed to run unit test for file %q: \n%v", fileName, err)
-			failed = true
-		} else {
-			fmt.Println("  SUCCESS")
+	runTest := func() bool {
+		for fileName, file := range testfiles {
+			if err := ruleUnitTest(fileName, file, labels); err != nil {
+				fmt.Println("FAILED")
+				fmt.Printf("failed to run unit test for file %q: \n%v", fileName, err)
+				return true
+			}
+			fmt.Println("SUCCESS")
 		}
+		return false
 	}
 
+	finishCh := make(chan struct{}, 1)
+	go func() {
+		failed = runTest()
+		finishCh <- struct{}{}
+	}()
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	select {
+	case sig := <-sigs:
+		fmt.Printf("received signal %s\n", sig)
+		failed = true
+		break
+	case <-finishCh:
+		break
+	}
 	return failed
 }
 
@@ -211,8 +262,8 @@ func processFlags() {
 		{flag: "search.disableCache", value: "true"},
 		// set storage retention time to 100 years, allow to store series from 1970-01-01T00:00:00.
 		{flag: "retentionPeriod", value: "100y"},
-		{flag: "datasource.url", value: testDataSourcePath},
-		{flag: "remoteWrite.url", value: testRemoteWritePath},
+		{flag: "datasource.url", value: fmt.Sprintf("http://127.0.0.1:%s/prometheus", httpListenAddr)},
+		{flag: "remoteWrite.url", value: fmt.Sprintf("http://127.0.0.1:%s", httpListenAddr)},
 		{flag: "notifier.blackhole", value: "true"},
 	} {
 		// panics if flag doesn't exist
@@ -224,27 +275,10 @@ func processFlags() {
 
 func setUp() {
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
-	var ab flagutil.ArrayBool
-	go httpserver.Serve([]string{httpListenAddr}, &ab, func(w http.ResponseWriter, r *http.Request) bool {
-		switch r.URL.Path {
-		case "/prometheus/api/v1/query":
-			if err := prometheus.QueryHandler(nil, time.Now(), w, r); err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		case "/prometheus/api/v1/write", "/api/v1/write":
-			if err := promremotewrite.InsertHandler(r); err != nil {
-				httpserver.Errorf(w, r, "%s", err)
-			}
-			return true
-		default:
-		}
-		return false
-	})
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	readyCheckFunc := func() bool {
-		resp, err := http.Get(testHealthHTTPPath)
+		resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%s/health", httpListenAddr))
 		if err != nil {
 			return false
 		}
@@ -266,9 +300,6 @@ checkCheck:
 }
 
 func tearDown() {
-	if err := httpserver.Stop([]string{httpListenAddr}); err != nil {
-		logger.Errorf("cannot stop the webservice: %s", err)
-	}
 	vmstorage.Stop()
 	metrics.UnregisterAllMetrics()
 	fs.MustRemoveAll(storagePath)
@@ -283,7 +314,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 	if tg.Interval == nil {
 		tg.Interval = promutils.NewDuration(evalInterval)
 	}
-	err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, testPromWriteHTTPPath)
+	err := writeInputSeries(tg.InputSeries, tg.Interval, testStartTime, fmt.Sprintf("http://127.0.0.1:%s/api/v1/write", httpListenAddr))
 	if err != nil {
 		return []error{err}
 	}

app/vmalert-tool/unittest/unittest_test.go

@@ -8,7 +8,7 @@ func TestUnitTest_Failure(t *testing.T) {
 	f := func(files []string) {
 		t.Helper()
 
-		failed := UnitTest(files, false, nil, "", "")
+		failed := UnitTest(files, false, nil, "", "", "")
 		if !failed {
 			t.Fatalf("expecting failed test")
 		}
@@ -20,19 +20,20 @@ func TestUnitTest_Failure(t *testing.T) {
 }
 
 func TestUnitTest_Success(t *testing.T) {
-	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL string) {
+	f := func(disableGroupLabel bool, files []string, externalLabels []string, externalURL, httpPort string) {
 		t.Helper()
 
-		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL, "")
+		failed := UnitTest(files, disableGroupLabel, externalLabels, externalURL, httpPort, "")
 		if failed {
 			t.Fatalf("unexpected failed test")
 		}
 	}
 
-	// run multi files
-	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000")
+	// run multi files with random http port
+	f(false, []string{"./testdata/test1.yaml", "./testdata/test2.yaml"}, []string{"cluster=prod"}, "http://grafana:3000", "")
 
 	// disable group label
 	// template with null external values
-	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "")
+	// specify httpListenAddr
+	f(true, []string{"./testdata/disable-group-label.yaml"}, nil, "", "8880")
 }

app/vmalert/config/config_test.go

@@ -157,6 +157,19 @@ func TestGroupValidate_Failure(t *testing.T) {
 
 	f(&Group{}, false, "group name must be set")
 
+	f(&Group{
+		Name: "both record and alert are not set",
+		Rules: []Rule{
+			{
+				Expr: "sum(up == 0 ) by (host)",
+				For:  promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Expr: "sumSeries(time('foo.bar',10))",
+			},
+		},
+	}, false, "invalid rule")
+
 	f(&Group{
 		Name:     "negative interval",
 		Interval: promutils.NewDuration(-1),
@@ -240,59 +253,6 @@ func TestGroupValidate_Failure(t *testing.T) {
 		},
 	}, false, "duplicate")
 
-	f(&Group{
-		Name: "test graphite with prometheus expr",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
-
-	f(&Group{
-		Name: "test graphite inherit",
-		Type: NewGraphiteType(),
-		Rules: []Rule{
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sum(up == 0 ) by (host)",
-			},
-		},
-	}, false, "either `record` or `alert` must be set")
-
-	f(&Group{
-		Name: "test vlogs with prometheus expr",
-		Type: NewVLogsType(),
-		Rules: []Rule{
-			{
-				Expr: "sum(up == 0 ) by (host)",
-				For:  promutils.NewDuration(10 * time.Millisecond),
-			},
-			{
-				Expr: "sumSeries(time('foo.bar',10))",
-			},
-		},
-	}, false, "invalid rule")
-
-	// validate expressions
-	f(&Group{
-		Name: "test",
-		Rules: []Rule{
-			{
-				Record: "record",
-				Expr:   "up | 0",
-			},
-		},
-	}, true, "invalid expression")
-
 	f(&Group{
 		Name: "test thanos",
 		Type: NewRawType("thanos"),
@@ -303,8 +263,20 @@ func TestGroupValidate_Failure(t *testing.T) {
 		},
 	}, true, "unknown datasource type")
 
+	// validate expressions
 	f(&Group{
-		Name: "test graphite",
+		Name: "test prometheus expr",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{
+				Record: "record",
+				Expr:   "up | 0",
+			},
+		},
+	}, true, "bad prometheus expr")
+
+	f(&Group{
+		Name: "test graphite expr",
 		Type: NewGraphiteType(),
 		Rules: []Rule{
 			{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
@@ -314,14 +286,63 @@ func TestGroupValidate_Failure(t *testing.T) {
 	}, true, "bad graphite expr")
 
 	f(&Group{
-		Name: "test vlogs",
+		Name: "test vlogs expr",
 		Type: NewVLogsType(),
 		Rules: []Rule{
-			{Alert: "alert", Expr: "stats count(*) as requests", Labels: map[string]string{
-				"description": "some-description",
-			}},
+			{Alert: "alert", Expr: "stats count(*) as requests"},
 		},
 	}, true, "bad LogsQL expr")
+
+	f(&Group{
+		Name: "test vlogs expr",
+		Type: NewVLogsType(),
+		Rules: []Rule{
+			{Alert: "alert", Expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"},
+		},
+	}, true, "bad LogsQL expr")
+
+	f(&Group{
+		Name: "test graphite with prometheus expr",
+		Type: NewGraphiteType(),
+		Rules: []Rule{
+			{
+				Record: "r1",
+				ID:     1,
+				Expr:   "sumSeries(time('foo.bar',10))",
+				For:    promutils.NewDuration(10 * time.Millisecond),
+			},
+			{
+				Record: "r2",
+				ID:     2,
+				Expr:   "sum(up == 0 ) by (host)",
+			},
+		},
+	}, true, "bad graphite expr")
+
+	f(&Group{
+		Name: "test vlogs with prometheus exp",
+		Type: NewVLogsType(),
+		Rules: []Rule{
+			{
+				Record: "r1",
+				Expr:   "sum(up == 0 ) by (host)",
+				For:    promutils.NewDuration(10 * time.Millisecond),
+			},
+		},
+	}, true, "bad LogsQL expr")
+
+	f(&Group{
+		Name: "test prometheus with vlogs exp",
+		Type: NewPrometheusType(),
+		Rules: []Rule{
+			{
+				Record: "r1",
+				Expr:   "* | stats by (path) count()",
+				For:    promutils.NewDuration(10 * time.Millisecond),
+			},
+		},
+	}, true, "bad prometheus expr")
+
 }
 
 func TestGroupValidate_Success(t *testing.T) {

app/vmalert/config/types.go

@@ -71,9 +71,18 @@ func (t *Type) ValidateExpr(expr string) error {
 			return fmt.Errorf("bad prometheus expr: %q, err: %w", expr, err)
 		}
 	case "vlogs":
-		if _, err := logstorage.ParseStatsQuery(expr, 0); err != nil {
+		q, err := logstorage.ParseStatsQuery(expr, 0)
+		if err != nil {
 			return fmt.Errorf("bad LogsQL expr: %q, err: %w", expr, err)
 		}
+		fields, _ := q.GetStatsByFields()
+		for i := range fields {
+			// VictoriaLogs inserts `_time` field as a label in result when query with `stats by (_time:step)`,
+			// making the result meaningless and may lead to cardinality issues.
+			if fields[i] == "_time" {
+				return fmt.Errorf("bad LogsQL expr: %q, err: cannot contain time buckets stats pipe `stats by (_time:step)`", expr)
+			}
+		}
 	default:
 		return fmt.Errorf("unknown datasource type=%q", t.Name)
 	}

app/vmalert/main.go

@@ -405,6 +405,9 @@ func configsEqual(a, b []config.Group) bool {
 		if a[i].Checksum != b[i].Checksum {
 			return false
 		}
+		if a[i].File != b[i].File {
+			return false
+		}
 	}
 	return true
 }

app/vmalert/notifier/alertmanager.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
@@ -29,25 +31,34 @@ type AlertManager struct {
 	// stores already parsed RelabelConfigs object
 	relabelConfigs *promrelabel.ParsedConfigs
 
-	metrics *metrics
+	metrics *notifierMetrics
 }
 
-type metrics struct {
-	alertsSent       *utils.Counter
-	alertsSendErrors *utils.Counter
+type notifierMetrics struct {
+	set *metrics.Set
+
+	alertsSent       *metrics.Counter
+	alertsSendErrors *metrics.Counter
 }
 
-func newMetrics(addr string) *metrics {
-	return &metrics{
-		alertsSent:       utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
-		alertsSendErrors: utils.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
+func newNotifierMetrics(addr string) *notifierMetrics {
+	set := metrics.NewSet()
+	metrics.RegisterSet(set)
+
+	return &notifierMetrics{
+		set:              set,
+		alertsSent:       set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_sent_total{addr=%q}", addr)),
+		alertsSendErrors: set.GetOrCreateCounter(fmt.Sprintf("vmalert_alerts_send_errors_total{addr=%q}", addr)),
 	}
 }
 
+func (nm *notifierMetrics) close() {
+	metrics.UnregisterSet(nm.set, true)
+}
+
 // Close is a destructor method for AlertManager
 func (am *AlertManager) Close() {
-	am.metrics.alertsSent.Unregister()
-	am.metrics.alertsSendErrors.Unregister()
+	am.metrics.close()
 }
 
 // Addr returns address where alerts are sent.
@@ -70,7 +81,7 @@ func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[st
 
 func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	b := &bytes.Buffer{}
-	alertsToSend := alerts[:0]
+	alertsToSend := make([]Alert, 0, len(alerts))
 	lblss := make([][]prompbmarshal.Label, 0, len(alerts))
 	for _, a := range alerts {
 		lbls := a.applyRelabelingIfNeeded(am.relabelConfigs)
@@ -180,6 +191,6 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, authCfg proma
 		relabelConfigs: relabelCfg,
 		client:         &http.Client{Transport: tr},
 		timeout:        timeout,
-		metrics:        newMetrics(alertManagerURL),
+		metrics:        newNotifierMetrics(alertManagerURL),
 	}, nil
 }

app/vmalert/notifier/init.go

@@ -54,6 +54,7 @@ var (
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
 	oauth2Scopes = flagutil.NewArrayString("notifier.oauth2.scopes", "Optional OAuth2 scopes to use for -notifier.url. Scopes must be delimited by ';'. "+
 		"If multiple args are set, then they are applied independently for the corresponding -notifier.url")
+	sendTimeout = flagutil.NewArrayDuration("notifier.sendTimeout", 10*time.Second, "Timeout when sending alerts to the corresponding -notifier.url")
 )
 
 // cw holds a configWatcher for configPath configuration file
@@ -175,7 +176,7 @@ func notifiersFromFlags(gen AlertURLGenerator) ([]Notifier, error) {
 		}
 
 		addr = strings.TrimSuffix(addr, "/")
-		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, time.Second*10)
+		am, err := NewAlertManager(addr+alertManagerPath, gen, authCfg, nil, sendTimeout.GetOptionalArg(i))
 		if err != nil {
 			return nil, err
 		}

app/vmalert/notifier/notifier_blackhole.go

@@ -6,7 +6,7 @@ import "context"
 // to be sent.
 type blackHoleNotifier struct {
 	addr    string
-	metrics *metrics
+	metrics *notifierMetrics
 }
 
 // Send will send no notifications, but increase the metric.
@@ -22,8 +22,7 @@ func (bh blackHoleNotifier) Addr() string {
 
 // Close unregister the metrics
 func (bh *blackHoleNotifier) Close() {
-	bh.metrics.alertsSent.Unregister()
-	bh.metrics.alertsSendErrors.Unregister()
+	bh.metrics.close()
 }
 
 // newBlackHoleNotifier creates a new blackHoleNotifier
@@ -31,6 +30,6 @@ func newBlackHoleNotifier() *blackHoleNotifier {
 	address := "blackhole"
 	return &blackHoleNotifier{
 		addr:    address,
-		metrics: newMetrics(address),
+		metrics: newNotifierMetrics(address),
 	}
 }

app/vmalert/rule/alerting.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -57,6 +59,69 @@ type alertingRuleMetrics struct {
 	seriesFetched *utils.Gauge
 }
 
+func newAlertingRuleMetrics(set *metrics.Set, ar *AlertingRule) *alertingRuleMetrics {
+	labels := fmt.Sprintf(`alertname=%q, group=%q, file=%q, id="%d"`, ar.Name, ar.GroupName, ar.File, ar.ID())
+	arm := &alertingRuleMetrics{}
+
+	arm.pending = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
+		func() float64 {
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StatePending {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	arm.active = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
+		func() float64 {
+			ar.alertsMu.RLock()
+			defer ar.alertsMu.RUnlock()
+			var num int
+			for _, a := range ar.alerts {
+				if a.State == notifier.StateFiring {
+					num++
+				}
+			}
+			return float64(num)
+		})
+	arm.errors = utils.NewCounter(set, fmt.Sprintf(`vmalert_alerting_rules_errors_total{%s}`, labels))
+	arm.samples = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			return float64(e.Samples)
+		})
+	arm.seriesFetched = utils.NewGauge(set, fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_series_fetched{%s}`, labels),
+		func() float64 {
+			e := ar.state.getLast()
+			if e.SeriesFetched == nil {
+				// means seriesFetched is unsupported
+				return -1
+			}
+			seriesFetched := float64(*e.SeriesFetched)
+			if seriesFetched == 0 && e.Samples > 0 {
+				// `alert: 0.95` will fetch no series
+				// but will get one time series in response.
+				seriesFetched = float64(e.Samples)
+			}
+			return seriesFetched
+		})
+	return arm
+}
+
+func (arm *alertingRuleMetrics) close() {
+	if arm == nil {
+		return
+	}
+	arm.errors.Unregister()
+	arm.active.Unregister()
+	arm.pending.Unregister()
+	arm.samples.Unregister()
+	arm.seriesFetched.Unregister()
+}
+
 // NewAlertingRule creates a new AlertingRule
 func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule) *AlertingRule {
 	ar := &AlertingRule{
@@ -81,8 +146,7 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 			Headers:                   group.Headers,
 			Debug:                     cfg.Debug,
 		}),
-		alerts:  make(map[uint64]*notifier.Alert),
-		metrics: &alertingRuleMetrics{},
+		alerts: make(map[uint64]*notifier.Alert),
 	}
 
 	entrySize := *ruleUpdateEntriesLimit
@@ -95,63 +159,17 @@ func NewAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 	ar.state = &ruleState{
 		entries: make([]StateEntry, entrySize),
 	}
-
-	labels := fmt.Sprintf(`alertname=%q, group=%q, file=%q, id="%d"`, ar.Name, group.Name, group.File, ar.ID())
-	ar.metrics.pending = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_pending{%s}`, labels),
-		func() float64 {
-			ar.alertsMu.RLock()
-			defer ar.alertsMu.RUnlock()
-			var num int
-			for _, a := range ar.alerts {
-				if a.State == notifier.StatePending {
-					num++
-				}
-			}
-			return float64(num)
-		})
-	ar.metrics.active = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerts_firing{%s}`, labels),
-		func() float64 {
-			ar.alertsMu.RLock()
-			defer ar.alertsMu.RUnlock()
-			var num int
-			for _, a := range ar.alerts {
-				if a.State == notifier.StateFiring {
-					num++
-				}
-			}
-			return float64(num)
-		})
-	ar.metrics.errors = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_alerting_rules_errors_total{%s}`, labels))
-	ar.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_samples{%s}`, labels),
-		func() float64 {
-			e := ar.state.getLast()
-			return float64(e.Samples)
-		})
-	ar.metrics.seriesFetched = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_alerting_rules_last_evaluation_series_fetched{%s}`, labels),
-		func() float64 {
-			e := ar.state.getLast()
-			if e.SeriesFetched == nil {
-				// means seriesFetched is unsupported
-				return -1
-			}
-			seriesFetched := float64(*e.SeriesFetched)
-			if seriesFetched == 0 && e.Samples > 0 {
-				// `alert: 0.95` will fetch no series
-				// but will get one time series in response.
-				seriesFetched = float64(e.Samples)
-			}
-			return seriesFetched
-		})
+	ar.metrics = newAlertingRuleMetrics(group.metrics.set, ar)
 	return ar
 }
 
+func (ar *AlertingRule) registerMetrics(g *Group) {
+	ar.metrics = newAlertingRuleMetrics(g.metrics.set, ar)
+}
+
 // close unregisters rule metrics
-func (ar *AlertingRule) close() {
-	ar.metrics.active.Unregister()
-	ar.metrics.pending.Unregister()
-	ar.metrics.errors.Unregister()
-	ar.metrics.samples.Unregister()
-	ar.metrics.seriesFetched.Unregister()
+func (ar *AlertingRule) unregisterMetrics() {
+	ar.metrics.close()
 }
 
 // String implements Stringer interface

app/vmalert/rule/alerting_test.go

@@ -11,6 +11,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -1248,13 +1250,17 @@ func newTestAlertingRule(name string, waitFor time.Duration) *AlertingRule {
 		EvalInterval: waitFor,
 		alerts:       make(map[uint64]*notifier.Alert),
 		state:        &ruleState{entries: make([]StateEntry, 10)},
-		metrics: &alertingRuleMetrics{
-			errors: utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_alerting_rules_errors_total{alertname=%q}`, name)),
-		},
+		metrics:      getTestAlertingRuleMetrics(name),
 	}
 	return &rule
 }
 
+func getTestAlertingRuleMetrics(name string) *alertingRuleMetrics {
+	m := &alertingRuleMetrics{}
+	m.errors = utils.NewCounter(metrics.NewSet(), fmt.Sprintf(`vmalert_alerting_rules_errors_total{alertname=%q}`, name))
+	return m
+}
+
 func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, keepFiringFor time.Duration, annotation map[string]string) *AlertingRule {
 	rule := newTestAlertingRule(name, waitFor)
 	if evalInterval != 0 {

app/vmalert/rule/group.go

@@ -13,6 +13,8 @@ import (
 
 	"github.com/cheggaaa/pb/v3"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -20,7 +22,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -74,19 +75,23 @@ type Group struct {
 }
 
 type groupMetrics struct {
-	iterationTotal    *utils.Counter
-	iterationDuration *utils.Summary
-	iterationMissed   *utils.Counter
-	iterationInterval *utils.Gauge
+	set *metrics.Set
+
+	iterationTotal    *metrics.Counter
+	iterationDuration *metrics.Summary
+	iterationMissed   *metrics.Counter
+	iterationInterval *metrics.Gauge
 }
 
 func newGroupMetrics(g *Group) *groupMetrics {
 	m := &groupMetrics{}
+	m.set = metrics.NewSet()
+
 	labels := fmt.Sprintf(`group=%q, file=%q`, g.Name, g.File)
-	m.iterationTotal = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
-	m.iterationDuration = utils.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
-	m.iterationMissed = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
-	m.iterationInterval = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
+	m.iterationTotal = m.set.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
+	m.iterationDuration = m.set.GetOrCreateSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
+	m.iterationMissed = m.set.GetOrCreateCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
+	m.iterationInterval = m.set.GetOrCreateGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
 		g.mu.RLock()
 		i := g.Interval.Seconds()
 		g.mu.RUnlock()
@@ -95,6 +100,14 @@ func newGroupMetrics(g *Group) *groupMetrics {
 	return m
 }
 
+func (m *groupMetrics) start() {
+	metrics.RegisterSet(m.set)
+}
+
+func (m *groupMetrics) close() {
+	metrics.UnregisterSet(m.set, true)
+}
+
 // merges group rule labels into result map
 // set2 has priority over set1.
 func mergeLabels(groupName, ruleName string, set1, set2 map[string]string) map[string]string {
@@ -237,7 +250,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 		if !ok {
 			// old rule is not present in the new list
 			// so we mark it for removing
-			g.Rules[i].close()
+			g.Rules[i].unregisterMetrics()
 			g.Rules[i] = nil
 			continue
 		}
@@ -257,6 +270,7 @@ func (g *Group) updateWith(newGroup *Group) error {
 	}
 	// add the rest of rules from registry
 	for _, nr := range rulesRegistry {
+		nr.registerMetrics(g)
 		newRules = append(newRules, nr)
 	}
 	// note that g.Interval is not updated here
@@ -295,13 +309,7 @@ func (g *Group) Close() {
 	g.InterruptEval()
 	<-g.finishedCh
 
-	g.metrics.iterationDuration.Unregister()
-	g.metrics.iterationTotal.Unregister()
-	g.metrics.iterationMissed.Unregister()
-	g.metrics.iterationInterval.Unregister()
-	for _, rule := range g.Rules {
-		rule.close()
-	}
+	g.metrics.close()
 }
 
 // SkipRandSleepOnGroupStart will skip random sleep delay in group first evaluation
@@ -311,6 +319,8 @@ var SkipRandSleepOnGroupStart bool
 func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
+	g.metrics.start()
+
 	evalTS := time.Now()
 	// sleep random duration to spread group rules evaluation
 	// over time in order to reduce load on datasource.

app/vmalert/rule/group_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -41,6 +42,7 @@ func TestUpdateWith(t *testing.T) {
 		g := &Group{
 			Name: "test",
 		}
+		g.metrics = newGroupMetrics(g)
 		qb := &datasource.FakeQuerier{}
 		for _, r := range currentRules {
 			r.ID = config.HashRule(r)
@@ -50,6 +52,7 @@ func TestUpdateWith(t *testing.T) {
 		ng := &Group{
 			Name: "test",
 		}
+		ng.metrics = newGroupMetrics(ng)
 		for _, r := range newRules {
 			r.ID = config.HashRule(r)
 			ng.Rules = append(ng.Rules, ng.newRule(qb, r))
@@ -195,6 +198,7 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 		Interval: 100 * time.Hour,
 		updateCh: make(chan *Group),
 	}
+	g.metrics = newGroupMetrics(g)
 	go g.Start(context.Background(), nil, nil, nil)
 
 	rule1 := AlertingRule{
@@ -209,6 +213,7 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 			&rule1,
 		},
 	}
+	g1.metrics = newGroupMetrics(g1)
 	g.updateCh <- g1
 	time.Sleep(10 * time.Millisecond)
 	g.mu.RLock()
@@ -218,8 +223,9 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 	g.mu.RUnlock()
 
 	rule2 := AlertingRule{
-		Name: "jobDown",
-		Expr: "up{job=\"vmagent\"}==0",
+		RuleID: 1,
+		Name:   "jobDown",
+		Expr:   "up{job=\"vmagent\"}==0",
 		Labels: map[string]string{
 			"foo": "bar",
 			"baz": "qux",
@@ -227,17 +233,32 @@ func TestUpdateDuringRandSleep(t *testing.T) {
 	}
 	g2 := &Group{
 		Rules: []Rule{
+			&rule1,
 			&rule2,
 		},
 	}
+	g2.metrics = newGroupMetrics(g2)
 	g.updateCh <- g2
 	time.Sleep(10 * time.Millisecond)
 	g.mu.RLock()
-	if len(g.Rules[0].(*AlertingRule).Labels) != 2 {
+	if len(g.Rules) != 2 {
+		t.Fatalf("expected to have updated rules")
+	}
+
+	if len(g.Rules[1].(*AlertingRule).Labels) != 2 {
 		t.Fatalf("expected to have updated labels")
 	}
 	g.mu.RUnlock()
 
+	metricsAfter := metrics.GetDefaultSet().ListMetricNames()
+	metricsRegistry := make(map[string]struct{}, len(metricsAfter))
+	for _, m := range metricsAfter {
+		if _, ok := metricsRegistry[m]; ok {
+			t.Fatalf("duplicate metric name %q", m)
+		}
+		metricsRegistry[m] = struct{}{}
+	}
+
 	g.Close()
 }
 

app/vmalert/rule/recording.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
@@ -45,6 +47,28 @@ type recordingRuleMetrics struct {
 	samples *utils.Gauge
 }
 
+func newRecordingRuleMetrics(set *metrics.Set, rr *RecordingRule) *recordingRuleMetrics {
+	rmr := &recordingRuleMetrics{}
+
+	labels := fmt.Sprintf(`recording=%q, group=%q, file=%q, id="%d"`, rr.Name, rr.GroupName, rr.File, rr.ID())
+	rmr.errors = utils.NewCounter(set, fmt.Sprintf(`vmalert_recording_rules_errors_total{%s}`, labels))
+	rmr.samples = utils.NewGauge(set, fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
+		func() float64 {
+			e := rr.state.getLast()
+			return float64(e.Samples)
+		})
+
+	return rmr
+}
+
+func (m *recordingRuleMetrics) close() {
+	if m == nil {
+		return
+	}
+	m.errors.Unregister()
+	m.samples.Unregister()
+}
+
 // String implements Stringer interface
 func (rr *RecordingRule) String() string {
 	return rr.Name
@@ -67,7 +91,6 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		GroupID:   group.ID(),
 		GroupName: group.Name,
 		File:      group.File,
-		metrics:   &recordingRuleMetrics{},
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:            group.Type.String(),
 			ApplyIntervalAsTimeFilter: setIntervalAsTimeFilter(group.Type.String(), cfg.Expr),
@@ -87,21 +110,18 @@ func NewRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 	rr.state = &ruleState{
 		entries: make([]StateEntry, entrySize),
 	}
+	rr.metrics = newRecordingRuleMetrics(group.metrics.set, rr)
 
-	labels := fmt.Sprintf(`recording=%q, group=%q, file=%q, id="%d"`, rr.Name, group.Name, group.File, rr.ID())
-	rr.metrics.errors = utils.GetOrCreateCounter(fmt.Sprintf(`vmalert_recording_rules_errors_total{%s}`, labels))
-	rr.metrics.samples = utils.GetOrCreateGauge(fmt.Sprintf(`vmalert_recording_rules_last_evaluation_samples{%s}`, labels),
-		func() float64 {
-			e := rr.state.getLast()
-			return float64(e.Samples)
-		})
 	return rr
 }
 
+func (rr *RecordingRule) registerMetrics(g *Group) {
+	rr.metrics = newRecordingRuleMetrics(g.metrics.set, rr)
+}
+
 // close unregisters rule metrics
-func (rr *RecordingRule) close() {
-	rr.metrics.errors.Unregister()
-	rr.metrics.samples.Unregister()
+func (rr *RecordingRule) unregisterMetrics() {
+	rr.metrics.close()
 }
 
 // execRange executes recording rule on the given time range similarly to Exec.

app/vmalert/rule/recording_test.go

@@ -7,8 +7,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
@@ -306,15 +307,12 @@ func TestRecordingRuleLimit_Failure(t *testing.T) {
 
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-
 		rule := &RecordingRule{Name: "job:foo",
 			state: &ruleState{entries: make([]StateEntry, 10)},
 			Labels: map[string]string{
 				"source": "test_limit",
 			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
+			metrics: getTestRecordingRuleMetrics(),
 		}
 		rule.q = fq
 
@@ -344,15 +342,12 @@ func TestRecordingRuleLimit_Success(t *testing.T) {
 
 		fq := &datasource.FakeQuerier{}
 		fq.Add(testMetrics...)
-
 		rule := &RecordingRule{Name: "job:foo",
 			state: &ruleState{entries: make([]StateEntry, 10)},
 			Labels: map[string]string{
 				"source": "test_limit",
 			},
-			metrics: &recordingRuleMetrics{
-				errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-			},
+			metrics: getTestRecordingRuleMetrics(),
 		}
 		rule.q = fq
 
@@ -366,16 +361,19 @@ func TestRecordingRuleLimit_Success(t *testing.T) {
 	f(-1)
 }
 
+func getTestRecordingRuleMetrics() *recordingRuleMetrics {
+	m := newRecordingRuleMetrics(metrics.NewSet(), &RecordingRule{})
+	return m
+}
+
 func TestRecordingRuleExec_Negative(t *testing.T) {
 	rr := &RecordingRule{
 		Name: "job:foo",
 		Labels: map[string]string{
 			"job": "test",
 		},
-		state: &ruleState{entries: make([]StateEntry, 10)},
-		metrics: &recordingRuleMetrics{
-			errors: utils.GetOrCreateCounter(`vmalert_recording_rules_errors_total{alertname="job:foo"}`),
-		},
+		state:   &ruleState{entries: make([]StateEntry, 10)},
+		metrics: getTestRecordingRuleMetrics(),
 	}
 	fq := &datasource.FakeQuerier{}
 	expErr := "connection reset by peer"

app/vmalert/rule/rule.go

@@ -27,9 +27,10 @@ type Rule interface {
 	// updateWith performs modification of current Rule
 	// with fields of the given Rule.
 	updateWith(Rule) error
-	// close performs the shutdown procedures for rule
-	// such as metrics unregister
-	close()
+	// unregister Rule metrics
+	unregisterMetrics()
+	// register Rule metrics with the given group
+	registerMetrics(g *Group)
 }
 
 var errDuplicate = errors.New("result contains metrics with the same labelset during evaluation. See https://docs.victoriametrics.com/vmalert/#series-with-the-same-labelset for details")

app/vmalert/templates/template.go

@@ -33,7 +33,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/formatutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 // go template execution fails when it's tree is empty
@@ -259,7 +259,7 @@ func templateFuncs() textTpl.FuncMap {
 
 		// parseDuration parses a duration string such as "1h" into the number of seconds it represents
 		"parseDuration": func(s string) (float64, error) {
-			d, err := promutils.ParseDuration(s)
+			d, err := timeutil.ParseDuration(s)
 			if err != nil {
 				return 0, err
 			}
@@ -268,7 +268,7 @@ func templateFuncs() textTpl.FuncMap {
 
 		// same with parseDuration but returns a time.Duration
 		"parseDurationTime": func(s string) (time.Duration, error) {
-			d, err := promutils.ParseDuration(s)
+			d, err := timeutil.ParseDuration(s)
 			if err != nil {
 				return 0, err
 			}

app/vmalert/utils/metrics.go

@@ -4,11 +4,13 @@ import "github.com/VictoriaMetrics/metrics"
 
 type namedMetric struct {
 	Name string
+
+	set *metrics.Set
 }
 
 // Unregister removes the metric by name from default registry
 func (nm namedMetric) Unregister() {
-	metrics.UnregisterMetric(nm.Name)
+	nm.set.UnregisterMetric(nm.Name)
 }
 
 // Gauge is a metrics.Gauge with Name
@@ -17,11 +19,11 @@ type Gauge struct {
 	*metrics.Gauge
 }
 
-// GetOrCreateGauge creates a new Gauge with the given name
-func GetOrCreateGauge(name string, f func() float64) *Gauge {
+// NewGauge creates a new Gauge with the given name
+func NewGauge(set *metrics.Set, name string, f func() float64) *Gauge {
 	return &Gauge{
-		namedMetric: namedMetric{Name: name},
-		Gauge:       metrics.GetOrCreateGauge(name, f),
+		namedMetric: namedMetric{Name: name, set: set},
+		Gauge:       set.NewGauge(name, f),
 	}
 }
 
@@ -31,11 +33,11 @@ type Counter struct {
 	*metrics.Counter
 }
 
-// GetOrCreateCounter creates a new Counter with the given name
-func GetOrCreateCounter(name string) *Counter {
+// NewCounter creates a new Counter with the given name
+func NewCounter(set *metrics.Set, name string) *Counter {
 	return &Counter{
-		namedMetric: namedMetric{Name: name},
-		Counter:     metrics.GetOrCreateCounter(name),
+		namedMetric: namedMetric{Name: name, set: set},
+		Counter:     set.NewCounter(name),
 	}
 }
 
@@ -45,10 +47,10 @@ type Summary struct {
 	*metrics.Summary
 }
 
-// GetOrCreateSummary creates a new Summary with the given name
-func GetOrCreateSummary(name string) *Summary {
+// NewSummary creates a new Summary with the given name
+func NewSummary(set *metrics.Set, name string) *Summary {
 	return &Summary{
-		namedMetric: namedMetric{Name: name},
-		Summary:     metrics.GetOrCreateSummary(name),
+		namedMetric: namedMetric{Name: name, set: set},
+		Summary:     set.NewSummary(name),
 	}
 }

app/vmalert/web_test.go

@@ -21,14 +21,18 @@ func TestHandler(t *testing.T) {
 	fq.Add(datasource.Metric{
 		Values: []float64{1}, Timestamps: []int64{0},
 	})
-	g := &rule.Group{
+	g := rule.NewGroup(config.Group{
 		Name:        "group",
 		File:        "rules.yaml",
 		Concurrency: 1,
-	}
-	ar := rule.NewAlertingRule(fq, g, config.Rule{ID: 0, Alert: "alert"})
-	rr := rule.NewRecordingRule(fq, g, config.Rule{ID: 1, Record: "record"})
-	g.Rules = []rule.Rule{ar, rr}
+		Rules: []config.Rule{
+			{ID: 0, Alert: "alert"},
+			{ID: 1, Record: "record"},
+		},
+	}, fq, 1*time.Minute, nil)
+	ar := g.Rules[0].(*rule.AlertingRule)
+	rr := g.Rules[1].(*rule.RecordingRule)
+
 	g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
 
 	m := &manager{groups: map[uint64]*rule.Group{
@@ -162,7 +166,7 @@ func TestHandler(t *testing.T) {
 
 		gotRuleWithUpdates := apiRuleWithUpdates{}
 		getResp(t, ts.URL+"/"+expRule.APILink(), &gotRuleWithUpdates, 200)
-		if gotRuleWithUpdates.StateUpdates == nil || len(gotRuleWithUpdates.StateUpdates) < 1 {
+		if len(gotRuleWithUpdates.StateUpdates) < 1 {
 			t.Fatalf("expected %+v to have state updates field not empty", gotRuleWithUpdates.StateUpdates)
 		}
 	})

app/vmalert/web_types_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -15,20 +16,22 @@ func TestRecordingToApi(t *testing.T) {
 	fq.Add(datasource.Metric{
 		Values: []float64{1}, Timestamps: []int64{0},
 	})
-	g := &rule.Group{
+	entriesLimit := 44
+	g := rule.NewGroup(config.Group{
 		Name:        "group",
 		File:        "rules.yaml",
 		Concurrency: 1,
-	}
-
-	entriesLimit := 44
-	rr := rule.NewRecordingRule(fq, g, config.Rule{
-		ID:                 1248,
-		Record:             "record_name",
-		Expr:               "up",
-		Labels:             map[string]string{"label": "value"},
-		UpdateEntriesLimit: &entriesLimit,
-	})
+		Rules: []config.Rule{
+			{
+				ID:                 1248,
+				Record:             "record_name",
+				Expr:               "up",
+				Labels:             map[string]string{"label": "value"},
+				UpdateEntriesLimit: &entriesLimit,
+			},
+		},
+	}, fq, 1*time.Minute, nil)
+	rr := g.Rules[0].(*rule.RecordingRule)
 
 	expectedRes := apiRule{
 		Name:           "record_name",

app/vmauth/target_url_test.go

@@ -98,6 +98,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		up, hc := ui.getURLPrefixAndHeaders(u, u.Host, nil)
 		if up == nil {
 			t.Fatalf("cannot match available backend: %s", err)
+			return
 		}
 		bu := up.getBackendURL()
 		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
@@ -309,6 +310,7 @@ func TestUserInfoGetBackendURL_SRV(t *testing.T) {
 		up, _ := ui.getURLPrefixAndHeaders(u, u.Host, nil)
 		if up == nil {
 			t.Fatalf("cannot match available backend: %s", err)
+			return
 		}
 		bu := up.getBackendURL()
 		target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)

app/vmctl/flags.go

@@ -344,7 +344,7 @@ var (
 		},
 		&cli.BoolFlag{
 			Name:  influxSkipDatabaseLabel,
-			Usage: "Wether to skip adding the label 'db' to timeseries.",
+			Usage: "Whether to skip adding the label 'db' to timeseries.",
 			Value: false,
 		},
 		&cli.BoolFlag{

app/vmctl/influx.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -37,7 +38,7 @@ func newInfluxProcessor(ic *influx.Client, im *vm.Importer, cc int, separator st
 	}
 }
 
-func (ip *influxProcessor) run() error {
+func (ip *influxProcessor) run(ctx context.Context) error {
 	series, err := ip.ic.Explore()
 	if err != nil {
 		return fmt.Errorf("explore query failed: %s", err)
@@ -67,7 +68,7 @@ func (ip *influxProcessor) run() error {
 		go func() {
 			defer wg.Done()
 			for s := range seriesCh {
-				if err := ip.do(s); err != nil {
+				if err := ip.do(ctx, s); err != nil {
 					errCh <- fmt.Errorf("request failed for %q.%q: %s", s.Measurement, s.Field, err)
 					return
 				}
@@ -110,7 +111,7 @@ const dbLabel = "db"
 const nameLabel = "__name__"
 const valueField = "value"
 
-func (ip *influxProcessor) do(s *influx.Series) error {
+func (ip *influxProcessor) do(ctx context.Context, s *influx.Series) error {
 	cr, err := ip.ic.FetchDataPoints(s)
 	if err != nil {
 		return fmt.Errorf("failed to fetch datapoints: %s", err)
@@ -163,7 +164,7 @@ func (ip *influxProcessor) do(s *influx.Series) error {
 			Timestamps: time,
 			Values:     values,
 		}
-		if err := ip.im.Input(&ts); err != nil {
+		if err := ip.im.Input(ctx, &ts); err != nil {
 			return err
 		}
 	}

app/vmctl/influx/influx.go

@@ -144,9 +144,9 @@ func timeFilter(start, end string) string {
 // by checking available (non-empty) tags, fields and measurements
 // which unique combination represents all possible
 // time series existing in database.
-// The explore required to reduce the load on influx
+// Explore is required to reduce the load on influx
 // by querying field of the exact time series at once,
-// instead of fetching all of the values over and over.
+// instead of fetching all the values over and over.
 //
 // May contain non-existing time series.
 func (c *Client) Explore() ([]*Series, error) {
@@ -340,10 +340,7 @@ func (c *Client) fieldsByMeasurement() (map[string][]string, error) {
 }
 
 func (c *Client) getSeries() ([]*Series, error) {
-	com := "show series"
-	if c.filterSeries != "" {
-		com = fmt.Sprintf("%s %s", com, c.filterSeries)
-	}
+	com := c.getSeriesCommand()
 	q := influx.Query{
 		Command:         com,
 		Database:        c.database,
@@ -389,6 +386,21 @@ func (c *Client) getSeries() ([]*Series, error) {
 	return result, nil
 }
 
+func (c *Client) getSeriesCommand() string {
+	com := "show series"
+	if c.filterSeries != "" {
+		com = fmt.Sprintf("%s %s", com, c.filterSeries)
+	}
+	if c.filterTime != "" {
+		joinStatement := " where "
+		if strings.Contains(strings.ToLower(com), joinStatement) {
+			joinStatement = " AND "
+		}
+		com = fmt.Sprintf("%s%s%s", com, joinStatement, c.filterTime)
+	}
+	return com
+}
+
 // getMeasurementTags get the tags for each measurement.
 // tags are placed in a map without values (similar to a set) for quick lookups:
 // {"measurement1": {"tag1", "tag2"}, "measurement2": {"tag3", "tag4"}}

app/vmctl/influx/influx_test.go

@@ -103,3 +103,26 @@ func TestTimeFilter(t *testing.T) {
 	// both start and end filters
 	f("2020-01-01T20:07:00Z", "2020-01-01T21:07:00Z", "time >= '2020-01-01T20:07:00Z' and time <= '2020-01-01T21:07:00Z'")
 }
+
+func TestGetSeriesCommand(t *testing.T) {
+	f := func(filterSeries, filterTime, expCommand string) {
+		t.Helper()
+
+		c := &Client{
+			filterTime:   filterTime,
+			filterSeries: filterSeries,
+		}
+		gotCommand := c.getSeriesCommand()
+		if gotCommand != expCommand {
+			t.Fatalf("unexpected command\ngot\n%s\nwant\n%s", gotCommand, expCommand)
+		}
+	}
+
+	f("", "", "show series")
+	f("from cpu", "", "show series from cpu")
+	f("from cpu where arch='x86'", "", "show series from cpu where arch='x86'")
+	f("", "time >= '2020-01-01T20:07:00Z'", "show series where time >= '2020-01-01T20:07:00Z'")
+	f("from cpu", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where time >= '2020-01-01T20:07:00Z'")
+	f("from cpu where arch='x86'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND time >= '2020-01-01T20:07:00Z'")
+	f("from cpu where arch='x86' AND hostname='host_2753'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND hostname='host_2753' AND time >= '2020-01-01T20:07:00Z'")
+}

app/vmctl/main.go

@@ -97,7 +97,7 @@ func main() {
 					}
 
 					otsdbProcessor := newOtsdbProcessor(otsdbClient, importer, c.Int(otsdbConcurrency), c.Bool(globalVerbose))
-					return otsdbProcessor.run()
+					return otsdbProcessor.run(ctx)
 				},
 			},
 			{
@@ -158,7 +158,7 @@ func main() {
 						c.Bool(influxSkipDatabaseLabel),
 						c.Bool(influxPrometheusMode),
 						c.Bool(globalVerbose))
-					return processor.run()
+					return processor.run(ctx)
 				},
 			},
 			{
@@ -261,7 +261,7 @@ func main() {
 						cc:        c.Int(promConcurrency),
 						isVerbose: c.Bool(globalVerbose),
 					}
-					return pp.run()
+					return pp.run(ctx)
 				},
 			},
 			{

app/vmctl/opentsdb.go

@@ -1,14 +1,16 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
 	"time"
 
+	"github.com/cheggaaa/pb/v3"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/opentsdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/cheggaaa/pb/v3"
 )
 
 type otsdbProcessor struct {
@@ -37,7 +39,7 @@ func newOtsdbProcessor(oc *opentsdb.Client, im *vm.Importer, otsdbcc int, verbos
 	}
 }
 
-func (op *otsdbProcessor) run() error {
+func (op *otsdbProcessor) run(ctx context.Context) error {
 	log.Println("Loading all metrics from OpenTSDB for filters: ", op.oc.Filters)
 	var metrics []string
 	for _, filter := range op.oc.Filters {
@@ -93,7 +95,7 @@ func (op *otsdbProcessor) run() error {
 			go func() {
 				defer wg.Done()
 				for s := range seriesCh {
-					if err := op.do(s); err != nil {
+					if err := op.do(ctx, s); err != nil {
 						errCh <- fmt.Errorf("couldn't retrieve series for %s : %s", metric, err)
 						return
 					}
@@ -148,7 +150,7 @@ func (op *otsdbProcessor) run() error {
 	return nil
 }
 
-func (op *otsdbProcessor) do(s queryObj) error {
+func (op *otsdbProcessor) do(ctx context.Context, s queryObj) error {
 	start := s.StartTime - s.Tr.Start
 	end := s.StartTime - s.Tr.End
 	data, err := op.oc.GetData(s.Series, s.Rt, start, end, op.oc.MsecsTime)
@@ -168,5 +170,5 @@ func (op *otsdbProcessor) do(s queryObj) error {
 		Timestamps: data.Timestamps,
 		Values:     data.Values,
 	}
-	return op.im.Input(&ts)
+	return op.im.Input(ctx, &ts)
 }

app/vmctl/prometheus.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"sync"
@@ -30,7 +31,7 @@ type prometheusProcessor struct {
 	isVerbose bool
 }
 
-func (pp *prometheusProcessor) run() error {
+func (pp *prometheusProcessor) run(ctx context.Context) error {
 	blocks, err := pp.cl.Explore()
 	if err != nil {
 		return fmt.Errorf("explore failed: %s", err)
@@ -59,7 +60,7 @@ func (pp *prometheusProcessor) run() error {
 		go func() {
 			defer wg.Done()
 			for br := range blockReadersCh {
-				if err := pp.do(br); err != nil {
+				if err := pp.do(ctx, br); err != nil {
 					errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
 					return
 				}
@@ -100,7 +101,7 @@ func (pp *prometheusProcessor) run() error {
 	return nil
 }
 
-func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
+func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error {
 	ss, err := pp.cl.Read(b)
 	if err != nil {
 		return fmt.Errorf("failed to read block: %s", err)
@@ -150,7 +151,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 			Timestamps: timestamps,
 			Values:     values,
 		}
-		if err := pp.im.Input(&ts); err != nil {
+		if err := pp.im.Input(ctx, &ts); err != nil {
 			return err
 		}
 	}

app/vmctl/prometheus_test.go

@@ -160,7 +160,7 @@ func TestPrometheusProcessorRun(t *testing.T) {
 				go tt.fields.closer(importer)
 			}
 
-			if err := pp.run(); (err != nil) != tt.wantErr {
+			if err := pp.run(context.Background()); (err != nil) != tt.wantErr {
 				t.Fatalf("run() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})

app/vmctl/remoteread.go

@@ -112,7 +112,7 @@ func (rrp *remoteReadProcessor) run(ctx context.Context) error {
 
 func (rrp *remoteReadProcessor) do(ctx context.Context, filter *remoteread.Filter) error {
 	return rrp.src.Read(ctx, filter, func(series *vm.TimeSeries) error {
-		if err := rrp.dst.Input(series); err != nil {
+		if err := rrp.dst.Input(ctx, series); err != nil {
 			return fmt.Errorf(
 				"failed to read data for time range start: %d, end: %d, %s",
 				filter.StartTimestampMs, filter.EndTimestampMs, err)

app/vmctl/utils/time.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 )
 
 const (
@@ -16,7 +16,7 @@ const (
 // ParseTime parses time in s string and returns time.Time object
 // if parse correctly or error if not
 func ParseTime(s string) (time.Time, error) {
-	msecs, err := promutils.ParseTimeMsec(s)
+	msecs, err := timeutil.ParseTimeMsec(s)
 	if err != nil {
 		return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err)
 	}

app/vmctl/vm/vm.go

@@ -69,7 +69,6 @@ type Importer struct {
 	user       string
 	password   string
 
-	close  chan struct{}
 	input  chan *TimeSeries
 	errors chan *ImportError
 
@@ -143,7 +142,6 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		user:       cfg.User,
 		password:   cfg.Password,
 		rl:         limiter.NewLimiter(cfg.RateLimit),
-		close:      make(chan struct{}),
 		input:      make(chan *TimeSeries, cfg.Concurrency*4),
 		errors:     make(chan *ImportError, cfg.Concurrency),
 		backoff:    cfg.Backoff,
@@ -189,10 +187,10 @@ func (im *Importer) Errors() chan *ImportError { return im.errors }
 
 // Input returns a channel for sending timeseries
 // that need to be imported
-func (im *Importer) Input(ts *TimeSeries) error {
+func (im *Importer) Input(ctx context.Context, ts *TimeSeries) error {
 	select {
-	case <-im.close:
-		return fmt.Errorf("importer is closed")
+	case <-ctx.Done():
+		return ctx.Err()
 	case im.input <- ts:
 		return nil
 	case err := <-im.errors:
@@ -207,7 +205,6 @@ func (im *Importer) Input(ts *TimeSeries) error {
 // and waits until they are finished
 func (im *Importer) Close() {
 	im.once.Do(func() {
-		close(im.close)
 		close(im.input)
 		im.wg.Wait()
 		close(im.errors)
@@ -220,24 +217,34 @@ func (im *Importer) startWorker(ctx context.Context, bar barpool.Bar, batchSize,
 	var waitForBatch time.Time
 	for {
 		select {
-		case <-im.close:
+		case <-ctx.Done():
 			for ts := range im.input {
 				ts = roundTimeseriesValue(ts, significantFigures, roundDigits)
 				batch = append(batch, ts)
+				exitErr := &ImportError{
+					Batch: batch,
+				}
+				retryableFunc := func() error { return im.Import(batch) }
+				_, err := im.backoff.Retry(ctx, retryableFunc)
+				if err != nil {
+					exitErr.Err = err
+				}
+				im.errors <- exitErr
 			}
-			exitErr := &ImportError{
-				Batch: batch,
-			}
-			retryableFunc := func() error { return im.Import(batch) }
-			_, err := im.backoff.Retry(ctx, retryableFunc)
-			if err != nil {
-				exitErr.Err = err
-			}
-			im.errors <- exitErr
 			return
 		case ts, ok := <-im.input:
 			if !ok {
-				continue
+				// drain all batches before exit
+				exitErr := &ImportError{
+					Batch: batch,
+				}
+				retryableFunc := func() error { return im.Import(batch) }
+				_, err := im.backoff.Retry(ctx, retryableFunc)
+				if err != nil {
+					exitErr.Err = err
+				}
+				im.errors <- exitErr
+				return
 			}
 			// init waitForBatch when first
 			// value was received

app/vminsert/common/streamaggr.go

@@ -36,6 +36,9 @@ var (
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignoring-old-samples")
 	streamAggrIgnoreFirstIntervals = flag.Int("streamAggr.ignoreFirstIntervals", 0, "Number of aggregation intervals to skip after the start. Increase this value if you observe incorrect aggregation results after restarts. It could be caused by receiving unordered delayed data from clients pushing data into the database. "+
 		"See https://docs.victoriametrics.com/stream-aggregation/#ignore-aggregation-intervals-on-start")
+	streamAggrEnableWindows = flag.Bool("streamAggr.enableWindows", false, "Enables aggregation within fixed windows for all aggregators. "+
+		"This allows to get more precise results, but impacts resource usage as it requires twice more memory to store two states. "+
+		"See https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows.")
 )
 
 var (
@@ -62,6 +65,7 @@ func CheckStreamAggrConfig() error {
 		DropInputLabels:      *streamAggrDropInputLabels,
 		IgnoreOldSamples:     *streamAggrIgnoreOldSamples,
 		IgnoreFirstIntervals: *streamAggrIgnoreFirstIntervals,
+		EnableWindows:        *streamAggrEnableWindows,
 	}
 	sas, err := streamaggr.LoadFromFile(*streamAggrConfig, pushNoop, opts, "global")
 	if err != nil {
@@ -78,7 +82,7 @@ func InitStreamAggr() {
 	saCfgReloaderStopCh = make(chan struct{})
 	if *streamAggrConfig == "" {
 		if *streamAggrDedupInterval > 0 {
-			deduplicator = streamaggr.NewDeduplicator(pushAggregateSeries, *streamAggrDedupInterval, *streamAggrDropInputLabels, "global")
+			deduplicator = streamaggr.NewDeduplicator(pushAggregateSeries, *streamAggrEnableWindows, *streamAggrDedupInterval, *streamAggrDropInputLabels, "global")
 		}
 		return
 	}

app/vminsert/prompush/push.go

@@ -2,6 +2,7 @@ package prompush
 
 import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
@@ -49,6 +50,7 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 	}
 	ctx.Reset(rowsLen)
 	rowsTotal := 0
+	hasRelabeling := relabel.HasRelabeling()
 	for i := range tss {
 		ts := &tss[i]
 		rowsTotal += len(ts.Samples)
@@ -57,7 +59,7 @@ func push(ctx *common.InsertCtx, tss []prompbmarshal.TimeSeries) {
 			label := &ts.Labels[j]
 			ctx.AddLabel(label.Name, label.Value)
 		}
-		if !ctx.TryPrepareLabels(false) {
+		if !ctx.TryPrepareLabels(hasRelabeling) {
 			continue
 		}
 		var metricNameRaw []byte

app/vmselect/netstorage/netstorage.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"sync"
 	"sync/atomic"
-	"time"
 	"unsafe"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -792,7 +791,7 @@ func LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames i
 	if err != nil {
 		return nil, err
 	}
-	labels, err := vmstorage.SearchLabelNamesWithFiltersOnTimeRange(qt, tfss, tr, maxLabelNames, sq.MaxMetrics, deadline.Deadline())
+	labels, err := vmstorage.SearchLabelNames(qt, tfss, tr, maxLabelNames, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search on time range: %w", err)
 	}
@@ -865,7 +864,7 @@ func LabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQue
 	if err != nil {
 		return nil, err
 	}
-	labelValues, err := vmstorage.SearchLabelValuesWithFiltersOnTimeRange(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
+	labelValues, err := vmstorage.SearchLabelValues(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
 		return nil, fmt.Errorf("error during label values search on time range for labelName=%q: %w", labelName, err)
 	}
@@ -1002,9 +1001,7 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 
 	sr := getStorageSearch()
 	defer putStorageSearch(sr)
-	startTime := time.Now()
 	sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
-	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
 	workCh := make(chan *exportWork, gomaxprocs*8)
@@ -1142,9 +1139,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	defer vmstorage.WG.Done()
 
 	sr := getStorageSearch()
-	startTime := time.Now()
 	maxSeriesCount := sr.Init(qt, vmstorage.Storage, tfss, tr, sq.MaxMetrics, deadline.Deadline())
-	indexSearchDuration.UpdateDuration(startTime)
 	type blockRefs struct {
 		brs []blockRef
 	}
@@ -1296,8 +1291,6 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	return &rss, nil
 }
 
-var indexSearchDuration = metrics.NewHistogram(`vm_index_search_duration_seconds`)
-
 type blockRef struct {
 	partRef storage.PartRef
 	addr    tmpBlockAddr

app/vmselect/prometheus/prometheus.go

@@ -39,7 +39,7 @@ var (
 		"It can be overridden on per-query basis via latency_offset arg. "+
 		"Too small value can result in incomplete last points for query results")
 	maxQueryLen = flagutil.NewBytes("search.maxQueryLen", 16*1024, "The maximum search query length in bytes")
-	maxLookback = flag.Duration("search.maxLookback", 0, "Synonym to -search.lookback-delta from Prometheus. "+
+	maxLookback = flag.Duration("search.maxLookback", 0, "Synonym to -query.lookback-delta from Prometheus. "+
 		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+
 		"See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons")
 	maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+

app/vmselect/promql/rollup.go

@@ -373,9 +373,17 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 	func(values []float64, timestamps []int64), []*rollupConfig, error) {
 	preFunc := func(_ []float64, _ []int64) {}
 	funcName = strings.ToLower(funcName)
+
+	// window > lookbackDelta could result in negative delta.
+	// See issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8342
+	stalenessInterval := lookbackDelta
+	if stalenessInterval != 0 && stalenessInterval < window {
+		stalenessInterval = window
+	}
+
 	if rollupFuncsRemoveCounterResets[funcName] {
 		preFunc = func(values []float64, timestamps []int64) {
-			removeCounterResets(values, timestamps, lookbackDelta)
+			removeCounterResets(values, timestamps, stalenessInterval)
 		}
 	}
 	samplesScannedPerCall := rollupFuncsSamplesScannedPerCall[funcName]
@@ -487,7 +495,7 @@ func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start
 			if rollupFuncsRemoveCounterResets[aggrFuncName] {
 				// There is no need to save the previous preFunc, since it is either empty or the same.
 				preFunc = func(values []float64, timestamps []int64) {
-					removeCounterResets(values, timestamps, lookbackDelta)
+					removeCounterResets(values, timestamps, stalenessInterval)
 				}
 			}
 			rf := rollupAggrFuncs[aggrFuncName]

app/vmselect/vmui/asset-manifest.json

@@ -1,13 +0,0 @@
-{
-  "files": {
-    "main.css": "./static/css/main.af583aad.css",
-    "main.js": "./static/js/main.1413b18d.js",
-    "static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
-    "static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
-    "index.html": "./index.html"
-  },
-  "entrypoints": [
-    "static/css/main.af583aad.css",
-    "static/js/main.1413b18d.js"
-  ]
-}

app/vmselect/vmui/assets/vendor-D1GxaB_c.css

@@ -0,0 +1 @@
+.uplot,.uplot *,.uplot *:before,.uplot *:after{box-sizing:border-box}.uplot{font-family:system-ui,-apple-system,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol,"Noto Color Emoji";line-height:1.5;width:min-content}.u-title{text-align:center;font-size:18px;font-weight:700}.u-wrap{position:relative;-webkit-user-select:none;user-select:none}.u-over,.u-under{position:absolute}.u-under{overflow:hidden}.uplot canvas{display:block;position:relative;width:100%;height:100%}.u-axis{position:absolute}.u-legend{font-size:14px;margin:auto;text-align:center}.u-inline{display:block}.u-inline *{display:inline-block}.u-inline tr{margin-right:16px}.u-legend th{font-weight:600}.u-legend th>*{vertical-align:middle;display:inline-block}.u-legend .u-marker{width:1em;height:1em;margin-right:4px;background-clip:padding-box!important}.u-inline.u-live th:after{content:":";vertical-align:middle}.u-inline:not(.u-live) .u-value{display:none}.u-series>*{padding:4px}.u-series th{cursor:pointer}.u-legend .u-off>*{opacity:.3}.u-select{background:#00000012;position:absolute;pointer-events:none}.u-cursor-x,.u-cursor-y{position:absolute;left:0;top:0;pointer-events:none;will-change:transform}.u-hz .u-cursor-x,.u-vt .u-cursor-y{height:100%;border-right:1px dashed #607D8B}.u-hz .u-cursor-y,.u-vt .u-cursor-x{width:100%;border-bottom:1px dashed #607D8B}.u-cursor-pt{position:absolute;top:0;left:0;border-radius:50%;border:0 solid;pointer-events:none;will-change:transform;background-clip:padding-box!important}.u-axis.u-off,.u-select.u-off,.u-cursor-x.u-off,.u-cursor-y.u-off,.u-cursor-pt.u-off{display:none}

app/vmselect/vmui/index.html

@@ -1 +1,58 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.1413b18d.js"></script><link href="./static/css/main.af583aad.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <link rel="icon" href="./favicon.svg"/>
+  <link rel="apple-touch-icon" href="./favicon.svg"/>
+  <link rel="mask-icon" href="./favicon.svg" color="#000000">
+
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
+  <meta name="theme-color" content="#000000"/>
+  <meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/>
+  <!--
+    manifest.json provides metadata used when your web app is installed on a
+    user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
+  -->
+  <link rel="manifest" href="./manifest.json"/>
+  <!--
+    Notice the use of in the tags above.
+    It will be replaced with the URL of the `public` folder during the build.
+    Only files inside the `public` folder can be referenced from the HTML.
+
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
+    work correctly both with client-side routing and a non-root public URL.
+    Learn how to configure a non-root public URL by running `npm run build`.
+  -->
+  <title>vmui</title>
+  <script src="./dashboards/index.js" type="module"></script>
+
+  <meta name="twitter:card" content="summary">
+  <meta name="twitter:title" content="UI for VictoriaMetrics">
+  <meta name="twitter:site" content="@https://victoriametrics.com/">
+  <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <meta name="twitter:image" content="./preview.jpg">
+
+  <meta property="og:type" content="website">
+  <meta property="og:title" content="UI for VictoriaMetrics">
+  <meta property="og:url" content="https://victoriametrics.com/">
+  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+  <script type="module" crossorigin src="./assets/index-DzehQsnZ.js"></script>
+  <link rel="modulepreload" crossorigin href="./assets/vendor-DojlIpLz.js">
+  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-Cqbobgy7.css">
+</head>
+<body>
+<noscript>You need to enable JavaScript to run this app.</noscript>
+<div id="root"></div>
+<!--
+  This HTML file is a template.
+  If you open it directly in the browser, you will see an empty page.
+
+  You can add webfonts, meta tags, or analytics to this file.
+  The build step will place the bundled scripts into the <body> tag.
+
+  To begin the development, run `npm start` or `yarn start`.
+  To create a production bundle, use `npm run build` or `yarn build`.
+-->
+</body>
+</html>

app/vmselect/vmui/static/js/main.1413b18d.js.LICENSE.txt

@@ -1,38 +0,0 @@
-/*!
-	Copyright (c) 2018 Jed Watson.
-	Licensed under the MIT License (MIT), see
-	http://jedwatson.github.io/classnames
-*/
-
-/**
- * @remix-run/router v1.19.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router DOM v6.26.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */
-
-/**
- * React Router v6.26.2
- *
- * Copyright (c) Remix Software Inc.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE.md file in the root directory of this source tree.
- *
- * @license MIT
- */

app/vmstorage/main.go

@@ -71,6 +71,11 @@ var (
 		"See https://docs.victoriametrics.com/single-server-victoriametrics/#cache-tuning")
 	cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+
 		"See https://docs.victoriametrics.com/single-server-victoriametrics/#cache-tuning")
+
+	disablePerDayIndex = flag.Bool("disablePerDayIndex", false, "Disable per-day index and use global index for all searches. "+
+		"This may improve performance and decrease disk space usage for the use cases with fixed set of timeseries scattered across a "+
+		"big time range (for example, when loading years of historical data). "+
+		"See https://docs.victoriametrics.com/single-server-victoriametrics/#index-tuning")
 )
 
 // CheckTimeRange returns true if the given tr is denied for querying.
@@ -110,7 +115,14 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	logger.Infof("opening storage at %q with -retentionPeriod=%s", *DataPath, retentionPeriod)
 	startTime := time.Now()
 	WG = syncwg.WaitGroup{}
-	strg := storage.MustOpenStorage(*DataPath, retentionPeriod.Duration(), *maxHourlySeries, *maxDailySeries)
+
+	opts := storage.OpenOptions{
+		Retention:          retentionPeriod.Duration(),
+		MaxHourlySeries:    *maxHourlySeries,
+		MaxDailySeries:     *maxDailySeries,
+		DisablePerDayIndex: *disablePerDayIndex,
+	}
+	strg := storage.MustOpenStorage(*DataPath, opts)
 	Storage = strg
 	initStaleSnapshotsRemover(strg)
 
@@ -189,20 +201,19 @@ func SearchMetricNames(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr st
 	return metricNames, err
 }
 
-// SearchLabelNamesWithFiltersOnTimeRange searches for tag keys matching the given tfss on tr.
-func SearchLabelNamesWithFiltersOnTimeRange(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxTagKeys, maxMetrics int, deadline uint64) ([]string, error) {
+// SearchLabelNames searches for tag keys matching the given tfss on tr.
+func SearchLabelNames(qt *querytracer.Tracer, tfss []*storage.TagFilters, tr storage.TimeRange, maxTagKeys, maxMetrics int, deadline uint64) ([]string, error) {
 	WG.Add(1)
-	labelNames, err := Storage.SearchLabelNamesWithFiltersOnTimeRange(qt, tfss, tr, maxTagKeys, maxMetrics, deadline)
+	labelNames, err := Storage.SearchLabelNames(qt, tfss, tr, maxTagKeys, maxMetrics, deadline)
 	WG.Done()
 	return labelNames, err
 }
 
-// SearchLabelValuesWithFiltersOnTimeRange searches for label values for the given labelName, tfss and tr.
-func SearchLabelValuesWithFiltersOnTimeRange(qt *querytracer.Tracer, labelName string, tfss []*storage.TagFilters,
-	tr storage.TimeRange, maxLabelValues, maxMetrics int, deadline uint64,
-) ([]string, error) {
+// SearchLabelValues searches for label values for the given labelName, tfss and
+// tr.
+func SearchLabelValues(qt *querytracer.Tracer, labelName string, tfss []*storage.TagFilters, tr storage.TimeRange, maxLabelValues, maxMetrics int, deadline uint64) ([]string, error) {
 	WG.Add(1)
-	labelValues, err := Storage.SearchLabelValuesWithFiltersOnTimeRange(qt, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
+	labelValues, err := Storage.SearchLabelValues(qt, labelName, tfss, tr, maxLabelValues, maxMetrics, deadline)
 	WG.Done()
 	return labelValues, err
 }

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.23.5 AS build-web-stage
+FROM golang:1.24.0 AS build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.21.2
+FROM alpine:3.21.3
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/.env

@@ -1 +1 @@
-FAST_REFRESH=false
+VITE_APP_TYPE=victoriametrics

app/vmui/packages/vmui/.env.victorialogs

@@ -0,0 +1 @@
+VITE_APP_TYPE=victorialogs

app/vmui/packages/vmui/.env.vmanomaly

@@ -0,0 +1 @@
+VITE_APP_TYPE=vmanomaly

app/vmui/packages/vmui/.eslintrc.js

@@ -1,48 +0,0 @@
-// eslint-disable-next-line no-undef
-module.exports = {
-  "env": {
-    "browser": true,
-    "es2021": true
-  },
-  "extends": [
-    "eslint:recommended",
-    "plugin:react/recommended",
-    "plugin:@typescript-eslint/recommended"
-  ],
-  "parser": "@typescript-eslint/parser",
-  "parserOptions": {
-    "ecmaFeatures": { "jsx": true },
-    "ecmaVersion": 12,
-    "sourceType": "module"
-  },
-  "plugins": [
-    "react",
-    "@typescript-eslint"
-  ],
-  "rules": {
-    "@typescript-eslint/no-unused-vars": ["warn", { "varsIgnorePattern": "^_" }],
-    "react/jsx-closing-bracket-location": [1, "line-aligned"],
-    "react/jsx-max-props-per-line":[1, { "maximum": 1 }],
-    "react/jsx-first-prop-new-line": [1, "multiline"],
-    "object-curly-spacing": [2, "always"],
-    "indent": ["error", 2, { "SwitchCase": 1 }],
-    "linebreak-style": ["error", "unix"],
-    "quotes": ["error", "double"],
-    "semi": ["error", "always"],
-    "react/prop-types": 0
-  },
-  "settings": {
-    "react": {
-      "pragma": "React",  // Pragma to use, default to "React"
-      "version": "detect"
-    },
-    "linkComponents": [
-      // Components used as alternatives to <a> for linking, eg. <Link to={ url } />
-      "Hyperlink",
-      {
-        "name": "Link", "linkAttribute": "to"
-      }
-    ]
-  }
-};
-

app/vmui/packages/vmui/config-overrides.js

@@ -1,42 +0,0 @@
-/* eslint-disable */
-const { override, addExternalBabelPlugin, addWebpackAlias, addWebpackPlugin } = require("customize-cra");
-const webpack = require("webpack");
-const fs = require('fs');
-const path = require('path');
-
-// This will replace the default check
-const pathIndexHTML = (() => {
-  switch (process.env.REACT_APP_TYPE) {
-    case 'logs':
-      return 'src/html/victorialogs.html';
-    case 'anomaly':
-      return 'src/html/vmanomaly.html';
-    default:
-      return 'src/html/victoriametrics.html';
-  }
-})();
-const fileContent = fs.readFileSync(path.resolve(__dirname, pathIndexHTML), 'utf8');
-fs.writeFileSync(path.resolve(__dirname, 'public/index.html'), fileContent);
-
-module.exports = override(
-  addExternalBabelPlugin("@babel/plugin-proposal-nullish-coalescing-operator"),
-  addWebpackAlias({
-    "react": "preact/compat",
-    "react-dom/test-utils": "preact/test-utils",
-    "react-dom": "preact/compat", // Must be below test-utils
-    "react/jsx-runtime": "preact/jsx-runtime"
-  }),
-  addWebpackPlugin(
-    new webpack.NormalModuleReplacementPlugin(
-      /\.\/App/,
-      function (resource) {
-        if (process.env.REACT_APP_TYPE === "logs") {
-          resource.request = "./AppLogs";
-        }
-        if (process.env.REACT_APP_TYPE === "anomaly") {
-          resource.request = "./AppAnomaly";
-        }
-      }
-    )
-  )
-);

app/vmui/packages/vmui/config/plugins/dynamicIndexHtml.ts

@@ -0,0 +1,23 @@
+import { readFile } from "fs/promises";
+import { IndexHtmlTransform } from "vite";
+
+/**
+ * Vite plugin to dynamically load index.html based on the current mode.
+ * If a specific mode-based index file (e.g., index.victorialogs.html) exists, it is used.
+ * Otherwise, the default index.html is loaded.
+ */
+export default function dynamicIndexHtmlPlugin({ mode }) {
+  return {
+    name: "vm-dynamic-index-html",
+    transformIndexHtml: {
+      order: "pre",
+      handler: async () => {
+        try {
+          return await readFile(`./index.${mode}.html`, "utf8");
+        } catch (error) {
+          return await readFile("./index.html", "utf8");
+        }
+      }
+    } as IndexHtmlTransform
+  };
+}

app/vmui/packages/vmui/eslint.config.js

@@ -0,0 +1,90 @@
+import react from "eslint-plugin-react";
+import typescriptEslint from "@typescript-eslint/eslint-plugin";
+import globals from "globals";
+import tsParser from "@typescript-eslint/parser";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import js from "@eslint/js";
+import { FlatCompat } from "@eslint/eslintrc";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all
+});
+
+export default [...compat.extends(
+  "eslint:recommended",
+  "plugin:react/recommended",
+  "plugin:@typescript-eslint/recommended",
+), {
+  plugins: {
+    react,
+    "@typescript-eslint": typescriptEslint,
+  },
+
+  languageOptions: {
+    globals: {
+      ...globals.browser,
+    },
+
+    parser: tsParser,
+    ecmaVersion: 12,
+    sourceType: "module",
+
+    parserOptions: {
+      ecmaFeatures: {
+        jsx: true,
+      },
+    },
+  },
+
+  settings: {
+    react: {
+      pragma: "React",
+      version: "detect",
+    },
+
+    linkComponents: ["Hyperlink", {
+      name: "Link",
+      linkAttribute: "to",
+    }],
+  },
+
+  rules: {
+    "@typescript-eslint/no-unused-expressions": ["error", {
+      allowShortCircuit: true,
+      allowTernary: true
+    }],
+
+    "@typescript-eslint/no-unused-vars": ["warn", {
+      "argsIgnorePattern": "^_",
+      "caughtErrors": "none",
+      "caughtErrorsIgnorePattern": "^_",
+      "destructuredArrayIgnorePattern": "^_",
+      "varsIgnorePattern": "^_",
+      "ignoreRestSiblings": true
+    }],
+
+    "react/jsx-closing-bracket-location": [1, "line-aligned"],
+
+    "react/jsx-max-props-per-line": [1, {
+      maximum: 1,
+    }],
+
+    "react/jsx-first-prop-new-line": [1, "multiline"],
+    "object-curly-spacing": [2, "always"],
+
+    indent: ["error", 2, {
+      SwitchCase: 1,
+    }],
+
+    "linebreak-style": ["error", "unix"],
+    quotes: ["error", "double"],
+    semi: ["error", "always"],
+    "react/prop-types": 0,
+
+  },
+}];

app/vmui/packages/vmui/index.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
   <meta charset="utf-8"/>
-  <link rel="icon" href="%PUBLIC_URL%/favicon.svg" />
-  <link rel="apple-touch-icon" href="%PUBLIC_URL%/favicon.svg" />
-  <link rel="mask-icon" href="%PUBLIC_URL%/favicon.svg" color="#000000">
+  <link rel="icon" href="/favicon.svg"/>
+  <link rel="apple-touch-icon" href="/favicon.svg"/>
+  <link rel="mask-icon" href="/favicon.svg" color="#000000">
 
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
   <meta name="theme-color" content="#000000"/>
@@ -13,24 +13,24 @@
     manifest.json provides metadata used when your web app is installed on a
     user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
   -->
-  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <link rel="manifest" href="/manifest.json"/>
   <!--
-    Notice the use of %PUBLIC_URL% in the tags above.
+    Notice the use of in the tags above.
     It will be replaced with the URL of the `public` folder during the build.
     Only files inside the `public` folder can be referenced from the HTML.
 
-    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
     work correctly both with client-side routing and a non-root public URL.
     Learn how to configure a non-root public URL by running `npm run build`.
   -->
   <title>vmui</title>
-  <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
+  <script src="/dashboards/index.js" type="module"></script>
 
   <meta name="twitter:card" content="summary">
   <meta name="twitter:title" content="UI for VictoriaMetrics">
   <meta name="twitter:site" content="@https://victoriametrics.com/">
   <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:image" content="/preview.jpg">
 
   <meta property="og:type" content="website">
   <meta property="og:title" content="UI for VictoriaMetrics">
@@ -50,5 +50,6 @@
   To begin the development, run `npm start` or `yarn start`.
   To create a production bundle, use `npm run build` or `yarn build`.
 -->
+<script type="module" src="/src/index.tsx"></script>
 </body>
 </html>

app/vmui/packages/vmui/index.victorialogs.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
   <meta charset="utf-8"/>
-  <link rel="icon" href="%PUBLIC_URL%/favicon.svg" />
-  <link rel="apple-touch-icon" href="%PUBLIC_URL%/favicon.svg" />
-  <link rel="mask-icon" href="%PUBLIC_URL%/favicon.svg" color="#000000">
+  <link rel="icon" href="/favicon.svg" />
+  <link rel="apple-touch-icon" href="/favicon.svg" />
+  <link rel="mask-icon" href="/favicon.svg" color="#000000">
 
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
   <meta name="theme-color" content="#000000"/>
@@ -13,13 +13,13 @@
     manifest.json provides metadata used when your web app is installed on a
     user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
   -->
-  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <link rel="manifest" href="/manifest.json"/>
   <!--
-    Notice the use of %PUBLIC_URL% in the tags above.
+    Notice the use of in the tags above.
     It will be replaced with the URL of the `public` folder during the build.
     Only files inside the `public` folder can be referenced from the HTML.
 
-    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
     work correctly both with client-side routing and a non-root public URL.
     Learn how to configure a non-root public URL by running `npm run build`.
   -->
@@ -29,7 +29,7 @@
   <meta name="twitter:title" content="UI for VictoriaLogs">
   <meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/">
   <meta name="twitter:description" content="Explore your log data with VictoriaLogs UI">
-  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:image" content="/preview.jpg">
 
   <meta property="og:type" content="website">
   <meta property="og:title" content="UI for VictoriaLogs">
@@ -49,5 +49,6 @@
   To begin the development, run `npm start` or `yarn start`.
   To create a production bundle, use `npm run build` or `yarn build`.
 -->
+<script type="module" src="/src/index.tsx"></script>
 </body>
 </html>

app/vmui/packages/vmui/index.vmanomaly.html

@@ -2,9 +2,9 @@
 <html lang="en">
 <head>
   <meta charset="utf-8"/>
-  <link rel="icon" href="%PUBLIC_URL%/favicon.svg" />
-  <link rel="apple-touch-icon" href="%PUBLIC_URL%/favicon.svg" />
-  <link rel="mask-icon" href="%PUBLIC_URL%/favicon.svg" color="#000000">
+  <link rel="icon" href="/favicon.svg" />
+  <link rel="apple-touch-icon" href="/favicon.svg" />
+  <link rel="mask-icon" href="/favicon.svg" color="#000000">
 
   <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
   <meta name="theme-color" content="#000000"/>
@@ -13,13 +13,13 @@
     manifest.json provides metadata used when your web app is installed on a
     user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
   -->
-  <link rel="manifest" href="%PUBLIC_URL%/manifest.json"/>
+  <link rel="manifest" href="/manifest.json"/>
   <!--
-    Notice the use of %PUBLIC_URL% in the tags above.
+    Notice the use of in the tags above.
     It will be replaced with the URL of the `public` folder during the build.
     Only files inside the `public` folder can be referenced from the HTML.
 
-    Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+    Unlike "/favicon.ico" or "favicon.ico", "/favicon.ico" will
     work correctly both with client-side routing and a non-root public URL.
     Learn how to configure a non-root public URL by running `npm run build`.
   -->
@@ -29,7 +29,7 @@
   <meta name="twitter:title" content="UI for VictoriaMetrics Anomaly Detection">
   <meta name="twitter:site" content="@https://victoriametrics.com/products/enterprise/anomaly-detection/">
   <meta name="twitter:description" content="Detect anomalies in your metrics with VictoriaMetrics Anomaly Detection UI">
-  <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+  <meta name="twitter:image" content="/preview.jpg">
 
   <meta property="og:type" content="website">
   <meta property="og:title" content="UI for VictoriaMetrics Anomaly Detection">
@@ -49,5 +49,6 @@
   To begin the development, run `npm start` or `yarn start`.
   To create a production bundle, use `npm run build` or `yarn build`.
 -->
+<script type="module" src="/src/index.tsx"></script>
 </body>
 </html>

app/vmui/packages/vmui/package.json

@@ -3,50 +3,40 @@
   "version": "0.1.0",
   "private": true,
   "homepage": "./",
+  "type": "module",
   "dependencies": {
     "@types/lodash.debounce": "^4.0.9",
     "@types/lodash.get": "^4.4.9",
-    "@types/lodash.throttle": "^4.1.9",
-    "@types/node": "^22.5.4",
-    "@types/qs": "^6.9.15",
-    "@types/react-input-mask": "^3.0.5",
+    "@types/qs": "^6.9.18",
+    "@types/react": "^19.0.8",
+    "@types/react-input-mask": "^3.0.6",
     "@types/react-router-dom": "^5.3.3",
-    "@types/webpack-env": "^1.18.5",
     "classnames": "^2.5.1",
     "dayjs": "^1.11.13",
     "lodash.debounce": "^4.0.8",
     "lodash.get": "^4.4.2",
-    "lodash.throttle": "^4.1.1",
-    "marked": "^14.1.2",
-    "marked-emoji": "^1.4.2",
-    "preact": "^10.23.2",
-    "qs": "^6.13.0",
+    "marked": "^15.0.6",
+    "marked-emoji": "^1.4.3",
+    "preact": "^10.25.4",
+    "qs": "^6.14.0",
     "react-input-mask": "^2.0.4",
-    "react-router-dom": "^6.26.2",
-    "sass": "^1.78.0",
-    "source-map-explorer": "^2.5.3",
-    "typescript": "~4.6.2",
-    "uplot": "^1.6.30",
-    "web-vitals": "^4.2.3"
+    "react-router-dom": "^7.1.5",
+    "uplot": "^1.6.31",
+    "vite": "^6.0.11",
+    "web-vitals": "^4.2.4"
   },
   "scripts": {
     "prestart": "npm run copy-metricsql-docs",
-    "start": "react-app-rewired start",
-    "start:logs": "cross-env REACT_APP_TYPE=logs npm run start",
-    "start:anomaly": "cross-env REACT_APP_TYPE=anomaly npm run start",
-    "build": "GENERATE_SOURCEMAP=false react-app-rewired build",
-    "build:logs": "cross-env REACT_APP_TYPE=logs npm run build",
-    "build:anomaly": "cross-env REACT_APP_TYPE=anomaly npm run build",
-    "lint": "eslint src --ext tsx,ts",
-    "lint:fix": "eslint src --ext tsx,ts --fix",
-    "analyze": "source-map-explorer 'build/static/js/*.js'",
-    "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true"
-  },
-  "eslintConfig": {
-    "extends": [
-      "react-app",
-      "react-app/jest"
-    ]
+    "start": "vite",
+    "start:logs": "vite --mode victorialogs",
+    "start:anomaly": "vite --mode vmanomaly",
+    "build": "vite build",
+    "build:logs": "vite build --mode victorialogs",
+    "build:anomaly": "vite build --mode vmanomaly",
+    "lint": "eslint 'src/**/*.{ts,tsx}'",
+    "lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
+    "copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
+    "preview": "vite preview"
   },
   "browserslist": {
     "production": [
@@ -61,26 +51,24 @@
     ]
   },
   "devDependencies": {
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.16.7",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6",
     "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
-    "@types/react-dom": "^18.3.0",
-    "@typescript-eslint/eslint-plugin": "^5.15.0",
-    "@typescript-eslint/parser": "^5.15.0",
+    "@eslint/eslintrc": "^3.2.0",
+    "@eslint/js": "^9.19.0",
+    "@preact/preset-vite": "^2.10.1",
+    "@types/node": "^22.13.0",
+    "@typescript-eslint/eslint-plugin": "^8.22.0",
+    "@typescript-eslint/parser": "^8.22.0",
     "cross-env": "^7.0.3",
-    "customize-cra": "^1.0.0",
-    "eslint": "^8.44.0",
-    "eslint-config-react-app": "^7.0.1",
-    "eslint-plugin-react": "^7.36.1",
-    "http-proxy-middleware": "^3.0.2",
-    "react-app-rewired": "^2.2.1",
-    "webpack": "^5.94.0"
-  },
-  "overrides": {
-    "react-app-rewired": {
-      "nth-check": "^2.0.1"
-    },
-    "css-select": {
-      "nth-check": "^2.0.1"
-    }
+    "eslint": "^9.19.0",
+    "eslint-plugin-react": "^7.37.4",
+    "globals": "^15.14.0",
+    "http-proxy-middleware": "^3.0.3",
+    "postcss": "^8.5.1",
+    "rollup-plugin-visualizer": "^5.14.0",
+    "sass": "^1.83.4",
+    "sass-embedded": "^1.83.4",
+    "typescript": "^5.7.3",
+    "webpack": "^5.97.1"
   }
 }

app/vmui/packages/vmui/src/AppAnomaly.tsx

@@ -38,4 +38,4 @@ const AppAnomaly: FC = () => {
   </>;
 };
 
-export default AppAnomaly;
+export default AppAnomaly;

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -1,7 +1,6 @@
-import React, { FC, useCallback, useEffect, useRef, useState } from "preact/compat";
+import React, { FC, useCallback, useEffect, useRef, useState, createPortal } from "preact/compat";
 import { MouseEvent as ReactMouseEvent } from "react";
 import useEventListener from "../../../hooks/useEventListener";
-import ReactDOM from "react-dom";
 import classNames from "classnames";
 import uPlot from "uplot";
 import Button from "../../Main/Button/Button";
@@ -49,7 +48,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
     onClose && onClose(id);
   };
 
-  const handleMouseDown = (e: ReactMouseEvent) => {
+  const handleMouseDown = (e: ReactMouseEvent<HTMLButtonElement>) => {
     setMoved(true);
     setMoving(true);
     const { clientX, clientY } = e;
@@ -107,7 +106,7 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
 
   if (!u) return null;
 
-  return ReactDOM.createPortal((
+  return createPortal((
     <div
       className={classNames({
         "vm-chart-tooltip": true,

app/vmui/packages/vmui/src/components/Chart/Heatmap/LegendHeatmap/LegendHeatmap.tsx

@@ -1,22 +1,18 @@
 import React, { FC, useEffect, useMemo, useState } from "preact/compat";
 import { gradMetal16 } from "../../../../utils/uplot";
-import { SeriesItem, LegendItemType } from "../../../../types";
 import "./style.scss";
-import LegendItem from "../../Line/Legend/LegendItem/LegendItem";
 import { ChartTooltipProps } from "../../ChartTooltip/ChartTooltip";
 
 interface LegendHeatmapProps {
   min: number
   max: number
   legendValue: ChartTooltipProps | null,
-  series: SeriesItem[]
 }
 
 const LegendHeatmap: FC<LegendHeatmapProps> = ({
   min,
   max,
   legendValue,
-  series
 }) => {
 
   const [percent, setPercent] = useState(0);
@@ -54,13 +50,6 @@ const LegendHeatmap: FC<LegendHeatmapProps> = ({
         <div className="vm-legend-heatmap__value">{minFormat}</div>
         <div className="vm-legend-heatmap__value">{maxFormat}</div>
       </div>
-      {series[1] && (
-        <LegendItem
-          key={series[1]?.label}
-          legend={series[1] as LegendItemType}
-          isHeatmap
-        />
-      )}
     </div>
   );
 };

app/vmui/packages/vmui/src/components/Chart/Line/Legend/Legend.tsx

@@ -1,8 +1,15 @@
-import React, { FC, useMemo } from "preact/compat";
+import React, { FC } from "preact/compat";
 import { LegendItemType } from "../../../../types";
-import LegendItem from "./LegendItem/LegendItem";
 import Accordion from "../../../Main/Accordion/Accordion";
 import "./style.scss";
+import LegendGroup from "./LegendGroup";
+import { useLegendGroup } from "./hooks/useLegendGroup";
+import { useGroupSeries } from "./hooks/useGroupSeries";
+
+export type QueryGroup = {
+  group: number | string;
+  items: LegendItemType[]
+}
 
 interface LegendProps {
   labels: LegendItemType[];
@@ -12,42 +19,34 @@ interface LegendProps {
 }
 
 const Legend: FC<LegendProps> = ({ labels, query, isAnomalyView, onChange }) => {
-  const groups = useMemo(() => {
-    return Array.from(new Set(labels.map(l => l.group)));
-  }, [labels]);
-  const showQueryNum = groups.length > 1;
+  const { groupByLabel } = useLegendGroup();
+  const groupSeries = useGroupSeries({ labels, query, groupByLabel });
 
   return <>
     <div className="vm-legend">
-      {groups.map((group) => (
-        <div
-          className="vm-legend-group"
-          key={group}
-        >
-          <Accordion
-            defaultExpanded={true}
-            title={(
-              <div className="vm-legend-group-title">
-                {showQueryNum && (
-                  <span className="vm-legend-group-title__count">Query {group}: </span>
-                )}
-                <span className="vm-legend-group-title__query">{query[group - 1]}</span>
-              </div>
-            )}
+      <div>
+        {groupSeries.map(({ group, items }) => (
+          <div
+            className="vm-legend-group"
+            key={group}
           >
-            <div>
-              {labels.filter(l => l.group === group).sort((x, y) => (y.median || 0) - (x.median || 0)).map((legendItem: LegendItemType) =>
-                <LegendItem
-                  key={legendItem.label}
-                  legend={legendItem}
-                  isAnomalyView={isAnomalyView}
-                  onChange={onChange}
-                />
+            <Accordion
+              defaultExpanded={true}
+              title={(
+                <div className="vm-legend-group-title">
+                  Group by{groupByLabel ? "" : " query"}: <b>{group}</b>
+                </div>
               )}
-            </div>
-          </Accordion>
-        </div>
-      ))}
+            >
+              <LegendGroup
+                labels={items}
+                isAnomalyView={isAnomalyView}
+                onChange={onChange}
+              />
+            </Accordion>
+          </div>
+        ))}
+      </div>
     </div>
   </>;
 };

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendConfigs/LegendConfigs.tsx

@@ -0,0 +1,104 @@
+import React, { FC, useMemo } from "preact/compat";
+import Switch from "../../../../Main/Switch/Switch";
+import { LegendDisplayType, useLegendView } from "../hooks/useLegendView";
+import { useHideDuplicateFields } from "../hooks/useHideDuplicateFields";
+import { useShowStats } from "../hooks/useShowStats";
+import TextField from "../../../../Main/TextField/TextField";
+import { useLegendFormat } from "../hooks/useLegendFormat";
+import { WITHOUT_GROUPING } from "../../../../../constants/logs";
+import Select from "../../../../Main/Select/Select";
+import { useLegendGroup } from "../hooks/useLegendGroup";
+import "./style.scss";
+import { MetricResult } from "../../../../../api/types";
+
+type Props = {
+  data: MetricResult[]
+}
+
+const LegendConfigs: FC<Props> = ({ data }) => {
+  const { isTableView, onChange: onChangeView } = useLegendView();
+  const { hideDuplicates, onChange: onChangeDuplicates } = useHideDuplicateFields();
+  const { hideStats, onChange: onChangeStats } = useShowStats();
+  const { format, onChange: onChangeFormat, onApply: onApplyFormat } = useLegendFormat();
+  const { groupByLabel, onChange: onChangeGroup } = useLegendGroup();
+
+  const uniqueFields = useMemo(() => {
+    const fields = data.flatMap(d => Object.keys(d.metric));
+    return Array.from(new Set(fields));
+  }, [data]);
+
+  const handleChangeView = (val: boolean) => {
+    const value = val ? LegendDisplayType.table : LegendDisplayType.lines;
+    onChangeView(value);
+  };
+
+  return (
+    <>
+      <div className="vm-legend-configs-item vm-legend-configs-item_switch">
+        <span className="vm-legend-configs-item__label">Table View</span>
+        <Switch
+          label={isTableView ? "Enabled" : "Disabled"}
+          value={isTableView}
+          onChange={handleChangeView}
+        />
+        <span className="vm-legend-configs-item__info">
+          Switches between table and lines view.
+        </span>
+      </div>
+
+      <div className="vm-legend-configs-item vm-legend-configs-item_switch">
+        <span className="vm-legend-configs-item__label">Common Labels</span>
+        <Switch
+          label={hideDuplicates ? "Hide" : "Show"}
+          value={!hideDuplicates}
+          onChange={onChangeDuplicates}
+        />
+        <span className="vm-legend-configs-item__info">
+          Shows or hides labels that are the same for all series.
+        </span>
+      </div>
+
+      <div className="vm-legend-configs-item vm-legend-configs-item_switch">
+        <span className="vm-legend-configs-item__label">Statistics</span>
+        <Switch
+          label={hideStats ? "Hide" : "Show"}
+          value={!hideStats}
+          onChange={onChangeStats}
+        />
+        <span className="vm-legend-configs-item__info">
+          Displays min, median, and max values.
+        </span>
+      </div>
+
+      <div className="vm-legend-configs-item">
+        <TextField
+          label="Custom Legend Format"
+          placeholder={"{{label_name}}"}
+          value={format}
+          onChange={onChangeFormat}
+          onBlur={onApplyFormat}
+          onEnter={onApplyFormat}
+        />
+        <span className="vm-legend-configs-item__info vm-legend-configs-item__info_input">
+          Customize legend labels with text and &#123;&#123;label_name&#125;&#125; placeholders.
+        </span>
+      </div>
+
+      <div className="vm-legend-configs-item">
+        <Select
+          label="Group Legend By"
+          value={groupByLabel}
+          list={[WITHOUT_GROUPING, ...uniqueFields]}
+          placeholder={WITHOUT_GROUPING}
+          onChange={onChangeGroup}
+          searchable
+        />
+        <span className="vm-legend-configs-item__info">
+          Choose a label to group the legend. By default, legends are grouped by query.
+        </span>
+      </div>
+    </>
+  );
+};
+
+export default LegendConfigs;

app/vmui/packages/vmui/src/components/Chart/Line/Legend/LegendConfigs/style.scss

@@ -0,0 +1,26 @@
+@use "src/styles/variables" as *;
+
+.vm-legend-configs {
+  &-item {
+    display: grid;
+    align-items: center;
+    justify-content: stretch;
+    margin-top: calc($padding-global/2);
+
+    &_switch {
+      grid-template-columns: 1fr 100px;
+      margin-top: 0;
+    }
+
+    &__info {
+      margin-top: calc($padding-global/2);
+      font-size: $font-size-small;
+      color: $color-text-secondary;
+      line-height: 130%;
+
+      &_input {
+        margin-top: 0;
+      }
+    }
+  }
+}