docs/CHANGELOG.md: cut v1.87.9

Fix the bug when Group's `params` fields with multiple values were
overriding each other instead of adding up.
The bug was introduced in this commit eccecdf177
 starting from v1.91.1 https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.91.1

 https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908

Signed-off-by: hagen1778 <roman@victoriametrics.com>
(cherry picked from commit 6351d07da8)

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.19.5
+          go-version: 1.21.1
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis-js.yml

@@ -13,6 +13,10 @@ on:
   schedule:
     - cron: "30 18 * * 2"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+  
 jobs:
   analyze:
     name: Analyze

.github/workflows/codeql-analysis.yml

@@ -15,6 +15,7 @@ on:
   push:
     branches: [master, cluster]
     paths-ignore:
+      - "docs/**"
       - "**.md"
       - "**.txt"
       - "**.js"
@@ -22,12 +23,17 @@ on:
     # The branches below must be a subset of the branches above
     branches: [master, cluster]
     paths-ignore:
+      - "docs/**"
       - "**.md"
       - "**.txt"
       - "**.js"
   schedule:
     - cron: "30 18 * * 2"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze
@@ -51,7 +57,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.5
+          go-version: 1.21.1
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main-test.yml

@@ -1,66 +0,0 @@
-name: main - test
-on:
-  push:
-    branches:
-      - master
-      - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    branches:
-      - master
-      - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-permissions:
-  contents: read
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19.5
-          check-latest: true
-          cache: true
-
-      - name: Dependencies
-        run: |
-          make install-golangci-lint
-          make check-all
-          git diff --exit-code
-
-  test:
-    needs: lint
-    strategy:
-      matrix:
-        scenario: ["test-full", "test-pure", "test-full-386"]
-    name: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19.5
-          check-latest: true
-          cache: true
-
-      - name: run tests
-        run: |
-          make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.txt

.github/workflows/main.yml

@@ -1,30 +1,95 @@
 name: main
 on:
-  workflow_run:
-    workflows: ["main - test"]
-    types:
-      - completed
-
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build:
-    name: Build
+  lint:
+    name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
         uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.workflow_run.head_branch }}
 
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.5
+          go-version: 1.21.1
           check-latest: true
           cache: true
 
+      - name: Dependencies
+        run: |
+          make install-golangci-lint
+          make check-all
+          git diff --exit-code
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.1
+          check-latest: true
+          cache: true
+
+      - name: run tests
+        run: |
+          make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.txt
+
+  build:
+    needs: test
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21.1
+          check-latest: true
+          cache: true
+
+      - uses: actions/cache@v3
+        with:
+          path: gocache-for-docker
+          key:  gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
+
       - name: Build
         run: |
           make victoria-metrics-crossbuild

.github/workflows/nightly-build.yml

@@ -1,39 +0,0 @@
-name: nightly-build
-on:
-  schedule:
-    # Daily at 2:48am
-    - cron: '48 2 * * *'
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-    name: Build
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
-        name: Setup Go
-        uses: actions/setup-go@main
-        with:
-          go-version: 1.19.5
-        id: go
-      -
-        name: Setup docker scan
-        run: |
-          mkdir -p ~/.docker/cli-plugins && \
-          curl https://github.com/docker/scan-cli-plugin/releases/latest/download/docker-scan_linux_amd64 -L -s -S -o ~/.docker/cli-plugins/docker-scan &&\
-          chmod +x ~/.docker/cli-plugins/docker-scan
-      -
-        name: Code checkout
-        uses: actions/checkout@master
-      -
-        name: Publish
-        run: |
-          LATEST_TAG=nightly PKG_TAG=nightly make publish

Makefile

@@ -169,6 +169,7 @@ vmutils-crossbuild: \
 	vmutils-windows-amd64
 
 publish-release:
+	rm -rf bin/*
 	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
 		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
 		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
@@ -380,7 +381,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.50.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2
 
 govulncheck: install-govulncheck
 	govulncheck ./...

README.md

@@ -2130,7 +2130,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
 
 ```
   -bigMergeConcurrency int
-     The maximum number of CPU cores to use for big merges. Default value is used if set to 0
+     Deprecated: this flag does nothing. Please use -smallMergeConcurrency for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage
   -cacheExpireDuration duration
      Items are removed from in-memory caches after they aren't accessed for this duration. Lower values may reduce memory usage at the cost of higher CPU usage. See also -prevCacheRemovalPercent (default 30m0s)
   -configAuthKey string
@@ -2469,7 +2469,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -selfScrapeJob string
      Value for 'job' label, which is added to self-scraped metrics (default "victoria-metrics")
   -smallMergeConcurrency int
-     The maximum number of CPU cores to use for small merges. Default value is used if set to 0
+     The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries
   -snapshotAuthKey string
      authKey, which must be passed in query string to /snapshot* pages
   -snapshotsMaxAge value

app/victoria-metrics/main.go

@@ -26,7 +26,8 @@ import (
 var (
 	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+

app/victoria-metrics/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmagent/README.md

@@ -324,7 +324,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
 ## Automatically generated metrics
 
 `vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)
-and attaches target-specific `instance` and `job` labels to these metrics:
+and attaches `instance`, `job` and other target-specific labels to these metrics:
 
 * `up` - this metric exposes `1` value on successful scrape and `0` value on unsuccessful scrape. This allows monitoring
   failing scrapes with the following [MetricsQL query](https://docs.victoriametrics.com/MetricsQL.html):

app/vmagent/main.go

@@ -46,7 +46,8 @@ var (
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
 		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
 		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
 		"See also -influxListenAddr.useProxyProtocol")

app/vmagent/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmagent/remotewrite/relabel.go

@@ -114,9 +114,9 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 			for j := range tmpLabels {
 				label := &tmpLabels[j]
 				if label.Name == "__name__" {
-					label.Value = promrelabel.SanitizeName(label.Value)
+					label.Value = promrelabel.SanitizeMetricName(label.Value)
 				} else {
-					label.Name = promrelabel.SanitizeName(label.Name)
+					label.Name = promrelabel.SanitizeLabelName(label.Name)
 				}
 			}
 		}

app/vmagent/remotewrite/remotewrite.go

@@ -524,6 +524,11 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 }
 
 func (rwctx *remoteWriteCtx) MustStop() {
+	// sas must be stopped before rwctx is closed
+	// because sas can write pending series to rwctx.pss if there are any
+	rwctx.sas.MustStop()
+	rwctx.sas = nil
+
 	for _, ps := range rwctx.pss {
 		ps.MustStop()
 	}
@@ -532,8 +537,7 @@ func (rwctx *remoteWriteCtx) MustStop() {
 	rwctx.fq.UnblockAllReaders()
 	rwctx.c.MustStop()
 	rwctx.c = nil
-	rwctx.sas.MustStop()
-	rwctx.sas = nil
+
 	rwctx.fq.MustClose()
 	rwctx.fq = nil
 

app/vmalert/Makefile

@@ -73,6 +73,7 @@ test-vmalert:
 	go test -v -race -cover ./app/vmalert/notifier
 	go test -v -race -cover ./app/vmalert/config
 	go test -v -race -cover ./app/vmalert/remotewrite
+	go test -v -race -cover ./app/vmalert/utils
 
 run-vmalert: vmalert
 	./bin/vmalert -rule=app/vmalert/config/testdata/rules/rules2-good.rules \

app/vmalert/README.md

@@ -907,10 +907,6 @@ The shortlist of configuration flags is the following:
      Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880")
   -httpListenAddr.useProxyProtocol
      Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
-  -insert.maxQueueDuration duration
-     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
-  -internStringMaxLen int
-     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -927,13 +923,6 @@ The shortlist of configuration flags is the following:
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-  -maxConcurrentInserts int
-     The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8)
-  -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
-  -memory.allowedPercent float
-     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
@@ -1023,7 +1012,7 @@ The shortlist of configuration flags is the following:
   -remoteRead.headers string
      Optional HTTP headers to send with each request to the corresponding -remoteRead.url. For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'
   -remoteRead.ignoreRestoreErrors
-     Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
+     Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases. (default true)
   -remoteRead.lookback duration
      Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
   -remoteRead.oauth2.clientID string
@@ -1101,7 +1090,7 @@ The shortlist of configuration flags is the following:
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int
-     Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
+     Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
      Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration

app/vmalert/alerting.go

@@ -421,7 +421,9 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
 	ar.EvalInterval = nr.EvalInterval
+	ar.Debug = nr.Debug
 	ar.q = nr.q
+	ar.state = nr.state
 	return nil
 }
 
@@ -498,6 +500,7 @@ func (ar *AlertingRule) ToAPI() APIRule {
 		LastSamples:    lastState.samples,
 		MaxUpdates:     ar.state.size(),
 		Updates:        ar.state.getAll(),
+		Debug:          ar.Debug,
 
 		// encode as strings to avoid rounding in JSON
 		ID:      fmt.Sprintf("%d", ar.ID()),
@@ -604,54 +607,59 @@ func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.Time
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
-// Restore restores the state of active alerts basing on previously written time series.
-// Restore restores only ActiveAt field. Field State will be always Pending and supposed
-// to be updated on next Exec, as well as Value field.
-// Only rules with For > 0 will be restored.
-func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
-	if q == nil {
-		return fmt.Errorf("querier is nil")
+// Restore restores the value of ActiveAt field for active alerts,
+// based on previously written time series `alertForStateMetricName`.
+// Only rules with For > 0 can be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, ts time.Time, lookback time.Duration) error {
+	if ar.For < 1 {
+		return nil
 	}
 
-	ts := time.Now()
-	qFn := func(query string) ([]datasource.Metric, error) {
-		res, _, err := ar.q.Query(ctx, query, ts)
-		return res, err
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
+
+	if len(ar.alerts) < 1 {
+		return nil
 	}
 
-	// account for external labels in filter
-	var labelsFilter string
-	for k, v := range labels {
-		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
-	}
-
-	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
-		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, _, err := q.Query(ctx, expr, ts)
-	if err != nil {
-		return err
-	}
-
-	for _, m := range qMetrics {
-		ls := &labelSet{
-			origin:    make(map[string]string, len(m.Labels)),
-			processed: make(map[string]string, len(m.Labels)),
+	for _, a := range ar.alerts {
+		if a.Restored || a.State != notifier.StatePending {
+			continue
 		}
-		for _, l := range m.Labels {
-			if l.Name == "__name__" {
-				continue
-			}
-			ls.origin[l.Name] = l.Value
-			ls.processed[l.Name] = l.Value
+
+		var labelsFilter []string
+		for k, v := range a.Labels {
+			labelsFilter = append(labelsFilter, fmt.Sprintf("%s=%q", k, v))
 		}
-		a, err := ar.newAlert(m, ls, time.Unix(int64(m.Values[0]), 0), qFn)
+		sort.Strings(labelsFilter)
+		expr := fmt.Sprintf("last_over_time(%s{%s}[%ds])",
+			alertForStateMetricName, strings.Join(labelsFilter, ","), int(lookback.Seconds()))
+
+		ar.logDebugf(ts, nil, "restoring alert state via query %q", expr)
+
+		qMetrics, _, err := q.Query(ctx, expr, ts)
 		if err != nil {
-			return fmt.Errorf("failed to create alert: %w", err)
+			return err
 		}
-		a.ID = hash(ls.processed)
-		a.State = notifier.StatePending
+
+		if len(qMetrics) < 1 {
+			ar.logDebugf(ts, nil, "no response was received from restore query")
+			continue
+		}
+
+		// only one series expected in response
+		m := qMetrics[0]
+		// __name__ supposed to be alertForStateMetricName
+		m.DelLabel("__name__")
+
+		// we assume that restore query contains all label matchers,
+		// so all received labels will match anyway if their number is equal.
+		if len(m.Labels) != len(a.Labels) {
+			ar.logDebugf(ts, nil, "state restore query returned not expected label-set %v", m.Labels)
+			continue
+		}
+		a.ActiveAt = time.Unix(int64(m.Values[0]), 0)
 		a.Restored = true
-		ar.alerts[a.ID] = a
 		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
 	}
 	return nil

app/vmalert/alerting_test.go

@@ -6,12 +6,15 @@ import (
 	"reflect"
 	"sort"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestAlertingRule_ToTimeSeries(t *testing.T) {
@@ -502,118 +505,156 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 	}
 }
 
-func TestAlertingRule_Restore(t *testing.T) {
-	testCases := []struct {
-		rule      *AlertingRule
-		metrics   []datasource.Metric
-		expAlerts map[uint64]*notifier.Alert
-	}{
-		{
-			newTestRuleWithLabels("no extra labels"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(nil): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-			},
-		},
-		{
-			newTestRuleWithLabels("metric labels"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					alertNameLabel, "metric labels",
-					alertGroupNameLabel, "groupID",
-					"foo", "bar",
-					"namespace", "baz",
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{
-					alertNameLabel:      "metric labels",
-					alertGroupNameLabel: "groupID",
-					"foo":               "bar",
-					"namespace":         "baz",
-				}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-			},
-		},
-		{
-			newTestRuleWithLabels("rule labels", "source", "vm"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"foo", "bar",
-					"namespace", "baz",
-					// extra labels set by rule
-					"source", "vm",
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{
-					"foo":       "bar",
-					"namespace": "baz",
-					"source":    "vm",
-				}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-			},
-		},
-		{
-			newTestRuleWithLabels("multiple alerts"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"host", "localhost-1",
-				),
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(2*time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"host", "localhost-2",
-				),
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(3*time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"host", "localhost-3",
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{"host": "localhost-1"}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-				hash(map[string]string{"host": "localhost-2"}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(2 * time.Hour)},
-				hash(map[string]string{"host": "localhost-3"}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(3 * time.Hour)},
-			},
-		},
+func TestGroup_Restore(t *testing.T) {
+	defaultTS := time.Now()
+	fqr := &fakeQuerierWithRegistry{}
+	fn := func(rules []config.Rule, expAlerts map[uint64]*notifier.Alert) {
+		t.Helper()
+		defer fqr.reset()
+
+		for _, r := range rules {
+			fqr.set(r.Expr, metricWithValueAndLabels(t, 0, "__name__", r.Alert))
+		}
+
+		fg := newGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			nts := func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }
+			fg.start(context.Background(), nts, nil, fqr)
+			wg.Done()
+		}()
+		fg.close()
+		wg.Wait()
+
+		gotAlerts := make(map[uint64]*notifier.Alert)
+		for _, rs := range fg.Rules {
+			alerts := rs.(*AlertingRule).alerts
+			for k, v := range alerts {
+				if !v.Restored {
+					// set not restored alerts to predictable timestamp
+					v.ActiveAt = defaultTS
+				}
+				gotAlerts[k] = v
+			}
+		}
+
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected %d alerts; got %d", len(expAlerts), len(gotAlerts))
+		}
+		for key, exp := range expAlerts {
+			got, ok := gotAlerts[key]
+			if !ok {
+				t.Fatalf("expected to have key %d", key)
+			}
+			if got.State != notifier.StatePending {
+				t.Fatalf("expected state %d; got %d", notifier.StatePending, got.State)
+			}
+			if got.ActiveAt != exp.ActiveAt {
+				t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
+			}
+		}
 	}
-	fakeGroup := Group{Name: "TestRule_Exec"}
-	for _, tc := range testCases {
-		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &fakeQuerier{}
-			tc.rule.GroupID = fakeGroup.ID()
-			tc.rule.q = fq
-			fq.add(tc.metrics...)
-			if err := tc.rule.Restore(context.TODO(), fq, time.Hour, nil); err != nil {
-				t.Fatalf("unexpected err: %s", err)
-			}
-			if len(tc.rule.alerts) != len(tc.expAlerts) {
-				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
-			}
-			for key, exp := range tc.expAlerts {
-				got, ok := tc.rule.alerts[key]
-				if !ok {
-					t.Fatalf("expected to have key %d", key)
-				}
-				if got.State != exp.State {
-					t.Fatalf("expected state %d; got %d", exp.State, got.State)
-				}
-				if got.ActiveAt != exp.ActiveAt {
-					t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
-				}
-			}
+
+	stateMetric := func(name string, value time.Time, labels ...string) datasource.Metric {
+		labels = append(labels, "__name__", alertForStateMetricName)
+		labels = append(labels, alertNameLabel, name)
+		labels = append(labels, alertGroupNameLabel, "TestRestore")
+		return metricWithValueAndLabels(t, float64(value.Unix()), labels...)
+	}
+
+	// one active alert, no previous state
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+	fqr.reset()
+
+	// one active alert with state restore
+	ts := time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+		stateMetric("foo", ts))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// two rules, two active alerts, one with state restored
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("foo", ts))
+	fn(
+		[]config.Rule{
+			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
+			{Alert: "bar", Expr: "bar", For: promutils.NewDuration(time.Second)},
+		},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// two rules, two active alerts, two with state restored
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+		stateMetric("foo", ts))
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("bar", ts))
+	fn(
+		[]config.Rule{
+			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
+			{Alert: "bar", Expr: "bar", For: promutils.NewDuration(time.Second)},
+		},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts,
+			},
+			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// one active alert but wrong state restore
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
+		stateMetric("wrong alert", ts))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+
+	// one active alert with labels
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+		stateMetric("foo", ts, "env", "dev"))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
+				ActiveAt: ts,
+			},
+		})
+
+	// one active alert with restore labels missmatch
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+		stateMetric("foo", ts, "env", "dev", "team", "foo"))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
+				ActiveAt: defaultTS,
+			},
 		})
-	}
 }
 
 func TestAlertingRule_Exec_Negative(t *testing.T) {

app/vmalert/datasource/datasource.go

@@ -72,6 +72,15 @@ func (m *Metric) AddLabel(key, value string) {
 	m.Labels = append(m.Labels, Label{Name: key, Value: value})
 }
 
+// DelLabel deletes the given label from the label set
+func (m *Metric) DelLabel(key string) {
+	for i, l := range m.Labels {
+		if l.Name == key {
+			m.Labels = append(m.Labels[:i], m.Labels[i+1:]...)
+		}
+	}
+}
+
 // Label returns the given label value.
 // If label is missing empty string will be returned
 func (m *Metric) Label(key string) string {

app/vmalert/datasource/vm.go

@@ -28,6 +28,7 @@ func toDatasourceType(s string) datasourceType {
 }
 
 // VMStorage represents vmstorage entity with ability to read and write metrics
+// WARN: when adding a new field, remember to update Clone() method.
 type VMStorage struct {
 	c                *http.Client
 	authCfg          *promauth.Config
@@ -53,29 +54,61 @@ type keyValue struct {
 
 // Clone makes clone of VMStorage, shares http client.
 func (s *VMStorage) Clone() *VMStorage {
-	return &VMStorage{
+	ns := &VMStorage{
 		c:                s.c,
 		authCfg:          s.authCfg,
 		datasourceURL:    s.datasourceURL,
+		appendTypePrefix: s.appendTypePrefix,
 		lookBack:         s.lookBack,
 		queryStep:        s.queryStep,
-		appendTypePrefix: s.appendTypePrefix,
-		dataSourceType:   s.dataSourceType,
+
+		dataSourceType:     s.dataSourceType,
+		evaluationInterval: s.evaluationInterval,
+
+		// init map so it can be populated below
+		extraParams: url.Values{},
+
+		debug: s.debug,
 	}
+	if len(s.extraHeaders) > 0 {
+		ns.extraHeaders = make([]keyValue, len(s.extraHeaders))
+		copy(ns.extraHeaders, s.extraHeaders)
+	}
+	for k, v := range s.extraParams {
+		ns.extraParams[k] = v
+	}
+
+	return ns
 }
 
 // ApplyParams - changes given querier params.
 func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 	s.dataSourceType = toDatasourceType(params.DataSourceType)
 	s.evaluationInterval = params.EvaluationInterval
-	s.extraParams = params.QueryParams
-	s.debug = params.Debug
+	if params.QueryParams != nil {
+		if s.extraParams == nil {
+			s.extraParams = url.Values{}
+		}
+		for k, vl := range params.QueryParams {
+			// custom query params are prior to default ones
+			if s.extraParams.Has(k) {
+				s.extraParams.Del(k)
+			}
+			for _, v := range vl {
+				// don't use .Set() instead of Del/Add since it is allowed
+				// for GET params to be duplicated
+				// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908
+				s.extraParams.Add(k, v)
+			}
+		}
+	}
 	if params.Headers != nil {
 		for key, value := range params.Headers {
 			kv := keyValue{key: key, value: value}
 			s.extraHeaders = append(s.extraHeaders, kv)
 		}
 	}
+	s.debug = params.Debug
 	return s
 }
 
@@ -94,6 +127,7 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 		lookBack:         lookBack,
 		queryStep:        queryStep,
 		dataSourceType:   datasourcePrometheus,
+		extraParams:      url.Values{},
 	}
 }
 

app/vmalert/datasource/vm_test.go

@@ -279,6 +279,9 @@ func TestRequestParams(t *testing.T) {
 	}
 	query := "up"
 	timestamp := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
+	storage := VMStorage{
+		extraParams: url.Values{"round_digits": {"10"}},
+	}
 	testCases := []struct {
 		name       string
 		queryRange bool
@@ -475,6 +478,28 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp, r.URL.RawQuery)
 			},
 		},
+		{
+			"custom params overrides the original params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"round_digits": {"2"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := fmt.Sprintf("query=%s&round_digits=2&time=%d", query, timestamp.Unix())
+				checkEqualString(t, exp, r.URL.RawQuery)
+			},
+		},
+		{
+			"allow duplicates in query params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
 		{
 			"graphite extra params",
 			false,

app/vmalert/group.go

@@ -158,23 +158,23 @@ func (g *Group) ID() uint64 {
 }
 
 // Restore restores alerts state for group rules
-func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error {
-	labels = mergeLabels(g.Name, "", labels, g.Labels)
+func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, ts time.Time, lookback time.Duration) error {
 	for _, rule := range g.Rules {
-		rr, ok := rule.(*AlertingRule)
+		ar, ok := rule.(*AlertingRule)
 		if !ok {
 			continue
 		}
-		if rr.For < 1 {
+		if ar.For < 1 {
 			continue
 		}
-		// ignore QueryParams on purpose, because they could contain
-		// query filters. This may affect the restore procedure.
 		q := qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType: g.Type.String(),
-			Headers:        g.Headers,
+			DataSourceType:     g.Type.String(),
+			EvaluationInterval: g.Interval,
+			QueryParams:        g.Params,
+			Headers:            g.Headers,
+			Debug:              ar.Debug,
 		})
-		if err := rr.Restore(ctx, q, lookback, labels); err != nil {
+		if err := ar.Restore(ctx, q, ts, lookback); err != nil {
 			return fmt.Errorf("error while restoring rule %q: %w", rule, err)
 		}
 	}
@@ -251,7 +251,7 @@ func (g *Group) close() {
 
 var skipRandSleepOnGroupStart bool
 
-func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
+func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
 	e := &executor{
@@ -259,26 +259,6 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		notifiers:                nts,
 		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
 
-	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
-	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
-		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
-		if randSleep < sleepOffset {
-			randSleep += uint64(g.Interval)
-		}
-		randSleep -= sleepOffset
-		sleepTimer := time.NewTimer(time.Duration(randSleep))
-		select {
-		case <-ctx.Done():
-			sleepTimer.Stop()
-			return
-		case <-g.doneCh:
-			sleepTimer.Stop()
-			return
-		case <-sleepTimer.C:
-		}
-	}
-
 	evalTS := time.Now()
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
@@ -309,6 +289,16 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
+
+	// restore the rules state after the first evaluation
+	// so only active alerts can be restored.
+	if rr != nil {
+		err := g.Restore(ctx, rr, evalTS, *remoteReadLookBack)
+		if err != nil {
+			logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
+		}
+	}
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -333,11 +323,17 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 				g.Interval = ng.Interval
 				t.Stop()
 				t = time.NewTicker(g.Interval)
+				evalTS = time.Now()
 			}
 			g.mu.Unlock()
 			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
 		case <-t.C:
 			missed := (time.Since(evalTS) / g.Interval) - 1
+			if missed < 0 {
+				// missed can become < 0 due to irregular delays during evaluation
+				// which can result in time.Since(evalTS) < g.Interval
+				missed = 0
+			}
 			if missed > 0 {
 				g.metrics.iterationMissed.Inc()
 			}

app/vmalert/group_test.go

@@ -209,7 +209,7 @@ func TestGroupStart(t *testing.T) {
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil)
+		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil, fs)
 		close(finished)
 	}()
 

app/vmalert/helpers_test.go

@@ -61,6 +61,49 @@ func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]dataso
 	return cp, req, nil
 }
 
+type fakeQuerierWithRegistry struct {
+	sync.Mutex
+	registry map[string][]datasource.Metric
+}
+
+func (fqr *fakeQuerierWithRegistry) set(key string, metrics ...datasource.Metric) {
+	fqr.Lock()
+	if fqr.registry == nil {
+		fqr.registry = make(map[string][]datasource.Metric)
+	}
+	fqr.registry[key] = metrics
+	fqr.Unlock()
+}
+
+func (fqr *fakeQuerierWithRegistry) reset() {
+	fqr.Lock()
+	fqr.registry = nil
+	fqr.Unlock()
+}
+
+func (fqr *fakeQuerierWithRegistry) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fqr
+}
+
+func (fqr *fakeQuerierWithRegistry) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
+	req, _, err := fqr.Query(ctx, q, time.Now())
+	return req, err
+}
+
+func (fqr *fakeQuerierWithRegistry) Query(_ context.Context, expr string, _ time.Time) ([]datasource.Metric, *http.Request, error) {
+	fqr.Lock()
+	defer fqr.Unlock()
+
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	metrics, ok := fqr.registry[expr]
+	if !ok {
+		return nil, req, nil
+	}
+	cp := make([]datasource.Metric, len(metrics))
+	copy(cp, metrics)
+	return cp, req, nil
+}
+
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert

app/vmalert/main.go

@@ -51,7 +51,8 @@ absolute path to all .tpl files in root.`)
 
 	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
@@ -73,7 +74,7 @@ absolute path to all .tpl files in root.`)
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
-	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")
+	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases.")
 
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 
@@ -94,6 +95,10 @@ func main() {
 	logger.Init()
 	pushmetrics.Init()
 
+	if !*remoteReadIgnoreRestoreErrors {
+		logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.")
+	}
+
 	err := templates.Load(*ruleTemplatesPath, true)
 	if err != nil {
 		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)

app/vmalert/manager.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"sort"
 	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -82,24 +83,38 @@ func (m *manager) close() {
 	m.wg.Wait()
 }
 
-func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) error {
-	if restore && m.rr != nil {
-		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
-		if err != nil {
-			if !*remoteReadIgnoreRestoreErrors {
-				return fmt.Errorf("failed to restore ruleState for group %q: %w", group.Name, err)
-			}
-			logger.Errorf("error while restoring ruleState for group %q: %s", group.Name, err)
-		}
-	}
-
+func (m *manager) startGroup(ctx context.Context, g *Group, restore bool) error {
 	m.wg.Add(1)
-	id := group.ID()
+	id := g.ID()
 	go func() {
-		group.start(ctx, m.notifiers, m.rw)
+		// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+		if !skipRandSleepOnGroupStart {
+			randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
+			sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+			if randSleep < sleepOffset {
+				randSleep += uint64(g.Interval)
+			}
+			randSleep -= sleepOffset
+			sleepTimer := time.NewTimer(time.Duration(randSleep))
+			select {
+			case <-ctx.Done():
+				sleepTimer.Stop()
+				return
+			case <-g.doneCh:
+				sleepTimer.Stop()
+				return
+			case <-sleepTimer.C:
+			}
+		}
+		if restore {
+			g.start(ctx, m.notifiers, m.rw, m.rr)
+		} else {
+			g.start(ctx, m.notifiers, m.rw, nil)
+		}
+
 		m.wg.Done()
 	}()
-	m.groups[id] = group
+	m.groups[id] = g
 	return nil
 }
 
@@ -190,6 +205,7 @@ func (g *Group) toAPI() APIGroup {
 		Headers:        headersToStrings(g.Headers),
 		Labels:         g.Labels,
 	}
+	ag.Rules = make([]APIRule, 0)
 	for _, r := range g.Rules {
 		ag.Rules = append(ag.Rules, r.ToAPI())
 	}

app/vmalert/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmalert/replay.go

@@ -26,7 +26,7 @@ var (
 			"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
 			"Keep it equal or bigger than -remoteWrite.flushInterval.")
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
-		"Max number of data points expected in one request. The higher the value, the less requests will be made during replay.")
+		"Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+

app/vmalert/rule.go

@@ -37,8 +37,6 @@ type ruleState struct {
 	sync.RWMutex
 	entries []ruleStateEntry
 	cur     int
-	// disabled defines whether ruleState tracks ruleStateEntry
-	disabled bool
 }
 
 type ruleStateEntry struct {
@@ -61,7 +59,7 @@ type ruleStateEntry struct {
 
 func newRuleState(size int) *ruleState {
 	if size < 1 {
-		return &ruleState{disabled: true}
+		size = 1
 	}
 	return &ruleState{
 		entries: make([]ruleStateEntry, size),
@@ -69,10 +67,6 @@ func newRuleState(size int) *ruleState {
 }
 
 func (s *ruleState) getLast() ruleStateEntry {
-	if s.disabled {
-		return ruleStateEntry{}
-	}
-
 	s.RLock()
 	defer s.RUnlock()
 	return s.entries[s.cur]
@@ -85,10 +79,6 @@ func (s *ruleState) size() int {
 }
 
 func (s *ruleState) getAll() []ruleStateEntry {
-	if s.disabled {
-		return nil
-	}
-
 	entries := make([]ruleStateEntry, 0)
 
 	s.RLock()
@@ -111,10 +101,6 @@ func (s *ruleState) getAll() []ruleStateEntry {
 }
 
 func (s *ruleState) add(e ruleStateEntry) {
-	if s.disabled {
-		return
-	}
-
 	s.Lock()
 	defer s.Unlock()
 

app/vmalert/rule_test.go

@@ -14,13 +14,13 @@ func TestRule_stateDisabled(t *testing.T) {
 	}
 
 	state.add(ruleStateEntry{at: time.Now()})
-	if !e.at.IsZero() {
-		t.Fatalf("expected entry to be zero")
-	}
+	state.add(ruleStateEntry{at: time.Now()})
+	state.add(ruleStateEntry{at: time.Now()})
 
-	if len(state.getAll()) != 0 {
+	if len(state.getAll()) != 1 {
+		// state should store at least one update at any circumstances
 		t.Fatalf("expected for state to have %d entries; got %d",
-			0, len(state.getAll()),
+			1, len(state.getAll()),
 		)
 	}
 }

app/vmalert/tpl/header.qtpl

@@ -1,5 +1,4 @@
 {% import (
-    "strings"
     "net/http"
     "path"
     "net/url"
@@ -85,10 +84,7 @@ type NavItem struct {
 
 {% func printNavItems(r *http.Request, current string, items []NavItem) %}
 {%code
-    prefix := "/vmalert/"
-    if strings.HasPrefix(r.URL.Path, prefix) {
-        prefix = ""
-    }
+    prefix := utils.Prefix(r.URL.Path)
 %}
 <nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
   <div class="container-fluid">

app/vmalert/tpl/header.qtpl.go

@@ -9,52 +9,51 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 )
 
-//line app/vmalert/tpl/header.qtpl:10
+//line app/vmalert/tpl/header.qtpl:9
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/tpl/header.qtpl:10
+//line app/vmalert/tpl/header.qtpl:9
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/tpl/header.qtpl:10
+//line app/vmalert/tpl/header.qtpl:9
 func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem, title string) {
-//line app/vmalert/tpl/header.qtpl:10
+//line app/vmalert/tpl/header.qtpl:9
 	qw422016.N().S(`
     `)
-//line app/vmalert/tpl/header.qtpl:11
+//line app/vmalert/tpl/header.qtpl:10
 	prefix := utils.Prefix(r.URL.Path)
 
-//line app/vmalert/tpl/header.qtpl:11
+//line app/vmalert/tpl/header.qtpl:10
 	qw422016.N().S(`
 <!DOCTYPE html>
 <html lang="en">
 <head>
     <title>vmalert`)
-//line app/vmalert/tpl/header.qtpl:15
+//line app/vmalert/tpl/header.qtpl:14
 	if title != "" {
-//line app/vmalert/tpl/header.qtpl:15
+//line app/vmalert/tpl/header.qtpl:14
 		qw422016.N().S(` - `)
-//line app/vmalert/tpl/header.qtpl:15
+//line app/vmalert/tpl/header.qtpl:14
 		qw422016.E().S(title)
-//line app/vmalert/tpl/header.qtpl:15
+//line app/vmalert/tpl/header.qtpl:14
 	}
-//line app/vmalert/tpl/header.qtpl:15
+//line app/vmalert/tpl/header.qtpl:14
 	qw422016.N().S(`</title>
     <link href="`)
-//line app/vmalert/tpl/header.qtpl:16
+//line app/vmalert/tpl/header.qtpl:15
 	qw422016.E().S(prefix)
-//line app/vmalert/tpl/header.qtpl:16
+//line app/vmalert/tpl/header.qtpl:15
 	qw422016.N().S(`static/css/bootstrap.min.css" rel="stylesheet" />
     <style>
         body{
@@ -114,139 +113,136 @@ func StreamHeader(qw422016 *qt422016.Writer, r *http.Request, navItems []NavItem
 </head>
 <body>
     `)
-//line app/vmalert/tpl/header.qtpl:74
+//line app/vmalert/tpl/header.qtpl:73
 	streamprintNavItems(qw422016, r, title, navItems)
-//line app/vmalert/tpl/header.qtpl:74
+//line app/vmalert/tpl/header.qtpl:73
 	qw422016.N().S(`
     <main class="px-2">
 `)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 }
 
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 func WriteHeader(qq422016 qtio422016.Writer, r *http.Request, navItems []NavItem, title string) {
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	StreamHeader(qw422016, r, navItems, title)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 }
 
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 func Header(r *http.Request, navItems []NavItem, title string) string {
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	WriteHeader(qb422016, r, navItems, title)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:76
+//line app/vmalert/tpl/header.qtpl:75
 }
 
-//line app/vmalert/tpl/header.qtpl:80
+//line app/vmalert/tpl/header.qtpl:79
 type NavItem struct {
 	Name string
 	Url  string
 }
 
-//line app/vmalert/tpl/header.qtpl:86
+//line app/vmalert/tpl/header.qtpl:85
 func streamprintNavItems(qw422016 *qt422016.Writer, r *http.Request, current string, items []NavItem) {
-//line app/vmalert/tpl/header.qtpl:86
+//line app/vmalert/tpl/header.qtpl:85
 	qw422016.N().S(`
 `)
-//line app/vmalert/tpl/header.qtpl:88
-	prefix := "/vmalert/"
-	if strings.HasPrefix(r.URL.Path, prefix) {
-		prefix = ""
-	}
+//line app/vmalert/tpl/header.qtpl:87
+	prefix := utils.Prefix(r.URL.Path)
 
-//line app/vmalert/tpl/header.qtpl:92
+//line app/vmalert/tpl/header.qtpl:88
 	qw422016.N().S(`
 <nav class="navbar navbar-expand-md navbar-dark fixed-top bg-dark">
   <div class="container-fluid">
     <div class="collapse navbar-collapse" id="navbarCollapse">
         <ul class="navbar-nav me-auto mb-2 mb-md-0">
             `)
-//line app/vmalert/tpl/header.qtpl:97
+//line app/vmalert/tpl/header.qtpl:93
 	for _, item := range items {
-//line app/vmalert/tpl/header.qtpl:97
+//line app/vmalert/tpl/header.qtpl:93
 		qw422016.N().S(`
                 <li class="nav-item">
                     `)
-//line app/vmalert/tpl/header.qtpl:100
+//line app/vmalert/tpl/header.qtpl:96
 		u, _ := url.Parse(item.Url)
 
-//line app/vmalert/tpl/header.qtpl:101
+//line app/vmalert/tpl/header.qtpl:97
 		qw422016.N().S(`
                     <a class="nav-link`)
-//line app/vmalert/tpl/header.qtpl:102
+//line app/vmalert/tpl/header.qtpl:98
 		if current == item.Name {
-//line app/vmalert/tpl/header.qtpl:102
+//line app/vmalert/tpl/header.qtpl:98
 			qw422016.N().S(` active`)
-//line app/vmalert/tpl/header.qtpl:102
+//line app/vmalert/tpl/header.qtpl:98
 		}
-//line app/vmalert/tpl/header.qtpl:102
+//line app/vmalert/tpl/header.qtpl:98
 		qw422016.N().S(`"
                        href="`)
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:99
 		if u.IsAbs() {
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:99
 			qw422016.E().S(item.Url)
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:99
 		} else {
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:99
 			qw422016.E().S(path.Join(prefix, item.Url))
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:99
 		}
-//line app/vmalert/tpl/header.qtpl:103
+//line app/vmalert/tpl/header.qtpl:99
 		qw422016.N().S(`">
                         `)
-//line app/vmalert/tpl/header.qtpl:104
+//line app/vmalert/tpl/header.qtpl:100
 		qw422016.E().S(item.Name)
-//line app/vmalert/tpl/header.qtpl:104
+//line app/vmalert/tpl/header.qtpl:100
 		qw422016.N().S(`
                     </a>
                 </li>
             `)
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:103
 	}
-//line app/vmalert/tpl/header.qtpl:107
+//line app/vmalert/tpl/header.qtpl:103
 	qw422016.N().S(`
         </ul>
   </div>
 </nav>
 `)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 }
 
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 func writeprintNavItems(qq422016 qtio422016.Writer, r *http.Request, current string, items []NavItem) {
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	streamprintNavItems(qw422016, r, current, items)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 }
 
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 func printNavItems(r *http.Request, current string, items []NavItem) string {
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	writeprintNavItems(qb422016, r, current, items)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	qs422016 := string(qb422016.B)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 	return qs422016
-//line app/vmalert/tpl/header.qtpl:111
+//line app/vmalert/tpl/header.qtpl:107
 }

app/vmalert/utils/links.go

@@ -1,13 +1,24 @@
 package utils
 
-import "strings"
+import (
+	"net/url"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+)
 
 const prefix = "/vmalert/"
 
 // Prefix returns "/vmalert/" prefix if it is missing in the path.
 func Prefix(path string) string {
+	pp := httpserver.GetPathPrefix()
+	path = strings.TrimLeft(path, pp)
 	if strings.HasPrefix(path, prefix) {
-		return ""
+		return pp
 	}
-	return prefix
+	res, err := url.JoinPath(pp, prefix)
+	if err != nil {
+		return path
+	}
+	return res
 }

app/vmalert/web.go

@@ -211,7 +211,7 @@ func (rh *requestHandler) groups() []APIGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	var groups []APIGroup
+	groups := make([]APIGroup, 0)
 	for _, g := range rh.m.groups {
 		groups = append(groups, g.toAPI())
 	}
@@ -276,6 +276,7 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
+	lr.Data.Alerts = make([]*APIAlert, 0)
 	for _, g := range rh.m.groups {
 		for _, r := range g.Rules {
 			a, ok := r.(*AlertingRule)

app/vmalert/web.qtpl

@@ -40,9 +40,9 @@
             for _, g := range groups {
                 for _, r := range g.Rules {
                     if r.LastError != "" {
-                        rNotOk[g.Name]++
+                        rNotOk[g.ID]++
                     } else {
-                        rOk[g.Name]++
+                        rOk[g.ID]++
                     }
                 }
             }
@@ -50,11 +50,11 @@
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
         {% for _, g := range groups  %}
-              <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
+              <div class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
                 <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
-                 {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
-                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
+                 {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
+                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.ID] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>
                 {% if len(g.Params) > 0 %}
                     <div class="fs-6 fw-lighter">Extra params
@@ -427,6 +427,16 @@
         </div>
       </div>
     </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Debug
+        </div>
+        <div class="col">
+           {%v rule.Debug %}
+        </div>
+      </div>
+    </div>
     {% endif %}
     <div class="container border-bottom p-2">
       <div class="row">

app/vmalert/web.qtpl.go

@@ -171,9 +171,9 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 		for _, g := range groups {
 			for _, r := range g.Rules {
 				if r.LastError != "" {
-					rNotOk[g.Name]++
+					rNotOk[g.ID]++
 				} else {
-					rOk[g.Name]++
+					rOk[g.ID]++
 				}
 			}
 		}
@@ -189,7 +189,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 			qw422016.N().S(`
               <div class="group-heading`)
 //line app/vmalert/web.qtpl:53
-			if rNotOk[g.Name] > 0 {
+			if rNotOk[g.ID] > 0 {
 //line app/vmalert/web.qtpl:53
 				qw422016.N().S(` alert-danger`)
 //line app/vmalert/web.qtpl:53
@@ -230,11 +230,11 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 			qw422016.N().S(`s)</a>
                  `)
 //line app/vmalert/web.qtpl:56
-			if rNotOk[g.Name] > 0 {
+			if rNotOk[g.ID] > 0 {
 //line app/vmalert/web.qtpl:56
 				qw422016.N().S(`<span class="badge bg-danger" title="Number of rules with status Error">`)
 //line app/vmalert/web.qtpl:56
-				qw422016.N().D(rNotOk[g.Name])
+				qw422016.N().D(rNotOk[g.ID])
 //line app/vmalert/web.qtpl:56
 				qw422016.N().S(`</span> `)
 //line app/vmalert/web.qtpl:56
@@ -243,7 +243,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 			qw422016.N().S(`
                 <span class="badge bg-success" title="Number of rules withs status Ok">`)
 //line app/vmalert/web.qtpl:57
-			qw422016.N().D(rOk[g.Name])
+			qw422016.N().D(rOk[g.ID])
 //line app/vmalert/web.qtpl:57
 			qw422016.N().S(`</span>
                 <p class="fs-6 fw-lighter">`)
@@ -1313,10 +1313,24 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
         </div>
       </div>
     </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Debug
+        </div>
+        <div class="col">
+           `)
+//line app/vmalert/web.qtpl:436
+		qw422016.E().V(rule.Debug)
+//line app/vmalert/web.qtpl:436
+		qw422016.N().S(`
+        </div>
+      </div>
+    </div>
     `)
-//line app/vmalert/web.qtpl:430
+//line app/vmalert/web.qtpl:440
 	}
-//line app/vmalert/web.qtpl:430
+//line app/vmalert/web.qtpl:440
 	qw422016.N().S(`
     <div class="container border-bottom p-2">
       <div class="row">
@@ -1325,17 +1339,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
         </div>
         <div class="col">
            <a target="_blank" href="`)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.E().S(prefix)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.N().S(`groups#group-`)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.E().S(rule.GroupID)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.E().S(rule.GroupID)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.N().S(`</a>
         </div>
       </div>
@@ -1343,13 +1357,13 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
 
     <br>
     <div class="display-6 pb-3">Last `)
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().D(len(rule.Updates))
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().S(`/`)
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().D(rule.MaxUpdates)
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().S(` updates</span>:</div>
         <table class="table table-striped table-hover table-sm">
             <thead>
@@ -1364,201 +1378,201 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
             <tbody>
 
      `)
-//line app/vmalert/web.qtpl:456
+//line app/vmalert/web.qtpl:466
 	for _, u := range rule.Updates {
-//line app/vmalert/web.qtpl:456
+//line app/vmalert/web.qtpl:466
 		qw422016.N().S(`
              <tr`)
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 		if u.err != nil {
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 			qw422016.N().S(` class="alert-danger"`)
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 		}
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 		qw422016.N().S(`>
                  <td>
                     <span class="badge bg-primary rounded-pill me-3" title="Updated at">`)
-//line app/vmalert/web.qtpl:459
+//line app/vmalert/web.qtpl:469
 		qw422016.E().S(u.time.Format(time.RFC3339))
-//line app/vmalert/web.qtpl:459
+//line app/vmalert/web.qtpl:469
 		qw422016.N().S(`</span>
                  </td>
                  <td class="text-center" wi>`)
-//line app/vmalert/web.qtpl:461
+//line app/vmalert/web.qtpl:471
 		qw422016.N().D(u.samples)
-//line app/vmalert/web.qtpl:461
+//line app/vmalert/web.qtpl:471
 		qw422016.N().S(`</td>
                  <td class="text-center">`)
-//line app/vmalert/web.qtpl:462
+//line app/vmalert/web.qtpl:472
 		qw422016.N().FPrec(u.duration.Seconds(), 3)
-//line app/vmalert/web.qtpl:462
+//line app/vmalert/web.qtpl:472
 		qw422016.N().S(`s</td>
                  <td class="text-center">`)
-//line app/vmalert/web.qtpl:463
+//line app/vmalert/web.qtpl:473
 		qw422016.E().S(u.at.Format(time.RFC3339))
-//line app/vmalert/web.qtpl:463
+//line app/vmalert/web.qtpl:473
 		qw422016.N().S(`</td>
                  <td>
                     <textarea class="curl-area" rows="1" onclick="this.focus();this.select()">`)
-//line app/vmalert/web.qtpl:465
+//line app/vmalert/web.qtpl:475
 		qw422016.E().S(u.curl)
-//line app/vmalert/web.qtpl:465
+//line app/vmalert/web.qtpl:475
 		qw422016.N().S(`</textarea>
                 </td>
              </tr>
           </li>
           `)
-//line app/vmalert/web.qtpl:469
+//line app/vmalert/web.qtpl:479
 		if u.err != nil {
-//line app/vmalert/web.qtpl:469
+//line app/vmalert/web.qtpl:479
 			qw422016.N().S(`
              <tr`)
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 			if u.err != nil {
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 				qw422016.N().S(` class="alert-danger"`)
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 			}
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 			qw422016.N().S(`>
                <td colspan="5">
                    <span class="alert-danger">`)
-//line app/vmalert/web.qtpl:472
+//line app/vmalert/web.qtpl:482
 			qw422016.E().V(u.err)
-//line app/vmalert/web.qtpl:472
+//line app/vmalert/web.qtpl:482
 			qw422016.N().S(`</span>
                </td>
              </tr>
           `)
-//line app/vmalert/web.qtpl:475
+//line app/vmalert/web.qtpl:485
 		}
-//line app/vmalert/web.qtpl:475
+//line app/vmalert/web.qtpl:485
 		qw422016.N().S(`
      `)
-//line app/vmalert/web.qtpl:476
+//line app/vmalert/web.qtpl:486
 	}
-//line app/vmalert/web.qtpl:476
+//line app/vmalert/web.qtpl:486
 	qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:478
+//line app/vmalert/web.qtpl:488
 	tpl.StreamFooter(qw422016, r)
-//line app/vmalert/web.qtpl:478
+//line app/vmalert/web.qtpl:488
 	qw422016.N().S(`
 `)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 }
 
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 func WriteRuleDetails(qq422016 qtio422016.Writer, r *http.Request, rule APIRule) {
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	StreamRuleDetails(qw422016, r, rule)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 }
 
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 func RuleDetails(r *http.Request, rule APIRule) string {
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	WriteRuleDetails(qb422016, r, rule)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	return qs422016
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 }
 
-//line app/vmalert/web.qtpl:483
+//line app/vmalert/web.qtpl:493
 func streambadgeState(qw422016 *qt422016.Writer, state string) {
-//line app/vmalert/web.qtpl:483
+//line app/vmalert/web.qtpl:493
 	qw422016.N().S(`
 `)
-//line app/vmalert/web.qtpl:485
+//line app/vmalert/web.qtpl:495
 	badgeClass := "bg-warning text-dark"
 	if state == "firing" {
 		badgeClass = "bg-danger"
 	}
 
-//line app/vmalert/web.qtpl:489
+//line app/vmalert/web.qtpl:499
 	qw422016.N().S(`
 <span class="badge `)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.E().S(badgeClass)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.E().S(state)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.N().S(`</span>
 `)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 }
 
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 func writebadgeState(qq422016 qtio422016.Writer, state string) {
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	streambadgeState(qw422016, state)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 }
 
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 func badgeState(state string) string {
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	writebadgeState(qb422016, state)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	return qs422016
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 }
 
-//line app/vmalert/web.qtpl:493
+//line app/vmalert/web.qtpl:503
 func streambadgeRestored(qw422016 *qt422016.Writer) {
-//line app/vmalert/web.qtpl:493
+//line app/vmalert/web.qtpl:503
 	qw422016.N().S(`
 <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
 `)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 }
 
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 func writebadgeRestored(qq422016 qtio422016.Writer) {
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	streambadgeRestored(qw422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 }
 
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 func badgeRestored() string {
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	writebadgeRestored(qb422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	return qs422016
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 }

app/vmalert/web_test.go

@@ -144,5 +144,81 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/1/0/status", func(t *testing.T) {
 		getResp(ts.URL+"/api/v1/1/0/status", nil, 404)
 	})
-
+}
+
+func TestEmptyResponse(t *testing.T) {
+	rhWithNoGroups := &requestHandler{m: &manager{groups: make(map[uint64]*Group)}}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rhWithNoGroups.handler(w, r) }))
+	defer ts.Close()
+
+	getResp := func(url string, to interface{}, code int) {
+		t.Helper()
+		resp, err := http.Get(url)
+		if err != nil {
+			t.Fatalf("unexpected err %s", err)
+		}
+		if code != resp.StatusCode {
+			t.Errorf("unexpected status code %d want %d", resp.StatusCode, code)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				t.Errorf("err closing body %s", err)
+			}
+		}()
+		if to != nil {
+			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
+				t.Errorf("unexpected err %s", err)
+			}
+		}
+	}
+
+	t.Run("no groups /api/v1/alerts", func(t *testing.T) {
+		lr := listAlertsResponse{}
+		getResp(ts.URL+"/api/v1/alerts", &lr, 200)
+		if lr.Data.Alerts == nil {
+			t.Errorf("expected /api/v1/alerts response to have non-nil data")
+		}
+
+		lr = listAlertsResponse{}
+		getResp(ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
+		if lr.Data.Alerts == nil {
+			t.Errorf("expected /api/v1/alerts response to have non-nil data")
+		}
+	})
+
+	t.Run("no groups /api/v1/rules", func(t *testing.T) {
+		lr := listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
+		if lr.Data.Groups == nil {
+			t.Errorf("expected /api/v1/rules response to have non-nil data")
+		}
+
+		lr = listGroupsResponse{}
+		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		if lr.Data.Groups == nil {
+			t.Errorf("expected /api/v1/rules response to have non-nil data")
+		}
+	})
+
+	rhWithEmptyGroup := &requestHandler{m: &manager{groups: map[uint64]*Group{0: {Name: "test"}}}}
+	ts.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rhWithEmptyGroup.handler(w, r) })
+
+	t.Run("empty group /api/v1/rules", func(t *testing.T) {
+		lr := listGroupsResponse{}
+		getResp(ts.URL+"/api/v1/rules", &lr, 200)
+		if lr.Data.Groups == nil {
+			t.Fatalf("expected /api/v1/rules response to have non-nil data")
+		}
+
+		lr = listGroupsResponse{}
+		getResp(ts.URL+"/vmalert/api/v1/rules", &lr, 200)
+		if lr.Data.Groups == nil {
+			t.Fatalf("expected /api/v1/rules response to have non-nil data")
+		}
+
+		group := lr.Data.Groups[0]
+		if group.Rules == nil {
+			t.Fatalf("expected /api/v1/rules response to have non-nil rules for group")
+		}
+	})
 }

app/vmalert/web_types.go

@@ -113,18 +113,20 @@ type APIRule struct {
 
 	// Additional fields
 
-	// Type of the rule: recording or alerting
+	// DatasourceType of the rule: prometheus or graphite
 	DatasourceType string `json:"datasourceType"`
 	LastSamples    int    `json:"lastSamples"`
 	// ID is a unique Alert's ID within a group
 	ID string `json:"id"`
 	// GroupID is an unique Group's ID
 	GroupID string `json:"group_id"`
+	// Debug shows whether debug mode is enabled
+	Debug bool `json:"debug"`
 
 	// MaxUpdates is the max number of recorded ruleStateEntry objects
 	MaxUpdates int `json:"max_updates_entries"`
 	// Updates contains the ordered list of recorded ruleStateEntry objects
-	Updates []ruleStateEntry `json:"updates"`
+	Updates []ruleStateEntry `json:"-"`
 }
 
 // WebLink returns a link to the alert which can be used in UI.

app/vmauth/main.go

@@ -28,7 +28,8 @@ import (
 var (
 	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
 	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
-		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . "+
+		"With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing")
 	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
 		"See also -maxConcurrentRequests")
 	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
@@ -94,15 +95,15 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	ui := ac[authToken]
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
-		err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
 		if *logInvalidAuthTokens {
+			err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        err,
 				StatusCode: http.StatusUnauthorized,
 			}
 			httpserver.Errorf(w, r, "%s", err)
 		} else {
-			http.Error(w, err.Error(), http.StatusUnauthorized)
+			http.Error(w, "Unauthorized", http.StatusUnauthorized)
 		}
 		return true
 	}

app/vmauth/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmbackup/main.go

@@ -49,29 +49,35 @@ func main() {
 	logger.Init()
 	pushmetrics.Init()
 
+	// Storing snapshot delete function to be able to call it in case
+	// of error since logger.Fatal will exit the program without
+	// calling deferred functions.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2055
+	deleteSnapshot := func() {}
+
 	if len(*snapshotCreateURL) > 0 {
 		// create net/url object
-		createUrl, err := url.Parse(*snapshotCreateURL)
+		createURL, err := url.Parse(*snapshotCreateURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotCreateURL: %s", err)
 		}
 		if len(*snapshotName) > 0 {
 			logger.Fatalf("-snapshotName shouldn't be set if -snapshot.createURL is set, since snapshots are created automatically in this case")
 		}
-		logger.Infof("Snapshot create url %s", createUrl.Redacted())
+		logger.Infof("Snapshot create url %s", createURL.Redacted())
 		if len(*snapshotDeleteURL) <= 0 {
 			err := flag.Set("snapshot.deleteURL", strings.Replace(*snapshotCreateURL, "/create", "/delete", 1))
 			if err != nil {
 				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
 			}
 		}
-		deleteUrl, err := url.Parse(*snapshotDeleteURL)
+		deleteURL, err := url.Parse(*snapshotDeleteURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotDeleteURL: %s", err)
 		}
-		logger.Infof("Snapshot delete url %s", deleteUrl.Redacted())
+		logger.Infof("Snapshot delete url %s", deleteURL.Redacted())
 
-		name, err := snapshot.Create(createUrl.String())
+		name, err := snapshot.Create(createURL.String())
 		if err != nil {
 			logger.Fatalf("cannot create snapshot: %s", err)
 		}
@@ -80,32 +86,48 @@ func main() {
 			logger.Fatalf("cannot set snapshotName flag: %v", err)
 		}
 
-		defer func() {
-			err := snapshot.Delete(deleteUrl.String(), name)
+		deleteSnapshot = func() {
+			err := snapshot.Delete(deleteURL.String(), name)
 			if err != nil {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
-		}()
+		}
 	} else if len(*snapshotName) == 0 {
 		logger.Fatalf("`-snapshotName` or `-snapshot.createURL` must be provided")
 	}
-	if err := snapshot.Validate(*snapshotName); err != nil {
-		logger.Fatalf("invalid -snapshotName=%q: %s", *snapshotName, err)
-	}
 
 	go httpserver.Serve(*httpListenAddr, false, nil)
 
+	err := makeBackup()
+	deleteSnapshot()
+	if err != nil {
+		logger.Fatalf("cannot create backup: %s", err)
+	}
+
+	startTime := time.Now()
+	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
+	if err := httpserver.Stop(*httpListenAddr); err != nil {
+		logger.Fatalf("cannot stop http server for metrics: %s", err)
+	}
+	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
+}
+
+func makeBackup() error {
+	if err := snapshot.Validate(*snapshotName); err != nil {
+		return fmt.Errorf("invalid -snapshotName=%q: %s", *snapshotName, err)
+	}
+
 	srcFS, err := newSrcFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	dstFS, err := newDstFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	originFS, err := newOriginFS()
 	if err != nil {
-		logger.Fatalf("%s", err)
+		return err
 	}
 	a := &actions.Backup{
 		Concurrency: *concurrency,
@@ -114,18 +136,12 @@ func main() {
 		Origin:      originFS,
 	}
 	if err := a.Run(); err != nil {
-		logger.Fatalf("cannot create backup: %s", err)
+		return err
 	}
 	srcFS.MustStop()
 	dstFS.MustStop()
 	originFS.MustStop()
-
-	startTime := time.Now()
-	logger.Infof("gracefully shutting down http server for metrics at %q", *httpListenAddr)
-	if err := httpserver.Stop(*httpListenAddr); err != nil {
-		logger.Fatalf("cannot stop http server for metrics: %s", err)
-	}
-	logger.Infof("successfully shut down http server for metrics in %.3f seconds", time.Since(startTime).Seconds())
+	return nil
 }
 
 func usage() {
@@ -189,7 +205,20 @@ func hasFilepathPrefix(path, prefix string) bool {
 	if err != nil {
 		return false
 	}
-	return strings.HasPrefix(pathAbs, prefixAbs)
+	if prefixAbs == pathAbs {
+		return true
+	}
+	rel, err := filepath.Rel(prefixAbs, pathAbs)
+	if err != nil {
+		// if paths can't be related - they don't match
+		return false
+	}
+	if i := strings.Index(rel, "."); i == 0 {
+		// if path can be related only with . as first char - they still don't match
+		return false
+	}
+	// if paths are related - it is a match
+	return true
 }
 
 func newOriginFS() (common.OriginFS, error) {

app/vmbackup/main_test.go

@@ -26,4 +26,9 @@ func TestHasFilepathPrefix(t *testing.T) {
 	f("fs://"+pwd+"/foo", pwd+"/foo/bar", false)
 	f("fs://"+pwd+"/foo/bar", pwd+"/foo", true)
 	f("fs://"+pwd+"/foo", pwd+"/bar", false)
+	f("fs:///data1", "/data", false)
+	f("fs:///data", "/data1", false)
+	f("fs:///data", "/data/foo", false)
+	f("fs:///data/foo", "/data", true)
+	f("fs:///data/foo/", "/data/", true)
 }

app/vmbackup/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmctl/flags.go

@@ -434,9 +434,10 @@ var (
 			Value: 1,
 		},
 		&cli.TimestampFlag{
-			Name:   remoteReadFilterTimeStart,
-			Usage:  "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
-			Layout: time.RFC3339,
+			Name:     remoteReadFilterTimeStart,
+			Usage:    "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
+			Layout:   time.RFC3339,
+			Required: true,
 		},
 		&cli.TimestampFlag{
 			Name:   remoteReadFilterTimeEnd,

app/vmctl/influx/influx.go

@@ -153,6 +153,10 @@ func (c *Client) Explore() ([]*Series, error) {
 		return nil, fmt.Errorf("failed to get field keys: %s", err)
 	}
 
+	if len(mFields) < 1 {
+		return nil, fmt.Errorf("found no numeric fields for import in database %q", c.database)
+	}
+
 	series, err := c.getSeries()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get series: %s", err)
@@ -162,7 +166,8 @@ func (c *Client) Explore() ([]*Series, error) {
 	for _, s := range series {
 		fields, ok := mFields[s.Measurement]
 		if !ok {
-			return nil, fmt.Errorf("can't find field keys for measurement %q", s.Measurement)
+			log.Printf("skip measurement %q since it has no fields", s.Measurement)
+			continue
 		}
 		for _, field := range fields {
 			is := &Series{

app/vmctl/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmctl/opentsdb/parser.go

@@ -180,7 +180,7 @@ func modifyData(msg Metric, normalize bool) (Metric, error) {
 	/*
 		replace bad characters in metric name with _ per the data model
 	*/
-	finalMsg.Metric = promrelabel.SanitizeName(name)
+	finalMsg.Metric = promrelabel.SanitizeMetricName(name)
 	// replace bad characters in tag keys with _ per the data model
 	for key, value := range msg.Tags {
 		// if normalization requested, lowercase the key and value
@@ -191,7 +191,7 @@ func modifyData(msg Metric, normalize bool) (Metric, error) {
 		/*
 			replace all explicitly bad characters with _
 		*/
-		key = promrelabel.SanitizeName(key)
+		key = promrelabel.SanitizeLabelName(key)
 		// tags that start with __ are considered custom stats for internal prometheus stuff, we should drop them
 		if !strings.HasPrefix(key, "__") {
 			finalMsg.Tags[key] = value

app/vminsert/common/streamaggr.go

@@ -99,6 +99,7 @@ func pushAggregateSeries(tss []prompbmarshal.TimeSeries) {
 	ctx.skipStreamAggr = true
 	for _, ts := range tss {
 		labels := ts.Labels
+		ctx.Labels = ctx.Labels[:0]
 		for _, label := range labels {
 			name := label.Name
 			if name == "__name__" {

app/vminsert/relabel/relabel.go

@@ -128,9 +128,9 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 		for i := range tmpLabels {
 			label := &tmpLabels[i]
 			if label.Name == "__name__" {
-				label.Value = promrelabel.SanitizeName(label.Value)
+				label.Value = promrelabel.SanitizeMetricName(label.Value)
 			} else {
-				label.Name = promrelabel.SanitizeName(label.Name)
+				label.Name = promrelabel.SanitizeLabelName(label.Name)
 			}
 		}
 	}

app/vmrestore/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmselect/main.go

@@ -114,6 +114,13 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			timerpool.Put(t)
 			qt.Printf("wait in queue because -search.maxConcurrentRequests=%d concurrent requests are executed", *maxConcurrentRequests)
 			defer func() { <-concurrencyLimitCh }()
+		case <-r.Context().Done():
+			timerpool.Put(t)
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			logger.Infof("client has cancelled the request after %.3f seconds: remoteAddr=%s, requestURI: %q",
+				d.Seconds(), remoteAddr, requestURI)
+			return true
 		case <-t.C:
 			timerpool.Put(t)
 			concurrencyLimitTimeout.Inc()

app/vmselect/netstorage/netstorage.go

@@ -148,40 +148,31 @@ func timeseriesWorker(qt *querytracer.Tracer, workChs []chan *timeseriesWork, wo
 	// Then help others with the remaining work.
 	rowsProcessed = 0
 	seriesProcessed = 0
-	idx := int(workerID)
-	for {
-		tsw, idxNext := stealTimeseriesWork(workChs, idx)
-		if tsw == nil {
-			// There is no more work
-			break
+	for i := uint(1); i < uint(len(workChs)); i++ {
+		idx := (i + workerID) % uint(len(workChs))
+		ch := workChs[idx]
+		for len(ch) > 0 {
+			// Do not call runtime.Gosched() here in order to give a chance
+			// the real owner of the work to complete it, since it consumes additional CPU
+			// and slows down the code on systems with big number of CPU cores.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966#issuecomment-1483208419
+
+			// It is expected that every channel in the workChs is already closed,
+			// so the next line should return immediately.
+			tsw, ok := <-ch
+			if !ok {
+				break
+			}
+			tsw.err = tsw.do(&tmpResult.rs, workerID)
+			rowsProcessed += tsw.rowsProcessed
+			seriesProcessed++
 		}
-		tsw.err = tsw.do(&tmpResult.rs, workerID)
-		rowsProcessed += tsw.rowsProcessed
-		seriesProcessed++
-		idx = idxNext
 	}
 	qt.Printf("others work processed: series=%d, samples=%d", seriesProcessed, rowsProcessed)
 
 	putTmpResult(tmpResult)
 }
 
-func stealTimeseriesWork(workChs []chan *timeseriesWork, startIdx int) (*timeseriesWork, int) {
-	for i := startIdx; i < startIdx+len(workChs); i++ {
-		// Give a chance other goroutines to perform their work
-		runtime.Gosched()
-
-		idx := i % len(workChs)
-		ch := workChs[idx]
-		// It is expected that every channel in the workChs is already closed,
-		// so the next line should return immediately.
-		tsw, ok := <-ch
-		if ok {
-			return tsw, idx
-		}
-	}
-	return nil, startIdx
-}
-
 func getTmpResult() *result {
 	v := resultPool.Get()
 	if v == nil {
@@ -207,10 +198,17 @@ type result struct {
 
 var resultPool sync.Pool
 
+// MaxWorkers returns the maximum number of workers netstorage can spin when calling RunParallel()
+func MaxWorkers() int {
+	return gomaxprocs
+}
+
+var gomaxprocs = cgroup.AvailableCPUs()
+
 // RunParallel runs f in parallel for all the results from rss.
 //
 // f shouldn't hold references to rs after returning.
-// workerID is the id of the worker goroutine that calls f.
+// workerID is the id of the worker goroutine that calls f. The workerID is in the range [0..MaxWorkers()-1].
 // Data processing is immediately stopped if f returns non-nil error.
 //
 // rss becomes unusable after the call to RunParallel.
@@ -244,7 +242,8 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 		tsw.f = f
 		tsw.mustStop = &mustStop
 	}
-	if gomaxprocs == 1 || tswsLen == 1 {
+	maxWorkers := MaxWorkers()
+	if maxWorkers == 1 || tswsLen == 1 {
 		// It is faster to process time series in the current goroutine.
 		tsw := getTimeseriesWork()
 		tmpResult := getTmpResult()
@@ -280,8 +279,8 @@ func (rss *Results) runParallel(qt *querytracer.Tracer, f func(rs *Result, worke
 
 	// Prepare worker channels.
 	workers := len(tsws)
-	if workers > gomaxprocs {
-		workers = gomaxprocs
+	if workers > maxWorkers {
+		workers = maxWorkers
 	}
 	itemsPerWorker := (len(tsws) + workers - 1) / workers
 	workChs := make([]chan *timeseriesWork, workers)
@@ -333,8 +332,6 @@ var (
 	seriesReadPerQuery = metrics.NewHistogram(`vm_series_read_per_query`)
 )
 
-var gomaxprocs = cgroup.AvailableCPUs()
-
 type packedTimeseries struct {
 	metricName string
 	brs        []blockRef
@@ -391,37 +388,25 @@ func unpackWorker(workChs []chan *unpackWork, workerID uint) {
 	}
 
 	// Then help others with their work.
-	idx := int(workerID)
-	for {
-		upw, idxNext := stealUnpackWork(workChs, idx)
-		if upw == nil {
-			// There is no more work
-			break
+	for i := uint(1); i < uint(len(workChs)); i++ {
+		idx := (i + workerID) % uint(len(workChs))
+		ch := workChs[idx]
+		for len(ch) > 0 {
+			// Give a chance other goroutines to perform their work
+			runtime.Gosched()
+			// It is expected that every channel in the workChs is already closed,
+			// so the next line should return immediately.
+			upw, ok := <-ch
+			if !ok {
+				break
+			}
+			upw.unpack(tmpBlock)
 		}
-		upw.unpack(tmpBlock)
-		idx = idxNext
 	}
 
 	putTmpStorageBlock(tmpBlock)
 }
 
-func stealUnpackWork(workChs []chan *unpackWork, startIdx int) (*unpackWork, int) {
-	for i := startIdx; i < startIdx+len(workChs); i++ {
-		// Give a chance other goroutines to perform their work
-		runtime.Gosched()
-
-		idx := i % len(workChs)
-		ch := workChs[idx]
-		// It is expected that every channel in the workChs is already closed,
-		// so the next line should return immediately.
-		upw, ok := <-ch
-		if ok {
-			return upw, idx
-		}
-	}
-	return nil, startIdx
-}
-
 func getTmpStorageBlock() *storage.Block {
 	v := tmpStorageBlockPool.Get()
 	if v == nil {
@@ -1017,7 +1002,6 @@ func ExportBlocks(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline sear
 	indexSearchDuration.UpdateDuration(startTime)
 
 	// Start workers that call f in parallel on available CPU cores.
-	gomaxprocs := cgroup.AvailableCPUs()
 	workCh := make(chan *exportWork, gomaxprocs*8)
 	var (
 		errGlobal     error
@@ -1187,8 +1171,10 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			putStorageSearch(sr)
 			return nil, fmt.Errorf("cannot write %d bytes to temporary file: %w", len(buf), err)
 		}
-		metricName := bytesutil.InternBytes(sr.MetricBlockRef.MetricName)
-		brs := m[metricName]
+		// Do not intern mb.MetricName, since it leads to increased memory usage.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692
+		metricName := sr.MetricBlockRef.MetricName
+		brs := m[string(metricName)]
 		if brs == nil {
 			brs = &blockRefs{}
 			brs.brs = brs.brsPrealloc[:0]
@@ -1198,8 +1184,9 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 			addr:    addr,
 		})
 		if len(brs.brs) == 1 {
-			orderedMetricNames = append(orderedMetricNames, metricName)
-			m[metricName] = brs
+			metricNameStr := string(metricName)
+			orderedMetricNames = append(orderedMetricNames, metricNameStr)
+			m[metricNameStr] = brs
 		}
 	}
 	if err := sr.Error(); err != nil {

app/vmselect/netstorage/tmp_blocks_file.go

@@ -142,6 +142,9 @@ func (tbf *tmpBlocksFile) Finalize() error {
 	// This should reduce the number of disk seeks, which is important
 	// for HDDs.
 	r.MustFadviseSequentialRead(true)
+	// Collect local stats in order to improve performance on systems with big number of CPU cores.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3966
+	r.SetUseLocalStats()
 	tbf.r = r
 	return nil
 }

app/vmselect/promql/aggr_incremental.go

@@ -3,8 +3,9 @@ package promql
 import (
 	"math"
 	"strings"
-	"sync"
+	"unsafe"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -63,31 +64,36 @@ var incrementalAggrFuncCallbacksMap = map[string]*incrementalAggrFuncCallbacks{
 	},
 }
 
+type incrementalAggrContextMap struct {
+	m map[string]*incrementalAggrContext
+
+	// The padding prevents false sharing on widespread platforms with
+	// 128 mod (cache line size) = 0 .
+	_ [128 - unsafe.Sizeof(map[string]*incrementalAggrContext{})%128]byte
+}
+
 type incrementalAggrFuncContext struct {
 	ae *metricsql.AggrFuncExpr
 
-	m sync.Map
+	byWorkerID []incrementalAggrContextMap
 
 	callbacks *incrementalAggrFuncCallbacks
 }
 
 func newIncrementalAggrFuncContext(ae *metricsql.AggrFuncExpr, callbacks *incrementalAggrFuncCallbacks) *incrementalAggrFuncContext {
 	return &incrementalAggrFuncContext{
-		ae:        ae,
-		callbacks: callbacks,
+		ae:         ae,
+		byWorkerID: make([]incrementalAggrContextMap, netstorage.MaxWorkers()),
+		callbacks:  callbacks,
 	}
 }
 
 func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, workerID uint) {
-	v, ok := iafc.m.Load(workerID)
-	if !ok {
-		// It is safe creating and storing m in iafc.m without locking,
-		// since it is guaranteed that only a single goroutine can execute
-		// code for the given workerID at a time.
-		v = make(map[string]*incrementalAggrContext, 1)
-		iafc.m.Store(workerID, v)
+	v := &iafc.byWorkerID[workerID]
+	if v.m == nil {
+		v.m = make(map[string]*incrementalAggrContext, 1)
 	}
-	m := v.(map[string]*incrementalAggrContext)
+	m := v.m
 
 	ts := tsOrig
 	keepOriginal := iafc.callbacks.keepOriginal
@@ -128,9 +134,9 @@ func (iafc *incrementalAggrFuncContext) updateTimeseries(tsOrig *timeseries, wor
 func (iafc *incrementalAggrFuncContext) finalizeTimeseries() []*timeseries {
 	mGlobal := make(map[string]*incrementalAggrContext)
 	mergeAggrFunc := iafc.callbacks.mergeAggrFunc
-	iafc.m.Range(func(k, v interface{}) bool {
-		m := v.(map[string]*incrementalAggrContext)
-		for k, iac := range m {
+	byWorkerID := iafc.byWorkerID
+	for i := range byWorkerID {
+		for k, iac := range byWorkerID[i].m {
 			iacGlobal := mGlobal[k]
 			if iacGlobal == nil {
 				if iafc.ae.Limit > 0 && len(mGlobal) >= iafc.ae.Limit {
@@ -142,8 +148,7 @@ func (iafc *incrementalAggrFuncContext) finalizeTimeseries() []*timeseries {
 			}
 			mergeAggrFunc(iacGlobal, iac)
 		}
-		return true
-	})
+	}
 	tss := make([]*timeseries, 0, len(mGlobal))
 	finalizeAggrFunc := iafc.callbacks.finalizeAggrFunc
 	for _, iac := range mGlobal {

app/vmselect/promql/aggr_incremental_test.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/metricsql"
 )
 
@@ -99,7 +100,7 @@ func TestIncrementalAggr(t *testing.T) {
 }
 
 func testIncrementalParallelAggr(iafc *incrementalAggrFuncContext, tssSrc, tssExpected []*timeseries) error {
-	const workersCount = 3
+	workersCount := netstorage.MaxWorkers()
 	tsCh := make(chan *timeseries)
 	var wg sync.WaitGroup
 	wg.Add(workersCount)

app/vmselect/promql/eval.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
@@ -893,31 +894,34 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	if err != nil {
 		return nil, err
 	}
-	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
-	var tssLock sync.Mutex
+
 	var samplesScannedTotal uint64
 	keepMetricNames := getKeepMetricNames(expr)
-	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) {
+	tsw := getTimeseriesByWorkerID()
+	seriesByWorkerID := tsw.byWorkerID
+	doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64) {
 		values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps)
 		preFunc(values, timestamps)
 		for _, rc := range rcs {
 			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &tsSQ.MetricName); tsm != nil {
 				samplesScanned := rc.DoTimeseriesMap(tsm, values, timestamps)
 				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-				tssLock.Lock()
-				tss = tsm.AppendTimeseriesTo(tss)
-				tssLock.Unlock()
+				seriesByWorkerID[workerID].tss = tsm.AppendTimeseriesTo(seriesByWorkerID[workerID].tss)
 				continue
 			}
 			var ts timeseries
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps)
 			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-			tssLock.Lock()
-			tss = append(tss, &ts)
-			tssLock.Unlock()
+			seriesByWorkerID[workerID].tss = append(seriesByWorkerID[workerID].tss, &ts)
 		}
 		return values, timestamps
 	})
+	tss := make([]*timeseries, 0, len(tssSQ)*len(rcs))
+	for i := range seriesByWorkerID {
+		tss = append(tss, seriesByWorkerID[i].tss...)
+	}
+	putTimeseriesByWorkerID(tsw)
+
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
 	qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal)
 	return tss, nil
@@ -941,28 +945,36 @@ func getKeepMetricNames(expr metricsql.Expr) bool {
 	return false
 }
 
-func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64) ([]float64, []int64)) {
-	concurrency := cgroup.AvailableCPUs()
-	if concurrency > len(tss) {
-		concurrency = len(tss)
+func doParallel(tss []*timeseries, f func(ts *timeseries, values []float64, timestamps []int64, workerID uint) ([]float64, []int64)) {
+	workers := netstorage.MaxWorkers()
+	if workers > len(tss) {
+		workers = len(tss)
 	}
-	workCh := make(chan *timeseries, concurrency)
+	seriesPerWorker := (len(tss) + workers - 1) / workers
+	workChs := make([]chan *timeseries, workers)
+	for i := range workChs {
+		workChs[i] = make(chan *timeseries, seriesPerWorker)
+	}
+	for i, ts := range tss {
+		idx := i % len(workChs)
+		workChs[idx] <- ts
+	}
+	for _, workCh := range workChs {
+		close(workCh)
+	}
+
 	var wg sync.WaitGroup
-	wg.Add(concurrency)
-	for i := 0; i < concurrency; i++ {
-		go func() {
+	wg.Add(workers)
+	for i := 0; i < workers; i++ {
+		go func(workerID uint) {
 			defer wg.Done()
 			var tmpValues []float64
 			var tmpTimestamps []int64
-			for ts := range workCh {
-				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps)
+			for ts := range workChs[workerID] {
+				tmpValues, tmpTimestamps = f(ts, tmpValues, tmpTimestamps, workerID)
 			}
-		}()
+		}(uint(i))
 	}
-	for _, ts := range tss {
-		workCh <- ts
-	}
-	close(workCh)
 	wg.Wait()
 }
 
@@ -1041,6 +1053,9 @@ func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcNa
 	} else {
 		minTimestamp -= ec.Step
 	}
+	if minTimestamp < 0 {
+		minTimestamp = 0
+	}
 	sq := storage.NewSearchQuery(minTimestamp, ec.End, tfss, ec.MaxSeries)
 	rss, err := netstorage.ProcessSearchQuery(qt, sq, ec.Deadline)
 	if err != nil {
@@ -1177,9 +1192,11 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 	preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) {
 	qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs)
 	defer qt.Done()
-	tss := make([]*timeseries, 0, rss.Len()*len(rcs))
-	var tssLock sync.Mutex
+
 	var samplesScannedTotal uint64
+	tsw := getTimeseriesByWorkerID()
+	seriesByWorkerID := tsw.byWorkerID
+	seriesLen := rss.Len()
 	err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error {
 		rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps)
 		preFunc(rs.Values, rs.Timestamps)
@@ -1187,23 +1204,25 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k
 			if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil {
 				samplesScanned := rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps)
 				atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-				tssLock.Lock()
-				tss = tsm.AppendTimeseriesTo(tss)
-				tssLock.Unlock()
+				seriesByWorkerID[workerID].tss = tsm.AppendTimeseriesTo(seriesByWorkerID[workerID].tss)
 				continue
 			}
 			var ts timeseries
 			samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps)
 			atomic.AddUint64(&samplesScannedTotal, samplesScanned)
-			tssLock.Lock()
-			tss = append(tss, &ts)
-			tssLock.Unlock()
+			seriesByWorkerID[workerID].tss = append(seriesByWorkerID[workerID].tss, &ts)
 		}
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	tss := make([]*timeseries, 0, seriesLen*len(rcs))
+	for i := range seriesByWorkerID {
+		tss = append(tss, seriesByWorkerID[i].tss...)
+	}
+	putTimeseriesByWorkerID(tsw)
+
 	rowsScannedPerQuery.Update(float64(samplesScannedTotal))
 	qt.Printf("samplesScanned=%d", samplesScannedTotal)
 	return tss, nil
@@ -1225,6 +1244,42 @@ func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConf
 	return samplesScanned
 }
 
+type timeseriesWithPadding struct {
+	tss []*timeseries
+
+	// The padding prevents false sharing on widespread platforms with
+	// 128 mod (cache line size) = 0 .
+	_ [128 - unsafe.Sizeof([]*timeseries{})%128]byte
+}
+
+type timeseriesByWorkerID struct {
+	byWorkerID []timeseriesWithPadding
+}
+
+func (tsw *timeseriesByWorkerID) reset() {
+	byWorkerID := tsw.byWorkerID
+	for i := range byWorkerID {
+		byWorkerID[i].tss = nil
+	}
+}
+
+func getTimeseriesByWorkerID() *timeseriesByWorkerID {
+	v := timeseriesByWorkerIDPool.Get()
+	if v == nil {
+		return &timeseriesByWorkerID{
+			byWorkerID: make([]timeseriesWithPadding, netstorage.MaxWorkers()),
+		}
+	}
+	return v.(*timeseriesByWorkerID)
+}
+
+func putTimeseriesByWorkerID(tsw *timeseriesByWorkerID) {
+	tsw.reset()
+	timeseriesByWorkerIDPool.Put(tsw)
+}
+
+var timeseriesByWorkerIDPool sync.Pool
+
 var bbPool bytesutil.ByteBufferPool
 
 func evalNumber(ec *EvalConfig, n float64) []*timeseries {

app/vmselect/promql/exec_test.go

@@ -6122,7 +6122,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() < 1300)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1200, 1200, 1200, 1200},
+			Values:     []float64{1000, 1200, nan, nan, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -6133,7 +6133,18 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() > 1500)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1600, 1600, 1600, 1600, 1800, 2000},
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`interpolate(tail_head_and_middle)`, func(t *testing.T) {
+		t.Parallel()
+		q := `interpolate(time() > 1100 and time() < 1300 default time() > 1700 and time() < 1900)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1200, 1400, 1600, 1800, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}

app/vmselect/promql/rollup_result_cache.go

@@ -441,7 +441,7 @@ func mustSaveRollupResultCacheKeyPrefix(path string) {
 var tooBigRollupResults = metrics.NewCounter("vm_too_big_rollup_results_total")
 
 // Increment this value every time the format of the cache changes.
-const rollupResultCacheVersion = 8
+const rollupResultCacheVersion = 9
 
 func marshalRollupResultCacheKey(dst []byte, expr metricsql.Expr, window, step int64, etfs [][]storage.TagFilter) []byte {
 	dst = append(dst, rollupResultCacheVersion)

app/vmselect/promql/timeseries.go

@@ -82,15 +82,17 @@ func marshalTimeseriesFast(dst []byte, tss []*timeseries, maxSize int, step int6
 		logger.Panicf("BUG: tss cannot be empty")
 	}
 
-	// Calculate the required size for marshaled tss.
-	size := 0
-	for _, ts := range tss {
-		size += ts.marshaledFastSizeNoTimestamps()
-	}
-	// timestamps are stored only once for all the tss, since they are identical.
+	// timestamps are stored only once for all the tss, since they must be identical
 	assertIdenticalTimestamps(tss, step)
-	size += 8 * len(tss[0].Timestamps)
+	timestamps := tss[0].Timestamps
 
+	// Calculate the required size for marshaled tss.
+	size := 8 + 8                          // 8 bytes for len(tss) and 8 bytes for len(timestamps)
+	size += 8 * len(timestamps)            // encoded timestamps
+	size += 8 * len(tss) * len(timestamps) // encoded values
+	for _, ts := range tss {
+		size += marshaledFastMetricNameSize(&ts.MetricName)
+	}
 	if size > maxSize {
 		// Do not marshal tss, since it would occupy too much space
 		return dst
@@ -98,176 +100,133 @@ func marshalTimeseriesFast(dst []byte, tss []*timeseries, maxSize int, step int6
 
 	// Allocate the buffer for the marshaled tss before its' marshaling.
 	// This should reduce memory fragmentation and memory usage.
-	dst = bytesutil.ResizeNoCopyMayOverallocate(dst, size)
-	dst = marshalFastTimestamps(dst[:0], tss[0].Timestamps)
+	dstLen := len(dst)
+	dst = bytesutil.ResizeWithCopyMayOverallocate(dst, size+dstLen)
+	dst = dst[:dstLen]
+
+	// Marshal timestamps and values at first, so they are 8-byte aligned.
+	// This prevents from SIGBUS error on arm architectures.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+	dst = encoding.MarshalUint64(dst, uint64(len(tss)))
+	dst = encoding.MarshalUint64(dst, uint64(len(timestamps)))
+	dst = marshalTimestampsFast(dst, timestamps)
 	for _, ts := range tss {
-		dst = ts.marshalFastNoTimestamps(dst)
+		dst = marshalValuesFast(dst, ts.Values)
+	}
+	for _, ts := range tss {
+		dst = marshalMetricNameFast(dst, &ts.MetricName)
 	}
 	return dst
 }
 
 // unmarshalTimeseriesFast unmarshals timeseries from src.
 //
-// The returned timeseries refer to src, so it is unsafe to modify it
-// until timeseries are in use.
+// The returned timeseries refer to src, so it is unsafe to modify it while timeseries are in use.
 func unmarshalTimeseriesFast(src []byte) ([]*timeseries, error) {
-	tail, timestamps, err := unmarshalFastTimestamps(src)
+	if len(src) < 16 {
+		return nil, fmt.Errorf("cannot unmarshal timeseries from %d bytes; need at least 16 bytes", len(src))
+	}
+	tssLen := encoding.UnmarshalUint64(src)
+	timestampsLen := encoding.UnmarshalUint64(src[8:])
+	src = src[16:]
+
+	// Unmarshal timestamps
+	tail, timestamps, err := unmarshalTimestampsFast(src, timestampsLen)
 	if err != nil {
 		return nil, err
 	}
 	src = tail
 
-	var tss []*timeseries
-	for len(src) > 0 {
+	tss := make([]*timeseries, tssLen)
+	for i := range tss {
 		var ts timeseries
-		ts.denyReuse = false
+		ts.denyReuse = true
 		ts.Timestamps = timestamps
+		tss[i] = &ts
+	}
 
-		tail, err := ts.unmarshalFastNoTimestamps(src)
+	// Unmarshal values
+	for _, ts := range tss {
+		tail, values, err := unmarshalValuesFast(src, timestampsLen)
+		if err != nil {
+			return nil, err
+		}
+		ts.Values = values
+		src = tail
+	}
+
+	// Unmarshal metric names for the time series
+	for _, ts := range tss {
+		tail, err := unmarshalMetricNameFast(&ts.MetricName, src)
 		if err != nil {
 			return nil, err
 		}
 		src = tail
+	}
 
-		tss = append(tss, &ts)
+	if len(src) > 0 {
+		return nil, fmt.Errorf("unexpected non-empty tail left after unmarshaling %d timeseries; len(tail)=%d", len(tss), len(src))
 	}
 	return tss, nil
 }
 
-// marshaledFastSizeNoTimestamps returns the size of marshaled ts
-// returned from marshalFastNoTimestamps.
-func (ts *timeseries) marshaledFastSizeNoTimestamps() int {
-	mn := &ts.MetricName
-	n := 2 + len(mn.MetricGroup)
+// marshaledFastMetricNameSize returns the size of marshaled mn returned from marshalMetricNameFast.
+func marshaledFastMetricNameSize(mn *storage.MetricName) int {
+	n := 0
+	n += 2 + len(mn.MetricGroup)
 	n += 2 // Length of tags.
 	for i := range mn.Tags {
 		tag := &mn.Tags[i]
 		n += 2 + len(tag.Key)
 		n += 2 + len(tag.Value)
 	}
-	n += 8 * len(ts.Values)
 	return n
 }
 
-// marshalFastNoTimestamps appends marshaled ts to dst and returns the result.
-//
-// It doesn't marshal timestamps.
-//
-// The result must be unmarshaled with unmarshalFastNoTimestamps.
-func (ts *timeseries) marshalFastNoTimestamps(dst []byte) []byte {
-	mn := &ts.MetricName
-	dst = marshalBytesFast(dst, mn.MetricGroup)
-	dst = encoding.MarshalUint16(dst, uint16(len(mn.Tags)))
-	// There is no need in tags' sorting - they must be sorted after unmarshaling.
-	for i := range mn.Tags {
-		tag := &mn.Tags[i]
-		dst = marshalBytesFast(dst, tag.Key)
-		dst = marshalBytesFast(dst, tag.Value)
-	}
-
-	// Do not marshal len(ts.Values), since it is already encoded as len(ts.Timestamps)
-	// during marshalFastTimestamps.
-	var valuesBuf []byte
-	if len(ts.Values) > 0 {
-		valuesBuf = float64ToByteSlice(ts.Values)
-	}
+func marshalValuesFast(dst []byte, values []float64) []byte {
+	// Do not marshal len(values), since it is already encoded as len(timestamps) at marshalTimestampsFast.
+	valuesBuf := float64ToByteSlice(values)
 	dst = append(dst, valuesBuf...)
 	return dst
 }
 
-func marshalFastTimestamps(dst []byte, timestamps []int64) []byte {
-	dst = encoding.MarshalUint32(dst, uint32(len(timestamps)))
-	var timestampsBuf []byte
-	if len(timestamps) > 0 {
-		timestampsBuf = int64ToByteSlice(timestamps)
+// it is unsafe modifying src while the returned values is in use.
+func unmarshalValuesFast(src []byte, valuesLen uint64) ([]byte, []float64, error) {
+	bufSize := valuesLen * 8
+	if uint64(len(src)) < bufSize {
+		return src, nil, fmt.Errorf("cannot unmarshal values; got %d ytes; want at least %d bytes", uint64(len(src)), bufSize)
 	}
+	values := byteSliceToFloat64(src[:bufSize])
+	return src[bufSize:], values, nil
+}
+
+func marshalTimestampsFast(dst []byte, timestamps []int64) []byte {
+	timestampsBuf := int64ToByteSlice(timestamps)
 	dst = append(dst, timestampsBuf...)
 	return dst
 }
 
 // it is unsafe modifying src while the returned timestamps is in use.
-func unmarshalFastTimestamps(src []byte) ([]byte, []int64, error) {
-	if len(src) < 4 {
-		return src, nil, fmt.Errorf("cannot decode len(timestamps); got %d bytes; want at least %d bytes", len(src), 4)
-	}
-	timestampsCount := int(encoding.UnmarshalUint32(src))
-	src = src[4:]
-	if timestampsCount == 0 {
-		return src, nil, nil
-	}
-
-	bufSize := timestampsCount * 8
-	if len(src) < bufSize {
+func unmarshalTimestampsFast(src []byte, timestampsLen uint64) ([]byte, []int64, error) {
+	bufSize := timestampsLen * 8
+	if uint64(len(src)) < bufSize {
 		return src, nil, fmt.Errorf("cannot unmarshal timestamps; got %d bytes; want at least %d bytes", len(src), bufSize)
 	}
 	timestamps := byteSliceToInt64(src[:bufSize])
-	src = src[bufSize:]
-
-	return src, timestamps, nil
+	return src[bufSize:], timestamps, nil
 }
 
-// unmarshalFastNoTimestamps unmarshals ts from src, so ts members reference src.
+// marshalMetricNameFast appends marshaled mn to dst and returns the result.
 //
-// It is expected that ts.Timestamps is already unmarshaled.
-//
-// It is unsafe to modify src while ts is in use.
-func (ts *timeseries) unmarshalFastNoTimestamps(src []byte) ([]byte, error) {
-	// ts members point to src, so they cannot be re-used.
-	ts.denyReuse = true
-
-	tail, err := unmarshalMetricNameFast(&ts.MetricName, src)
-	if err != nil {
-		return tail, fmt.Errorf("cannot unmarshal MetricName: %w", err)
-	}
-	src = tail
-
-	valuesCount := len(ts.Timestamps)
-	if valuesCount == 0 {
-		return src, nil
-	}
-	bufSize := valuesCount * 8
-	if len(src) < bufSize {
-		return src, fmt.Errorf("cannot unmarshal values; got %d bytes; want at least %d bytes", len(src), bufSize)
-	}
-	ts.Values = byteSliceToFloat64(src[:bufSize])
-
-	return src[bufSize:], nil
+// The result must be unmarshaled with unmarshalMetricNameFast.
+func marshalMetricNameFast(dst []byte, mn *storage.MetricName) []byte {
+	dst = marshalBytesFast(dst, mn.MetricGroup)
+	dst = encoding.MarshalUint16(dst, uint16(len(mn.Tags)))
+	// There is no need in tags' sorting - they must be sorted after unmarshaling.
+	return marshalMetricTagsFast(dst, mn.Tags)
 }
 
-func float64ToByteSlice(a []float64) (b []byte) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	sh.Data = uintptr(unsafe.Pointer(&a[0]))
-	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-func int64ToByteSlice(a []int64) (b []byte) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
-	sh.Data = uintptr(unsafe.Pointer(&a[0]))
-	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-func byteSliceToInt64(b []byte) (a []int64) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
-	sh.Data = uintptr(unsafe.Pointer(&b[0]))
-	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-func byteSliceToFloat64(b []byte) (a []float64) {
-	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
-	sh.Data = uintptr(unsafe.Pointer(&b[0]))
-	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
-	sh.Cap = sh.Len
-	return
-}
-
-// unmarshalMetricNameFast unmarshals mn from src, so mn members
-// hold references to src.
+// unmarshalMetricNameFast unmarshals mn from src, so mn members hold references to src.
 //
 // It is unsafe modifying src while mn is in use.
 func unmarshalMetricNameFast(mn *storage.MetricName, src []byte) ([]byte, error) {
@@ -320,9 +279,7 @@ func marshalMetricTagsFast(dst []byte, tags []storage.Tag) []byte {
 
 func marshalMetricNameSorted(dst []byte, mn *storage.MetricName) []byte {
 	dst = marshalBytesFast(dst, mn.MetricGroup)
-	sortMetricTags(mn)
-	dst = marshalMetricTagsFast(dst, mn.Tags)
-	return dst
+	return marshalMetricTagsSorted(dst, mn)
 }
 
 func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
@@ -348,6 +305,62 @@ func unmarshalBytesFast(src []byte) ([]byte, []byte, error) {
 	return src[n:], src[:n], nil
 }
 
+func float64ToByteSlice(a []float64) (b []byte) {
+	if len(a) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh.Data = uintptr(unsafe.Pointer(&a[0]))
+	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func int64ToByteSlice(a []int64) (b []byte) {
+	if len(a) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	sh.Data = uintptr(unsafe.Pointer(&a[0]))
+	sh.Len = len(a) * int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	return
+}
+
+func byteSliceToInt64(b []byte) (a []int64) {
+	if len(b) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	sh.Data = uintptr(unsafe.Pointer(&b[0]))
+	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	// Make sure that the returned slice is properly aligned to 8 bytes.
+	// This prevents from SIGBUS error on arm architectures, which deny unaligned access.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+	if sh.Data%8 != 0 {
+		logger.Panicf("BUG: the input byte slice b must be aligned to 8 bytes")
+	}
+	return
+}
+
+func byteSliceToFloat64(b []byte) (a []float64) {
+	if len(b) == 0 {
+		return nil
+	}
+	sh := (*reflect.SliceHeader)(unsafe.Pointer(&a))
+	sh.Data = uintptr(unsafe.Pointer(&b[0]))
+	sh.Len = len(b) / int(unsafe.Sizeof(a[0]))
+	sh.Cap = sh.Len
+	// Make sure that the returned slice is properly aligned to 8 bytes.
+	// This prevents from SIGBUS error on arm architectures, which deny unaligned access.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+	if sh.Data%8 != 0 {
+		logger.Panicf("BUG: the input byte slice b must be aligned to 8 bytes")
+	}
+	return
+}
+
 func stringMetricName(mn *storage.MetricName) string {
 	var dst []byte
 	dst = append(dst, mn.MetricGroup...)

app/vmselect/promql/timeseries_test.go

@@ -1,10 +1,10 @@
 package promql
 
 import (
-	"fmt"
 	"os"
 	"reflect"
 	"testing"
+	"unsafe"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
@@ -14,91 +14,98 @@ func TestMain(m *testing.M) {
 	os.Exit(n)
 }
 
-func TestTimeseriesMarshalUnmarshalFast(t *testing.T) {
-	t.Run("single", func(t *testing.T) {
-		var tsOrig timeseries
-		buf := tsOrig.marshalFastNoTimestamps(nil)
-		n := tsOrig.marshaledFastSizeNoTimestamps()
-		if n != len(buf) {
-			t.Fatalf("unexpected marshaled size; got %d; want %d", n, len(buf))
-		}
-
-		var tsGot timeseries
-		tail, err := tsGot.unmarshalFastNoTimestamps(buf)
+func TestMarshalTimeseriesFast(t *testing.T) {
+	f := func(tss []*timeseries) {
+		t.Helper()
+		data := marshalTimeseriesFast(nil, tss, 1e9, 10)
+		tss2, err := unmarshalTimeseriesFast(data)
 		if err != nil {
 			t.Fatalf("cannot unmarshal timeseries: %s", err)
 		}
-		if len(tail) > 0 {
-			t.Fatalf("unexpected non-empty tail left: len(tail)=%d; tail=%X", len(tail), tail)
-		}
-		tsOrig.denyReuse = true
-		tsOrig.MetricName.MetricGroup = []byte{}
-		if !reflect.DeepEqual(&tsOrig, &tsGot) {
-			t.Fatalf("unexpected ts\ngot:\n%s\nwant:\n%s", &tsGot, &tsOrig)
-		}
-	})
-	t.Run("multiple", func(t *testing.T) {
-		var dst []byte
-		var tssOrig []*timeseries
-		timestamps := []int64{2}
-		for i := 0; i < 10; i++ {
-			var ts timeseries
-			ts.denyReuse = true
-			ts.MetricName.MetricGroup = []byte(fmt.Sprintf("metricGroup %d", i))
-			ts.MetricName.Tags = []storage.Tag{{
-				Key:   []byte(fmt.Sprintf("key %d", i)),
-				Value: []byte(fmt.Sprintf("value %d", i)),
-			}}
-			ts.Values = []float64{float64(i) + 0.2}
-			ts.Timestamps = timestamps
-
-			dstLen := len(dst)
-			dst = ts.marshalFastNoTimestamps(dst)
-			n := ts.marshaledFastSizeNoTimestamps()
-			if n != len(dst)-dstLen {
-				t.Fatalf("unexpected marshaled size on iteration %d; got %d; want %d", i, n, len(dst)-dstLen)
-			}
-
-			var tsGot timeseries
-			tsGot.Timestamps = ts.Timestamps
-			tail, err := tsGot.unmarshalFastNoTimestamps(dst[dstLen:])
-			if err != nil {
-				t.Fatalf("cannot unmarshal timeseries on iteration %d: %s", i, err)
-			}
-			if len(tail) > 0 {
-				t.Fatalf("unexpected non-empty tail left on iteration %d: len(tail)=%d; tail=%x", i, len(tail), tail)
-			}
-			if !reflect.DeepEqual(&ts, &tsGot) {
-				t.Fatalf("unexpected ts on iteration %d\ngot:\n%s\nwant:\n%s", i, &tsGot, &ts)
-			}
-
-			tssOrig = append(tssOrig, &ts)
-		}
-		buf := marshalTimeseriesFast(nil, tssOrig, 1e6, 123)
-		tssGot, err := unmarshalTimeseriesFast(buf)
-		if err != nil {
-			t.Fatalf("error in unmarshalTimeseriesFast: %s", err)
-		}
-		if !reflect.DeepEqual(tssOrig, tssGot) {
-			t.Fatalf("unexpected unmarshaled timeseries\ngot:\n%s\nwant:\n%s", tssGot, tssOrig)
+		if !reflect.DeepEqual(tss, tss2) {
+			t.Fatalf("unexpected timeseries unmarshaled\ngot\n%#v\nwant\n%#v", tss2[0], tss[0])
 		}
 
-		src := dst
-		for i := 0; i < 10; i++ {
-			tsOrig := tssOrig[i]
-			var ts timeseries
-			ts.Timestamps = tsOrig.Timestamps
-			tail, err := ts.unmarshalFastNoTimestamps(src)
-			if err != nil {
-				t.Fatalf("cannot unmarshal timeseries[%d]: %s", i, err)
+		// Check 8-byte alignment.
+		// This prevents from SIGBUS error on arm architectures.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3927
+		for _, ts := range tss2 {
+			if len(ts.Values) == 0 {
+				continue
 			}
-			src = tail
-			if !reflect.DeepEqual(tsOrig, &ts) {
-				t.Fatalf("unexpected ts on iteration %d:\n%+v\nwant:\n%+v", i, &ts, tsOrig)
+
+			// check float64 alignment
+			addr := uintptr(unsafe.Pointer(&ts.Values[0]))
+			if mod := addr % unsafe.Alignof(ts.Values[0]); mod != 0 {
+				t.Fatalf("mis-aligned; &ts.Values[0]=%p; mod=%d", &ts.Values[0], mod)
+			}
+			// check int64 alignment
+			addr = uintptr(unsafe.Pointer(&ts.Timestamps[0]))
+			if mod := addr % unsafe.Alignof(ts.Timestamps[0]); mod != 0 {
+				t.Fatalf("mis-aligned; &ts.Timestamps[0]=%p; mod=%d", &ts.Timestamps[0], mod)
 			}
 		}
-		if len(src) > 0 {
-			t.Fatalf("unexpected tail left; len(tail)=%d; tail=%X", len(src), src)
-		}
+	}
+
+	// Single series
+	f([]*timeseries{{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte{},
+		},
+		denyReuse: true,
+	}})
+	f([]*timeseries{{
+		MetricName: storage.MetricName{
+			MetricGroup: []byte("foobar"),
+			Tags: []storage.Tag{
+				{
+					Key:   []byte("tag1"),
+					Value: []byte("value1"),
+				},
+				{
+					Key:   []byte("tag2"),
+					Value: []byte("value2"),
+				},
+			},
+		},
+		Values:     []float64{1, 2, 3.234},
+		Timestamps: []int64{10, 20, 30},
+		denyReuse:  true,
+	}})
+
+	// Multiple series
+	f([]*timeseries{
+		{
+			MetricName: storage.MetricName{
+				MetricGroup: []byte("foobar"),
+				Tags: []storage.Tag{
+					{
+						Key:   []byte("tag1"),
+						Value: []byte("value1"),
+					},
+					{
+						Key:   []byte("tag2"),
+						Value: []byte("value2"),
+					},
+				},
+			},
+			Values:     []float64{1, 2.34, -33},
+			Timestamps: []int64{-10, 0, 10},
+			denyReuse:  true,
+		},
+		{
+			MetricName: storage.MetricName{
+				MetricGroup: []byte("baz"),
+				Tags: []storage.Tag{
+					{
+						Key:   []byte("tag12"),
+						Value: []byte("value13"),
+					},
+				},
+			},
+			Values:     []float64{4, 1, 2.34},
+			Timestamps: []int64{-10, 0, 10},
+			denyReuse:  true,
+		},
 	})
 }

app/vmselect/promql/transform.go

@@ -552,18 +552,28 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 		for _, xs := range xss {
 			ts := xs.ts
 			if isZeroTS(ts) {
-				// Skip time series with zeros. They are substituted by xssNew below.
-				xsPrev = xs
+				// Skip buckets with zero values - they will be merged into a single bucket
+				// when the next non-zero bucket appears.
+
+				// Do not store xs in xsPrev in order to properly create `le` time series
+				// for zero buckets.
+				// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4021
 				continue
 			}
-			if xs.start != xsPrev.end && uniqTs[xs.startStr] == nil {
-				uniqTs[xs.startStr] = xs.ts
-				xssNew = append(xssNew, x{
-					endStr: xs.startStr,
-					end:    xs.start,
-					ts:     copyTS(ts, xs.startStr),
-				})
+			if xs.start != xsPrev.end {
+				// There is a gap between the previous bucket and the current bucket
+				// or the previous bucket is skipped because it was zero.
+				// Fill it with a time series with le=xs.start.
+				if uniqTs[xs.startStr] == nil {
+					uniqTs[xs.startStr] = xs.ts
+					xssNew = append(xssNew, x{
+						endStr: xs.startStr,
+						end:    xs.start,
+						ts:     copyTS(ts, xs.startStr),
+					})
+				}
 			}
+			// Convert the current time series to a time series with le=xs.end
 			ts.MetricName.AddTag("le", xs.endStr)
 			prevTs := uniqTs[xs.endStr]
 			if prevTs != nil {
@@ -575,7 +585,7 @@ func vmrangeBucketsToLE(tss []*timeseries) []*timeseries {
 			}
 			xsPrev = xs
 		}
-		if !math.IsInf(xsPrev.end, 1) && !isZeroTS(xsPrev.ts) {
+		if xsPrev.ts != nil && !math.IsInf(xsPrev.end, 1) && !isZeroTS(xsPrev.ts) {
 			xssNew = append(xssNew, x{
 				endStr: "+Inf",
 				end:    math.Inf(1),
@@ -1165,7 +1175,8 @@ func transformInterpolate(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 	rvs := args[0]
 	for _, ts := range rvs {
-		values := ts.Values
+		values := skipLeadingNaNs(ts.Values)
+		values = skipTrailingNaNs(values)
 		if len(values) == 0 {
 			continue
 		}

app/vmselect/promql/transform_test.go

@@ -78,6 +78,36 @@ foo{le="+Inf"} 1.23 456`,
 foo{le="+Inf"} 5.3 0`,
 	)
 
+	// Adjacent empty vmrange bucket
+	f(
+		`foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+
+	// Multiple adjacent empty vmrange bucket
+	f(
+		`foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123
+foo{vmrange="5.813e+05...6.813e+05"} 0 123
+`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+	f(
+		`foo{vmrange="8.799e+05...9.813e+05"} 0 123
+foo{vmrange="7.743e+05...8.799e+05"} 5 123
+foo{vmrange="6.813e+05...7.743e+05"} 0 123
+foo{vmrange="5.813e+05...6.813e+05"} 0 123
+`,
+		`foo{le="7.743e+05"} 0 123
+foo{le="8.799e+05"} 5 123
+foo{le="+Inf"} 5 123`,
+	)
+
 	// Multiple non-empty vmrange buckets
 	f(
 		`foo{vmrange="4.084e+02...4.642e+02"} 2 123

app/vmselect/vmui/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
-    "main.css": "./static/css/main.3f9cb68f.css",
-    "main.js": "./static/js/main.b1572032.js",
+    "main.css": "./static/css/main.b9c2d13c.css",
+    "main.js": "./static/js/main.40a4969a.js",
     "static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js",
     "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
     "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.3f9cb68f.css",
-    "static/js/main.b1572032.js"
+    "static/css/main.b9c2d13c.css",
+    "static/js/main.40a4969a.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.b1572032.js"></script><link href="./static/css/main.3f9cb68f.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.40a4969a.js"></script><link href="./static/css/main.b9c2d13c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmstorage/main.go

@@ -37,8 +37,10 @@ var (
 	finalMergeDelay = flag.Duration("finalMergeDelay", 0, "The delay before starting final merge for per-month partition after no new data is ingested into it. "+
 		"Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. "+
 		"Zero value disables final merge")
-	bigMergeConcurrency     = flag.Int("bigMergeConcurrency", 0, "The maximum number of CPU cores to use for big merges. Default value is used if set to 0")
-	smallMergeConcurrency   = flag.Int("smallMergeConcurrency", 0, "The maximum number of CPU cores to use for small merges. Default value is used if set to 0")
+	_ = flag.Int("bigMergeConcurrency", 0, "Deprecated: this flag does nothing. Please use -smallMergeConcurrency "+
+		"for controlling the concurrency of background merges. See https://docs.victoriametrics.com/#storage")
+	smallMergeConcurrency = flag.Int("smallMergeConcurrency", 0, "The maximum number of workers for background merges. See https://docs.victoriametrics.com/#storage . "+
+		"It isn't recommended tuning this flag in general case, since this may lead to uncontrolled increase in the number of parts and increased CPU usage during queries")
 	retentionTimezoneOffset = flag.Duration("retentionTimezoneOffset", 0, "The offset for performing indexdb rotation. "+
 		"If set to 0, then the indexdb rotation is performed at 4am UTC time per each -retentionPeriod. "+
 		"If set to 2h, then the indexdb rotation is performed at 4am EET time (the timezone with +2h offset)")
@@ -91,7 +93,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	resetResponseCacheIfNeeded = resetCacheIfNeeded
 	storage.SetLogNewSeries(*logNewSeries)
 	storage.SetFinalMergeDelay(*finalMergeDelay)
-	storage.SetBigMergeWorkersCount(*bigMergeConcurrency)
 	storage.SetMergeWorkersCount(*smallMergeConcurrency)
 	storage.SetRetentionTimezoneOffset(*retentionTimezoneOffset)
 	storage.SetFreeDiskSpaceLimit(minFreeDiskSpaceBytes.N)

app/vmui/Dockerfile-build

@@ -1,4 +1,4 @@
-FROM node:18-alpine3.15
+FROM node:18-alpine3.17
 
 RUN apk update && apk upgrade
 RUN apk add --no-cache bash bash-doc bash-completion libtool autoconf automake nasm pkgconfig libpng gcc make g++ zlib-dev gawk

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.19.5 as build-web-stage
+FROM golang:1.21.1 as build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.17.1
+FROM alpine:3.18.0
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/README.md

@@ -60,7 +60,7 @@ VMUI can be used to paste into other applications
 | Name                    |   Default   |                                                                           Description |
 |:------------------------|:-----------:|--------------------------------------------------------------------------------------:|
 | serverURL               | domain name |                                                          Can't be changed from the UI |
-| inputTenantID           |      -      |                            If the flag is present, the "Tenant ID" field is displayed |
+| useTenantID             |      -      |                           If the flag is present, the "Tenant ID" select is displayed |
 | headerStyles.background |  `#FFFFFF`  |                                                               Header background color |
 | headerStyles.color      |  `#3F51B5`  |                                                                     Header font color |
 | palette.primary         |  `#3F51B5`  |                               used to represent primary interface elements for a user |
@@ -74,7 +74,7 @@ VMUI can be used to paste into other applications
 ```json
 {
   "serverURL": "http://localhost:8428",
-  "inputTenantID": "true",
+  "useTenantID": true,
   "headerStyles": {
     "background": "#FFFFFF",
     "color": "#538DE8"
@@ -93,7 +93,7 @@ VMUI can be used to paste into other applications
 
 #### HTML example:
 ```html
-<div id="root" data-params='{"serverURL":"http://localhost:8428","inputTenantID":"true","headerStyles":{"background":"#FFFFFF","color":"#538DE8"},"palette":{"primary":"#538DE8","secondary":"#F76F8E","error":"#FD151B","warning":"#FFB30F","success":"#7BE622","info":"#0F5BFF"}}'></div>
+<div id="root" data-params='{"serverURL":"http://localhost:8428","useTenantID":true,"headerStyles":{"background":"#FFFFFF","color":"#538DE8"},"palette":{"primary":"#538DE8","secondary":"#F76F8E","error":"#FD151B","warning":"#FFB30F","success":"#7BE622","info":"#0F5BFF"}}'></div>
 ```
 
 

app/vmui/packages/vmui/public/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -1,5 +1,5 @@
 import React, { FC, useEffect, useMemo, useRef, useState } from "preact/compat";
-import uPlot, { Series } from "uplot";
+import uPlot from "uplot";
 import { MetricResult } from "../../../api/types";
 import { formatPrettyNumber } from "../../../utils/uplot/helpers";
 import dayjs from "dayjs";
@@ -11,12 +11,13 @@ import { CloseIcon, DragIcon } from "../../Main/Icons";
 import classNames from "classnames";
 import { MouseEvent as ReactMouseEvent } from "react";
 import "./style.scss";
+import { SeriesItem } from "../../../utils/uplot/series";
 
 export interface ChartTooltipProps {
   id: string,
   u: uPlot,
   metrics: MetricResult[],
-  series: Series[],
+  series: SeriesItem[],
   yRange: number[];
   unit?: string,
   isSticky?: boolean,
@@ -55,15 +56,16 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
 
   const color = series[seriesIdx]?.stroke+"";
 
+  const calculations = series[seriesIdx]?.calculations || {};
+
   const groups = new Set(metrics.map(m => m.group));
   const showQueryNum = groups.size > 1;
   const group = metrics[seriesIdx-1]?.group || 0;
 
   const metric = metrics[seriesIdx-1]?.metric || {};
-  const labelNames = Object.keys(metric).filter(x => x != "__name__");
-  const metricName = metric["__name__"] || "value";
 
   const fields = useMemo(() => {
+    const labelNames = Object.keys(metric).filter(x => x != "__name__");
     return labelNames.map(key => `${key}=${JSON.stringify(metric[key])}`);
   }, [metrics, seriesIdx]);
 
@@ -100,10 +102,15 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
     const overflowX = leftOnChart + tooltipWidth >= width ? tooltipWidth + (2 * margin) : 0;
     const overflowY = topOnChart + tooltipHeight >= height ? tooltipHeight + (2 * margin) : 0;
 
-    setPosition({
+    const position = {
       top: topOnChart + tooltipOffset.top + margin - overflowY,
       left: leftOnChart + tooltipOffset.left + margin - overflowX
-    });
+    };
+
+    if (position.left < 0) position.left = 20;
+    if (position.top < 0) position.top = 20;
+
+    setPosition(position);
   };
 
   useEffect(calcPosition, [u, value, dataTime, seriesIdx, tooltipOffset, tooltipRef]);
@@ -169,19 +176,21 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
           className="vm-chart-tooltip-data__marker"
           style={{ background: color }}
         />
-        <p>
-          {metricName}: 
-          <b className="vm-chart-tooltip-data__value">{valueFormat}</b>
-          {unit}
-        </p>
-      </div>
-      {!!fields.length && (
-        <div className="vm-chart-tooltip-info">
-          {fields.map((f, i) => (
-            <div key={`${f}_${i}`}>{f}</div>
-          ))}
+        <div>
+          curr:<b>{valueFormat}{unit}</b>, avg:<b>{calculations.avg}</b><br/>
+          min:<b>{calculations.min}</b>, max:<b>{calculations.max}</b>, last:<b>{calculations.last}</b>
         </div>
-      )}
+      </div>
+      <div className="vm-chart-tooltip-info">
+        {metric["__name__"]}
+        &#123;
+        {fields.map((f, i) => (
+          <span key="{i}">
+            {f}{i +1 < fields.length && ","}
+          </span>
+        ))}
+        &#125;
+      </div>
     </div>
   ), targetPortal);
 };

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -61,11 +61,6 @@ $chart-tooltip-y: -1 * ($padding-small + $chart-tooltip-half-icon);
     word-break: break-all;
     line-height: 12px;
 
-    &__value {
-      padding: 4px;
-      font-weight: bold;
-    }
-
     &__marker {
       width: 12px;
       height: 12px;

app/vmui/packages/vmui/src/components/Chart/Legend/LegendItem/LegendItem.tsx

@@ -14,6 +14,7 @@ interface LegendItemProps {
 const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
   const [copiedValue, setCopiedValue] = useState("");
   const freeFormFields = useMemo(() => getFreeFields(legend), [legend]);
+  const calculations = legend.calculations;
 
   const handleClickFreeField = async (val: string, id: string) => {
     await navigator.clipboard.writeText(val);
@@ -30,11 +31,11 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
     handleClickFreeField(freeField, id);
   };
 
-
   return (
     <div
       className={classNames({
         "vm-legend-item": true,
+        "vm-legend-row": true,
         "vm-legend-item_hide": !legend.checked,
       })}
       onClick={createHandlerClick(legend)}
@@ -45,30 +46,30 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
       />
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
-          {legend.freeFormFields["__name__"] || (freeFormFields.length == 0 ? "{}" : "")}
-        </span>
-        {freeFormFields.length > 0 &&
-          <span>
-            &#123;
-            {freeFormFields.map(f => (
-              <Tooltip
-                key={f.id}
-                open={copiedValue === f.id}
-                title={"Copied!"}
-                placement="top-center"
+          {legend.freeFormFields["__name__"]}
+          &#123;
+          {freeFormFields.map((f, i) => (
+            <Tooltip
+              key={f.id}
+              open={copiedValue === f.id}
+              title={"copied!"}
+              placement="top-center"
+            >
+              <span
+                className="vm-legend-item-info__free-fields"
+                key={f.key}
+                onClick={createHandlerCopy(f.freeField, f.id)}
+                title="copy to clipboard"
               >
-                <span
-                  className="vm-legend-item-info__free-fields"
-                  key={f.key}
-                  onClick={createHandlerCopy(f.freeField, f.id)}
-                >
-                  {f.freeField}
-                </span>
-              </Tooltip>
-            ))}
-            &#125;
-          </span>
-        }
+                {f.freeField}{i + 1 < freeFormFields.length && ","}
+              </span>
+            </Tooltip>
+          ))}
+          &#125;
+        </span>
+      </div>
+      <div className="vm-legend-item-values">
+        avg:{calculations.avg}, min:{calculations.min}, max:{calculations.max}, last:{calculations.last}
       </div>
     </div>
   );

app/vmui/packages/vmui/src/components/Chart/Legend/LegendItem/style.scss

@@ -6,10 +6,11 @@
   grid-gap: $padding-small;
   align-items: start;
   justify-content: start;
-  padding: $padding-small $padding-large $padding-small $padding-small;
+  padding: $padding-small;
   background-color: $color-background-block;
   cursor: pointer;
   transition: 0.2s ease;
+  margin-bottom: $padding-small;
 
   &:hover {
     background-color: rgba(0, 0, 0, 0.1);
@@ -30,22 +31,26 @@
 
   &-info {
     font-weight: normal;
+    word-break: break-all;
 
     &__label {
-
+      margin-right: 2px;
     }
 
     &__free-fields {
-      padding: 3px;
+      padding: 2px;
       cursor: pointer;
 
       &:hover {
         text-decoration: underline;
       }
-
-      &:not(:last-child):after {
-        content: ",";
-      }
     }
   }
+
+  &-values {
+    grid-column: 2;
+    display: flex;
+    align-items: center;
+    gap: $padding-small;
+  }
 }

app/vmui/packages/vmui/src/components/Chart/Legend/style.scss

@@ -9,12 +9,13 @@
 
   &-group {
     min-width: 23%;
+    width: 100%;
     margin: 0 $padding-global $padding-global 0;
 
     &-title {
       display: flex;
       align-items: center;
-      padding: 0 $padding-small $padding-small;
+      padding: $padding-small;
       margin-bottom: 1px;
       border-bottom: $border-divider;
 

app/vmui/packages/vmui/src/components/Chart/LineChart/LineChart.tsx

@@ -22,6 +22,7 @@ import classNames from "classnames";
 import ChartTooltip, { ChartTooltipProps } from "../ChartTooltip/ChartTooltip";
 import dayjs from "dayjs";
 import { useAppState } from "../../../state/common/StateContext";
+import { SeriesItem } from "../../../utils/uplot/series";
 
 export interface LineChartProps {
   metrics: MetricResult[];
@@ -55,6 +56,7 @@ const LineChart: FC<LineChartProps> = ({
   const [xRange, setXRange] = useState({ min: period.start, max: period.end });
   const [yRange, setYRange] = useState([0, 1]);
   const [uPlotInst, setUPlotInst] = useState<uPlot>();
+  const [startTouchDistance, setStartTouchDistance] = useState(0);
   const layoutSize = useResize(container);
 
   const [showTooltip, setShowTooltip] = useState(false);
@@ -84,6 +86,7 @@ const LineChart: FC<LineChartProps> = ({
       left: parseFloat(u.over.style.left),
       top: parseFloat(u.over.style.top)
     });
+
     u.over.addEventListener("mousedown", e => {
       const { ctrlKey, metaKey, button } = e;
       const leftClick = button === 0;
@@ -94,6 +97,10 @@ const LineChart: FC<LineChartProps> = ({
       }
     });
 
+    u.over.addEventListener("touchstart", e => {
+      dragChart({ u, e, setPanning, setPlotScale, factor });
+    });
+
     u.over.addEventListener("wheel", e => {
       if (!e.ctrlKey && !e.metaKey) return;
       e.preventDefault();
@@ -235,6 +242,47 @@ const LineChart: FC<LineChartProps> = ({
     };
   }, [xRange]);
 
+  const handleTouchStart = (e: TouchEvent) => {
+    if (e.touches.length !== 2) return;
+    e.preventDefault();
+
+    const dx = e.touches[0].clientX - e.touches[1].clientX;
+    const dy = e.touches[0].clientY - e.touches[1].clientY;
+    setStartTouchDistance(Math.sqrt(dx * dx + dy * dy));
+  };
+
+  const handleTouchMove = (e: TouchEvent) => {
+    if (e.touches.length !== 2 || !uPlotInst) return;
+    e.preventDefault();
+
+    const dx = e.touches[0].clientX - e.touches[1].clientX;
+    const dy = e.touches[0].clientY - e.touches[1].clientY;
+    const endTouchDistance = Math.sqrt(dx * dx + dy * dy);
+    const diffDistance = startTouchDistance - endTouchDistance;
+
+    const max = (uPlotInst.scales.x.max || xRange.max);
+    const min = (uPlotInst.scales.x.min || xRange.min);
+    const dur = max - min;
+    const dir = (diffDistance > 0 ? -1 : 1);
+
+    const zoomFactor = dur / 50 * dir;
+    uPlotInst.batch(() => setPlotScale({
+      u: uPlotInst,
+      min: min + zoomFactor,
+      max: max - zoomFactor
+    }));
+  };
+
+  useEffect(() => {
+    window.addEventListener("touchmove", handleTouchMove);
+    window.addEventListener("touchstart", handleTouchStart);
+
+    return () => {
+      window.removeEventListener("touchmove", handleTouchMove);
+      window.removeEventListener("touchstart", handleTouchStart);
+    };
+  }, [uPlotInst, startTouchDistance]);
+
   useEffect(() => updateChart(typeChartUpdate.data), [data]);
   useEffect(() => updateChart(typeChartUpdate.xRange), [xRange]);
   useEffect(() => updateChart(typeChartUpdate.yRange), [yaxis]);
@@ -256,6 +304,10 @@ const LineChart: FC<LineChartProps> = ({
         "vm-line-chart": true,
         "vm-line-chart_panning": isPanning
       })}
+      style={{
+        minWidth: `${layoutSize.width || 400}px`,
+        minHeight: `${height || 500}px`
+      }}
     >
       <div
         className="vm-line-chart__u-plot"
@@ -265,7 +317,7 @@ const LineChart: FC<LineChartProps> = ({
         <ChartTooltip
           unit={unit}
           u={uPlotInst}
-          series={series}
+          series={series as SeriesItem[]}
           metrics={metrics}
           yRange={yRange}
           tooltipIdx={tooltipIdx}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -14,10 +14,12 @@ import classNames from "classnames";
 import Timezones from "./Timezones/Timezones";
 import { useTimeDispatch, useTimeState } from "../../../state/time/TimeStateContext";
 import ThemeControl from "../ThemeControl/ThemeControl";
+import useDeviceDetect from "../../../hooks/useDeviceDetect";
 
 const title = "Settings";
 
-const GlobalSettings: FC = () => {
+const GlobalSettings: FC<{showTitle?: boolean}> = ({ showTitle }) => {
+  const { isMobile } = useDeviceDetect();
 
   const appModeEnable = getAppModeEnable();
   const { serverUrl: stateServerUrl } = useAppState();
@@ -49,7 +51,10 @@ const GlobalSettings: FC = () => {
   }, [stateServerUrl]);
 
   return <>
-    <Tooltip title={title}>
+    <Tooltip
+      open={showTitle === true ? false : undefined}
+      title={title}
+    >
       <Button
         className={classNames({
           "vm-header-button": !appModeEnable
@@ -58,14 +63,21 @@ const GlobalSettings: FC = () => {
         color="primary"
         startIcon={<SettingsIcon/>}
         onClick={handleOpen}
-      />
+      >
+        {showTitle && title}
+      </Button>
     </Tooltip>
     {open && (
       <Modal
         title={title}
         onClose={handleClose}
       >
-        <div className="vm-server-configurator">
+        <div
+          className={classNames({
+            "vm-server-configurator": true,
+            "vm-server-configurator_mobile": isMobile
+          })}
+        >
           {!appModeEnable && (
             <div className="vm-server-configurator__input">
               <ServerConfigurator
@@ -88,9 +100,11 @@ const GlobalSettings: FC = () => {
               onChange={setTimezone}
             />
           </div>
-          <div className="vm-server-configurator__input">
-            <ThemeControl/>
-          </div>
+          {!appModeEnable && (
+            <div className="vm-server-configurator__input">
+              <ThemeControl/>
+            </div>
+          )}
           <div className="vm-server-configurator__footer">
             <Button
               variant="outlined"

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -70,15 +70,16 @@ const LimitsConfigurator: FC<ServerConfiguratorProps> = ({ limits, onChange , on
       </div>
       <div className="vm-limits-configurator__inputs">
         {fields.map(f => (
-          <TextField
-            key={f.type}
-            label={f.label}
-            value={limits[f.type]}
-            error={error[f.type]}
-            onChange={createChangeHandler(f.type)}
-            onEnter={onEnter}
-            type="number"
-          />
+          <div key={f.type}>
+            <TextField
+              label={f.label}
+              value={limits[f.type]}
+              error={error[f.type]}
+              onChange={createChangeHandler(f.type)}
+              onEnter={onEnter}
+              type="number"
+            />
+          </div>
         ))}
       </div>
     </div>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/style.scss

@@ -12,10 +12,14 @@
   }
 
   &__inputs {
-    display: grid;
-    grid-template-columns: repeat(3, 1fr);
+    display: flex;
+    flex-wrap: wrap;
     align-items: center;
     justify-content: space-between;
     gap: $padding-global;
+
+    div {
+      flex-grow: 1;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration.tsx

@@ -1,7 +1,7 @@
 import React, { FC, useState, useRef, useEffect, useMemo } from "preact/compat";
 import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
 import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
-import { ArrowDownIcon, StorageIcons } from "../../../Main/Icons";
+import { ArrowDownIcon, StorageIcon } from "../../../Main/Icons";
 import Button from "../../../Main/Button/Button";
 import "./style.scss";
 import { replaceTenantId } from "../../../../utils/default-server-url";
@@ -9,17 +9,32 @@ import classNames from "classnames";
 import Popper from "../../../Main/Popper/Popper";
 import { getAppModeEnable } from "../../../../utils/app-mode";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import TextField from "../../../Main/TextField/TextField";
 
 const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
   const appModeEnable = getAppModeEnable();
+  const { isMobile } = useDeviceDetect();
 
   const { tenantId: tenantIdState, serverUrl } = useAppState();
   const dispatch = useAppDispatch();
   const timeDispatch = useTimeDispatch();
 
+  const [search, setSearch] = useState("");
   const [openOptions, setOpenOptions] = useState(false);
   const optionsButtonRef = useRef<HTMLDivElement>(null);
 
+  const accountIdsFiltered = useMemo(() => {
+    if (!search) return accountIds;
+    try {
+      const regexp = new RegExp(search, "i");
+      const found = accountIds.filter((item) => regexp.test(item));
+      return found.sort((a,b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
+    } catch (e) {
+      return [];
+    }
+  }, [search, accountIds]);
+
   const getTenantIdFromUrl = (url: string) => {
     const regexp = /(\/select\/)(\d+|\d.+)(\/)(.+)/;
     return (url.match(regexp) || [])[2];
@@ -71,8 +86,8 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
             variant="contained"
             color="primary"
             fullWidth
-            startIcon={<StorageIcons/>}
-            endIcon={(
+            startIcon={<StorageIcon/>}
+            endIcon={!isMobile ? (
               <div
                 className={classNames({
                   "vm-execution-controls-buttons__arrow": true,
@@ -81,22 +96,29 @@ const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
               >
                 <ArrowDownIcon/>
               </div>
-            )}
+            ) : undefined}
             onClick={toggleOpenOptions}
           >
-            {tenantIdState}
+            {!isMobile && tenantIdState}
           </Button>
         </div>
       </Tooltip>
       <Popper
         open={openOptions}
-        placement="bottom-left"
+        placement="bottom-right"
         onClose={handleCloseOptions}
         buttonRef={optionsButtonRef}
-        fullWidth
       >
-        <div className="vm-list">
-          {accountIds.map(id => (
+        <div className="vm-list vm-tenant-input-list">
+          <div className="vm-tenant-input-list__search">
+            <TextField
+              autofocus
+              label="Search"
+              value={search}
+              onChange={setSearch}
+            />
+          </div>
+          {accountIdsFiltered.map(id => (
             <div
               className={classNames({
                 "vm-list-item": true,

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds.ts

@@ -2,8 +2,10 @@ import { useAppState } from "../../../../../state/common/StateContext";
 import { useEffect, useMemo, useState } from "preact/compat";
 import { ErrorTypes } from "../../../../../types";
 import { getAccountIds } from "../../../../../api/accountId";
+import { getAppModeParams } from "../../../../../utils/app-mode";
 
 export const useFetchAccountIds = () => {
+  const { useTenantID } = getAppModeParams();
   const { serverUrl } = useAppState();
 
   const [isLoading, setIsLoading] = useState(false);
@@ -13,13 +15,14 @@ export const useFetchAccountIds = () => {
   const fetchUrl = useMemo(() => getAccountIds(serverUrl), [serverUrl]);
 
   useEffect(() => {
+    if (!useTenantID) return;
     const fetchData = async () => {
       setIsLoading(true);
       try {
         const response = await fetch(fetchUrl);
         const resp = await response.json();
         const data = (resp.data || []) as string[];
-        setAccountIds(data);
+        setAccountIds(data.sort((a, b) => a.localeCompare(b)));
 
         if (response.ok) {
           setError(undefined);

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/style.scss

@@ -2,4 +2,18 @@
 
 .vm-tenant-input {
   position: relative;
+
+  &-list {
+    max-height: 300px;
+    overflow: auto;
+    overscroll-behavior: none;
+    border-radius: $border-radius-medium;
+
+    &__search {
+      position: sticky;
+      top: 0;
+      padding: $padding-small;
+      background-color: $color-background-block;
+    }
+  }
 }

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/Timezones.tsx

@@ -91,6 +91,7 @@ const Timezones: FC<TimezonesProps> = ({ timezoneState, onChange }) => {
         buttonRef={targetRef}
         placement="bottom-left"
         onClose={handleCloseList}
+        fullWidth
       >
         <div className="vm-timezones-list">
           <div className="vm-timezones-list-header">

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/style.scss

@@ -46,7 +46,6 @@
   }
 
   &-list {
-    min-width: 600px;
     max-height: 200px;
     background-color: $color-background-block;
     border-radius: $border-radius-medium;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/style.scss

@@ -1,12 +1,25 @@
 @use "src/styles/variables" as *;
 
 .vm-server-configurator {
-  display: grid;
+  display: flex;
+  flex-direction: column;
   align-items: center;
   gap: $padding-medium;
   width: 600px;
 
+  &_mobile {
+    grid-auto-rows: min-content;
+    align-items: flex-start;
+    height: 100%;
+    width: 100%;
+  }
+
+  @media (max-width: 768px) {
+    width: 100%;
+  }
+
   &__input {
+    width: 100%;
 
     &_server {
       display: grid;
@@ -34,4 +47,10 @@
     margin-left: auto;
     margin-right: 0;
   }
+
+  &_mobile &__footer {
+    align-items: flex-end;
+    flex-grow: 1;
+    width: 100%;
+  }
 }

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/StepConfigurator.tsx

@@ -111,7 +111,12 @@ const StepConfigurator: FC = () => {
           startIcon={<TimelineIcon/>}
           onClick={toggleOpenOptions}
         >
-          STEP {customStep}
+          <p>
+            STEP
+            <p className="vm-step-control__value">
+              {customStep}
+            </p>
+          </p>
         </Button>
       </Tooltip>
       <Popper

app/vmui/packages/vmui/src/components/Configurators/StepConfigurator/style.scss

@@ -8,6 +8,15 @@
     text-transform: none;
   }
 
+  &__value {
+    display: inline;
+    margin-left: 3px;
+
+    @media (max-width: 500px) {
+      display: none;
+    }
+  }
+
   &-popper {
     display: grid;
     gap: $padding-small;

app/vmui/packages/vmui/src/components/Configurators/ThemeControl/ThemeControl.tsx

@@ -3,9 +3,12 @@ import "./style.scss";
 import { useAppDispatch, useAppState } from "../../../state/common/StateContext";
 import { Theme } from "../../../types";
 import Toggle from "../../Main/Toggle/Toggle";
+import useDeviceDetect from "../../../hooks/useDeviceDetect";
+import classNames from "classnames";
 
 const options = Object.values(Theme).map(value => ({ title: value, value }));
 const ThemeControl = () => {
+  const { isMobile } = useDeviceDetect();
   const { theme } = useAppState();
   const dispatch = useAppDispatch();
 
@@ -14,11 +17,19 @@ const ThemeControl = () => {
   };
 
   return (
-    <div className="vm-theme-control">
+    <div
+      className={classNames({
+        "vm-theme-control": true,
+        "vm-theme-control_mobile": isMobile
+      })}
+    >
       <div className="vm-server-configurator__title">
         Theme preferences
       </div>
-      <div className="vm-theme-control__toggle">
+      <div
+        className="vm-theme-control__toggle"
+        key={`${isMobile}`}
+      >
         <Toggle
           options={options}
           value={theme}

app/vmui/packages/vmui/src/components/Configurators/ThemeControl/style.scss

@@ -7,4 +7,8 @@
     min-width: 300px;
     text-transform: capitalize;
   }
+
+  &_mobile &__toggle {
+    display: flex;
+  }
 }

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/ExecutionControls/ExecutionControls.tsx

@@ -7,6 +7,7 @@ import Popper from "../../../Main/Popper/Popper";
 import "./style.scss";
 import classNames from "classnames";
 import Tooltip from "../../../Main/Tooltip/Tooltip";
+import useResize from "../../../../hooks/useResize";
 
 interface AutoRefreshOption {
   seconds: number
@@ -29,6 +30,7 @@ const delayOptions: AutoRefreshOption[] = [
 ];
 
 export const ExecutionControls: FC = () => {
+  const windowSize = useResize(document.body);
 
   const dispatch = useTimeDispatch();
   const appModeEnable = getAppModeEnable();
@@ -83,17 +85,20 @@ export const ExecutionControls: FC = () => {
       <div
         className={classNames({
           "vm-execution-controls-buttons": true,
-          "vm-header-button": !appModeEnable
+          "vm-header-button": !appModeEnable,
+          "vm-execution-controls-buttons_short": windowSize.width <= 360
         })}
       >
-        <Tooltip title="Refresh dashboard">
-          <Button
-            variant="contained"
-            color="primary"
-            onClick={handleUpdate}
-            startIcon={<RefreshIcon/>}
-          />
-        </Tooltip>
+        {windowSize.width > 360 && (
+          <Tooltip title="Refresh dashboard">
+            <Button
+              variant="contained"
+              color="primary"
+              onClick={handleUpdate}
+              startIcon={<RefreshIcon/>}
+            />
+          </Tooltip>
+        )}
         <Tooltip title="Auto-refresh control">
           <div ref={optionsButtonRef}>
             <Button

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/ExecutionControls/style.scss

@@ -9,6 +9,10 @@
     border-radius: calc($button-radius + 1px);
     min-width: 107px;
 
+    &_short {
+      min-width: auto;
+    }
+
     &__arrow {
       display: flex;
       align-items: center;

app/vmui/packages/vmui/src/components/Configurators/TimeRangeSettings/TimeSelector/style.scss

@@ -5,6 +5,11 @@
   grid-template-columns: repeat(2, 230px);
   padding: $padding-global 0;
 
+  @media (max-width: 500px) {
+    grid-template-columns: 1fr;
+    min-width: 250px;
+  }
+
   &-left {
     display: flex;
     flex-direction: column;
@@ -12,6 +17,12 @@
     border-right: $border-divider;
     padding: 0 $padding-global;
 
+    @media (max-width: 500px) {
+      border-right: none;
+      border-bottom: $border-divider;
+      padding-bottom: $padding-global;
+    }
+
     &-inputs {
       flex-grow: 1;
       display: grid;

app/vmui/packages/vmui/src/components/ExploreMetrics/ExploreMetricsHeader/style.scss

@@ -5,16 +5,18 @@
   flex-wrap: wrap;
   align-items: center;
   justify-content: flex-start;
-  gap: $padding-small calc($padding-small + 10px);
+  gap: $padding-global calc($padding-small + 10px);
 
   &__job {
-    flex-grow: 0.5;
-    min-width: 200px;
+    flex-grow: 1;
   }
 
   &__instance {
+    flex-grow: 2;
+  }
+
+  &__size {
     flex-grow: 1;
-    min-width: 300px;
   }
 
   &-metrics {

app/vmui/packages/vmui/src/components/Layout/Footer/style.scss

@@ -2,6 +2,7 @@
 
 .vm-footer {
   display: flex;
+  flex-wrap: wrap;
   align-items: center;
   justify-content: center;
   padding: $padding-medium;
@@ -21,6 +22,11 @@
 
   &__website {
     margin-right: $padding-global;
+
+    @media (max-width: 768px) {
+      margin-right: 0;
+      width: 100%;
+    }
   }
 
   &__link {
@@ -30,5 +36,10 @@
   &__copyright {
     text-align: right;
     flex-grow: 1;
+
+    @media (max-width: 768px) {
+      width: 100%;
+      text-align: center;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Layout/Header/Header.tsx

@@ -17,8 +17,13 @@ import { useAppState } from "../../../state/common/StateContext";
 import HeaderNav from "./HeaderNav/HeaderNav";
 import TenantsConfiguration from "../../Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration";
 import { useFetchAccountIds } from "../../Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds";
+import useResize from "../../../hooks/useResize";
+import SidebarHeader from "./SidebarNav/SidebarHeader";
 
 const Header: FC = () => {
+  const windowSize = useResize(document.body);
+  const displaySidebar = useMemo(() => window.innerWidth < 1000, [windowSize]);
+
   const { isDarkTheme } = useAppState();
   const appModeEnable = getAppModeEnable();
   const { accountIds } = useFetchAccountIds();
@@ -58,27 +63,37 @@ const Header: FC = () => {
     })}
     style={{ background, color }}
   >
-    {!appModeEnable && (
-      <div
-        className="vm-header-logo"
-        onClick={onClickLogo}
-        style={{ color }}
-      >
-        <LogoFullIcon/>
-      </div>
+    {displaySidebar ? (
+      <SidebarHeader
+        background={background}
+        color={color}
+        onClickLogo={onClickLogo}
+      />
+    ) : (
+      <>
+        {!appModeEnable && (
+          <div
+            className="vm-header-logo"
+            onClick={onClickLogo}
+            style={{ color }}
+          >
+            <LogoFullIcon/>
+          </div>
+        )}
+        <HeaderNav
+          color={color}
+          background={background}
+        />
+      </>
     )}
-    <HeaderNav
-      color={color}
-      background={background}
-    />
     <div className="vm-header__settings">
       {headerSetup?.tenant && <TenantsConfiguration accountIds={accountIds}/>}
       {headerSetup?.stepControl && <StepConfigurator/>}
       {headerSetup?.timeSelector && <TimeSelector/>}
       {headerSetup?.cardinalityDatePicker && <CardinalityDatePicker/>}
       {headerSetup?.executionControls && <ExecutionControls/>}
-      <GlobalSettings/>
-      <ShortcutKeys/>
+      {!displaySidebar && <GlobalSettings/>}
+      {!displaySidebar && <ShortcutKeys/>}
     </div>
   </header>;
 };