docs/CHANGELOG.md: document v1.79.9 release

- Return immediately on context cancel during the backoff sleep.
  This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3747

- Add a comment describing why the second attempt to obtain the response from remote side
  is perfromed immediately after the first attempt.

- Remove fasthttp dependency from lib/promscrape/discoveryutils

- Set context deadline before calling doRequestWithPossibleRetry().
  This simplifies the doRequestWithPossibleRetry() a bit.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3293

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.19.5
+          go-version: 1.21.0
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis-js.yml

@@ -13,6 +13,10 @@ on:
   schedule:
     - cron: "30 18 * * 2"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+  
 jobs:
   analyze:
     name: Analyze

.github/workflows/codeql-analysis.yml

@@ -15,6 +15,7 @@ on:
   push:
     branches: [master, cluster]
     paths-ignore:
+      - "docs/**"
       - "**.md"
       - "**.txt"
       - "**.js"
@@ -22,12 +23,17 @@ on:
     # The branches below must be a subset of the branches above
     branches: [master, cluster]
     paths-ignore:
+      - "docs/**"
       - "**.md"
       - "**.txt"
       - "**.js"
   schedule:
     - cron: "30 18 * * 2"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   analyze:
     name: Analyze
@@ -51,7 +57,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.5
+          go-version: 1.20.1
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main-test.yml

@@ -1,66 +0,0 @@
-name: main - test
-on:
-  push:
-    branches:
-      - master
-      - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-  pull_request:
-    branches:
-      - master
-      - cluster
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
-permissions:
-  contents: read
-
-jobs:
-  lint:
-    name: lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19.5
-          check-latest: true
-          cache: true
-
-      - name: Dependencies
-        run: |
-          make install-golangci-lint
-          make check-all
-          git diff --exit-code
-
-  test:
-    needs: lint
-    strategy:
-      matrix:
-        scenario: ["test-full", "test-pure", "test-full-386"]
-    name: test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v3
-
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19.5
-          check-latest: true
-          cache: true
-
-      - name: run tests
-        run: |
-          make ${{ matrix.scenario}}
-
-      - name: Publish coverage
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.txt

.github/workflows/main.yml

@@ -1,16 +1,29 @@
 name: main
 on:
-  workflow_run:
-    workflows: ["main - test"]
-    types:
-      - completed
-
+  push:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches:
+      - master
+      - cluster
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  build:
-    name: Build
+  lint:
+    name: lint
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
@@ -19,10 +32,64 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.5
+          go-version: 1.20.1
           check-latest: true
           cache: true
 
+      - name: Dependencies
+        run: |
+          make install-golangci-lint
+          make check-all
+          git diff --exit-code
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        scenario: ["test-full", "test-pure", "test-full-386"]
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.1
+          check-latest: true
+          cache: true
+
+      - name: run tests
+        run: |
+          make ${{ matrix.scenario}}
+
+      - name: Publish coverage
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.txt
+
+  build:
+    needs: test
+    name: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        id: go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.1
+          check-latest: true
+          cache: true
+
+      - uses: actions/cache@v3
+        with:
+          path: gocache-for-docker
+          key:  gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
+
       - name: Build
         run: |
           make victoria-metrics-crossbuild

.github/workflows/nightly-build.yml

@@ -12,28 +12,37 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Login to Docker Hub
+      - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
-        name: Setup Go
+      - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.19.5
+          go-version: 1.20.1
         id: go
-      -
-        name: Setup docker scan
+
+      - name: Setup docker scan
         run: |
           mkdir -p ~/.docker/cli-plugins && \
           curl https://github.com/docker/scan-cli-plugin/releases/latest/download/docker-scan_linux_amd64 -L -s -S -o ~/.docker/cli-plugins/docker-scan &&\
           chmod +x ~/.docker/cli-plugins/docker-scan
-      -
-        name: Code checkout
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      
+      - name: Code checkout
         uses: actions/checkout@master
-      -
-        name: Publish
+
+      - uses: actions/cache@v3
+        with:
+          path: gocache-for-docker
+          key: gocache-docker-${{ runner.os }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('go.mod') }}
+
+      - name: build & publish
         run: |
+          docker scan --severity=medium --login --token "$SNYK_TOKEN" --accept-license
           LATEST_TAG=nightly PKG_TAG=nightly make publish
+        env: 
+          SNYK_TOKEN: ${{ secrets.SNYK_AUTH_TOKEN }}

Makefile

@@ -144,6 +144,7 @@ vmutils-windows-amd64: \
 	vmctl-windows-amd64
 
 victoria-metrics-crossbuild: \
+	victoria-metrics-linux-386 \
 	victoria-metrics-linux-amd64 \
 	victoria-metrics-linux-arm64 \
 	victoria-metrics-linux-arm \
@@ -155,6 +156,7 @@ victoria-metrics-crossbuild: \
 	victoria-metrics-openbsd-amd64
 
 vmutils-crossbuild: \
+	vmutils-linux-386 \
 	vmutils-linux-amd64 \
 	vmutils-linux-arm64 \
 	vmutils-linux-arm \
@@ -177,6 +179,7 @@ release: \
 	release-vmutils
 
 release-victoria-metrics: \
+	release-victoria-metrics-linux-386 \
 	release-victoria-metrics-linux-amd64 \
 	release-victoria-metrics-linux-arm \
 	release-victoria-metrics-linux-arm64 \
@@ -185,6 +188,10 @@ release-victoria-metrics: \
 	release-victoria-metrics-freebsd-amd64 \
 	release-victoria-metrics-openbsd-amd64
 
+# adds i386 arch
+release-victoria-metrics-linux-386:
+	GOOS=linux GOARCH=386 $(MAKE) release-victoria-metrics-goos-goarch
+
 release-victoria-metrics-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-victoria-metrics-goos-goarch
 
@@ -216,6 +223,7 @@ release-victoria-metrics-goos-goarch: victoria-metrics-$(GOOS)-$(GOARCH)-prod
 	cd bin && rm -rf victoria-metrics-$(GOOS)-$(GOARCH)-prod
 
 release-vmutils: \
+	release-vmutils-linux-386 \
 	release-vmutils-linux-amd64 \
 	release-vmutils-linux-arm64 \
 	release-vmutils-linux-arm \
@@ -225,6 +233,9 @@ release-vmutils: \
 	release-vmutils-openbsd-amd64 \
 	release-vmutils-windows-amd64
 
+release-vmutils-linux-386:
+	GOOS=linux GOARCH=386 $(MAKE) release-vmutils-goos-goarch
+	
 release-vmutils-linux-amd64:
 	GOOS=linux GOARCH=amd64 $(MAKE) release-vmutils-goos-goarch
 
@@ -369,7 +380,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.50.1
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.1
 
 govulncheck: install-govulncheck
 	govulncheck ./...

README.md

@@ -1085,7 +1085,9 @@ The [deduplication](#deduplication) isn't applied for the data exported in nativ
 
 ## How to import time series data
 
-Time series data can be imported into VictoriaMetrics via any supported data ingestion protocol:
+VictoriaMetrics can discover and scrape metrics from Prometheus-compatible targets (aka "pull" protocol) -
+see [these docs](#how-to-scrape-prometheus-exporters-such-as-node-exporter).
+Additionally, VictoriaMetrics can accept metrics via the following popular data ingestion protocols (aka "push" protocols):
 
 * [Prometheus remote_write API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). See [these docs](#prometheus-setup) for details.
 * DataDog `submit metrics` API. See [these docs](#how-to-send-data-from-datadog-agent) for details.
@@ -1614,7 +1616,9 @@ VictoriaMetrics provides the following security-related command-line flags:
 Explicitly set internal network interface for TCP and UDP ports for data ingestion with Graphite and OpenTSDB formats.
 For example, substitute `-graphiteListenAddr=:2003` with `-graphiteListenAddr=<internal_iface_ip>:2003`. This protects from unexpected requests from untrusted network interfaces.
 
-VictoriaMetrics has achieved security certifications for Database Software Development and Software-Based Monitoring Services. We apply strict security measures in everything we do. See our [Security page](https://victoriametrics.com/security/) for more details.
+See also [security recommendation for VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#security)
+and [the general security page at VictoriaMetrics website](https://victoriametrics.com/security/).
+
 
 ## Tuning
 
@@ -2170,7 +2174,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -graphiteListenAddr string
-     TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty
+     TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. See also -graphiteListenAddr.useProxyProtocol
+  -graphiteListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -graphiteListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -graphiteTrimTimestamp duration
      Trim timestamps for Graphite data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -http.connTimeout duration
@@ -2190,7 +2196,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -httpAuth.username string
      Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-     TCP address to listen for http connections (default ":8428")
+     TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8428")
+  -httpListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -import.maxLineLen size
      The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 104857600)
@@ -2203,7 +2211,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -influxDBLabel string
      Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
   -influxListenAddr string
-     TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write
+     TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write . See also -influxListenAddr.useProxyProtocol
+  -influxListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -influxListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -influxMeasurementFieldSeparator string
      Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
@@ -2215,7 +2225,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -inmemoryDataFlushInterval duration
      The interval for guaranteed saving of in-memory data to disk. The saved data survives unclean shutdown such as OOM crash, hardware reset, SIGKILL, etc. Bigger intervals may help increasing lifetime of flash storage with limited write cycles (e.g. Raspberry PI). Smaller intervals increase disk IO load. Minimum supported value is 1s (default 5s)
   -insert.maxQueueDuration duration
-     The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
+     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringMaxLen int
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
   -logNewSeries
      Whether to log new series. This option is for debug purposes only. It can lead to performance issues when big number of new series are ingested into VictoriaMetrics
   -loggerDisableTimestamps
@@ -2251,9 +2263,13 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
+     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
+  -opentsdbHTTPListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbListenAddr string
-     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty
+     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
+  -opentsdbListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbTrimTimestamp duration
      Trim timestamps for OpenTSDB 'telnet put' data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -opentsdbhttp.maxInsertRequestSize size
@@ -2332,12 +2348,12 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -promscrape.minResponseSizeForStreamParse size
      The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000)
+  -promscrape.noStaleMarkers
+     Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.nomad.waitTime duration
      Wait time used by Nomad service discovery. Default value is used if not set
   -promscrape.nomadSDCheckInterval duration
      Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs for details (default 30s)
-  -promscrape.noStaleMarkers
-     Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
      Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#openstack_sd_configs for details (default 30s)
   -promscrape.seriesLimitPerTarget int
@@ -2483,9 +2499,11 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -storageDataPath string
      Path to storage data (default "victoria-metrics-data")
   -streamAggr.config string
-     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html .See also -remoteWrite.streamAggr.keepInput
+     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput and -streamAggr.dedupInterval
+  -streamAggr.dedupInterval duration
+     Input samples are de-duplicated with this interval before being aggregated. Only the last sample per each time series per each interval is aggregated if the interval is greater than zero
   -streamAggr.keepInput
-     Whether to keep input samples after the aggregation with -streamAggr.config .By default the input is dropped after the aggregation, so only the aggregate data is stored. See https://docs.victoriametrics.com/stream-aggregation.html
+     Whether to keep input samples after the aggregation with -streamAggr.config. By default the input is dropped after the aggregation, so only the aggregate data is stored. See https://docs.victoriametrics.com/stream-aggregation.html
   -tls
      Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string

app/victoria-metrics/main.go

@@ -24,7 +24,9 @@ import (
 )
 
 var (
-	httpListenAddr    = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections")
+	httpListenAddr   = flag.String("httpListenAddr", ":8428", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	minScrapeInterval = flag.Duration("dedup.minScrapeInterval", 0, "Leave only the last sample in every time series per each discrete interval "+
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
@@ -64,7 +66,7 @@ func main() {
 	vminsert.Init()
 	startSelfScraper()
 
-	go httpserver.Serve(*httpListenAddr, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started VictoriaMetrics in %.3f seconds", time.Since(startTime).Seconds())
 
 	sig := procutil.WaitForSigterm()

app/victoria-metrics/main_test.go

@@ -129,10 +129,10 @@ func setUp() {
 	storagePath = filepath.Join(os.TempDir(), testStorageSuffix)
 	processFlags()
 	logger.Init()
-	vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
+	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
-	go httpserver.Serve(*httpListenAddr, requestHandler)
+	go httpserver.Serve(*httpListenAddr, false, requestHandler)
 	readyStorageCheckFunc := func() bool {
 		resp, err := http.Get(testHealthHTTPPath)
 		if err != nil {
@@ -189,10 +189,8 @@ func tearDown() {
 
 func TestWriteRead(t *testing.T) {
 	t.Run("write", testWrite)
+	vmstorage.Storage.DebugFlush()
 	time.Sleep(1 * time.Second)
-	vmstorage.Stop()
-	// open storage after stop in write
-	vmstorage.InitWithoutMetrics(promql.ResetRollupResultCacheIfNeeded)
 	t.Run("read", testRead)
 }
 

app/victoria-metrics/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmagent/README.md

@@ -324,7 +324,7 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
 ## Automatically generated metrics
 
 `vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)
-and attaches target-specific `instance` and `job` labels to these metrics:
+and attaches `instance`, `job` and other target-specific labels to these metrics:
 
 * `up` - this metric exposes `1` value on successful scrape and `0` value on unsuccessful scrape. This allows monitoring
   failing scrapes with the following [MetricsQL query](https://docs.victoriametrics.com/MetricsQL.html):
@@ -763,7 +763,8 @@ scrape_configs:
 
 ## Cardinality limiter
 
-By default `vmagent` doesn't limit the number of time series each scrape target can expose. The limit can be enforced in the following places:
+By default `vmagent` doesn't limit the number of time series each scrape target can expose. 
+The limit can be enforced in the following places:
 
 * Via `-promscrape.seriesLimitPerTarget` command-line option. This limit is applied individually
   to all the scrape targets defined in the file pointed by `-promscrape.config`.
@@ -774,10 +775,7 @@ By default `vmagent` doesn't limit the number of time series each scrape target
   via [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) for targets,
   which may expose too high number of time series.
 
-See also `sample_limit` option at [scrape_config section](https://docs.victoriametrics.com/sd_configs.html#scrape_configs).
-
-Scraped metrics are dropped for time series exceeding the given limit.
-
+Scraped metrics are dropped for time series exceeding the given limit on the time window of 24h.
 `vmagent` creates the following additional per-target metrics for targets with non-zero series limit:
 
 - `scrape_series_limit_samples_dropped` - the number of dropped samples during the scrape when the unique series limit is exceeded.
@@ -791,6 +789,7 @@ These metrics allow building the following alerting rules:
 - `scrape_series_current / scrape_series_limit > 0.9` - alerts when the number of series exposed by the target reaches 90% of the limit.
 - `sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0` - alerts when some samples are dropped because the series limit on a particular target is reached.
 
+See also `sample_limit` option at [scrape_config section](https://docs.victoriametrics.com/sd_configs.html#scrape_configs).
 
 By default `vmagent` doesn't limit the number of time series written to remote storage systems specified at `-remoteWrite.url`.
 The limit can be enforced by setting the following command-line flags:
@@ -1162,7 +1161,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -fs.disableMmap
      Whether to use pread() instead of mmap() for reading data files. By default mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
   -graphiteListenAddr string
-     TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty
+     TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. See also -graphiteListenAddr.useProxyProtocol
+  -graphiteListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -graphiteListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -graphiteTrimTimestamp duration
      Trim timestamps for Graphite data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -http.connTimeout duration
@@ -1182,7 +1183,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -httpAuth.username string
      Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-     TCP address to listen for http connections. Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. Note that /targets and /metrics pages aren't available if -httpListenAddr='' (default ":8429")
+     TCP address to listen for http connections. Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol (default ":8429")
+  -httpListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -import.maxLineLen size
      The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with 'max_rows_per_line' query arg passed to /api/v1/export
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 104857600)
@@ -1195,7 +1198,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -influxDBLabel string
      Default label for the DB name sent over '?db={db_name}' query parameter (default "db")
   -influxListenAddr string
-     TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write
+     TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . See also -influxListenAddr.useProxyProtocol
+  -influxListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -influxListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -influxMeasurementFieldSeparator string
      Separator for '{measurement}{separator}{field_name}' metric name when inserted via InfluxDB line protocol (default "_")
   -influxSkipMeasurement
@@ -1205,7 +1210,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -influxTrimTimestamp duration
      Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms)
   -insert.maxQueueDuration duration
-     The maximum duration for waiting in the queue for insert requests due to -maxConcurrentInserts (default 1m0s)
+     The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
+  -internStringMaxLen int
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
   -kafka.consumer.topic array
      Kafka topic names for data consumption. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
@@ -1261,9 +1268,13 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -opentsdbHTTPListenAddr string
-     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty
+     TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbHTTPListenAddr.useProxyProtocol
+  -opentsdbHTTPListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbListenAddr string
-     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty
+     TCP and UDP address to listen for OpentTSDB metrics. Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol
+  -opentsdbListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -opentsdbTrimTimestamp duration
      Trim timestamps for OpenTSDB 'telnet put' data to this duration. Minimum practical duration is 1s. Higher duration (i.e. 1m) may be used for reducing disk space usage for timestamp data (default 1s)
   -opentsdbhttp.maxInsertRequestSize size
@@ -1340,12 +1351,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -promscrape.minResponseSizeForStreamParse size
      The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000)
+  -promscrape.noStaleMarkers
+     Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.nomad.waitTime duration
      Wait time used by Nomad service discovery. Default value is used if not set
   -promscrape.nomadSDCheckInterval duration
      Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs for details (default 30s)
-  -promscrape.noStaleMarkers
-     Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
   -promscrape.openstackSDCheckInterval duration
      Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#openstack_sd_configs for details (default 30s)
   -promscrape.seriesLimitPerTarget int
@@ -1468,8 +1479,11 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      The number of significant figures to leave in metric values before writing them to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits
      Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.streamAggr.config array
-     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput
+     Optional path to file with stream aggregation config. See https://docs.victoriametrics.com/stream-aggregation.html . See also -remoteWrite.streamAggr.keepInput and -remoteWrite.streamAggr.dedupInterval
      Supports an array of values separated by comma or specified via multiple flags.
+  -remoteWrite.streamAggr.dedupInterval array
+     Input samples are de-duplicated with this interval before being aggregated. Only the last sample per each time series per each interval is aggregated if the interval is greater than zero
+     Supports array of values separated by comma or specified via multiple flags.
   -remoteWrite.streamAggr.keepInput array
      Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. By default the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. See https://docs.victoriametrics.com/stream-aggregation.html
      Supports array of values separated by comma or specified via multiple flags.

app/vmagent/main.go

@@ -44,16 +44,29 @@ import (
 var (
 	httpListenAddr = flag.String("httpListenAddr", ":8429", "TCP address to listen for http connections. "+
 		"Set this flag to empty value in order to disable listening on any port. This mode may be useful for running multiple vmagent instances on the same server. "+
-		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''")
+		"Note that /targets and /metrics pages aren't available if -httpListenAddr=''. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
-		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write")
-	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
+		"This flag isn't needed when ingesting data over HTTP - just send it to http://<vmagent>:8429/write . "+
+		"See also -influxListenAddr.useProxyProtocol")
+	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. "+
+		"See also -graphiteListenAddr.useProxyProtocol")
+	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
-		"Usually :4242 must be set. Doesn't work if empty")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
-	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
-	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
+		"Usually :4242 must be set. Doesn't work if empty. See also -opentsdbListenAddr.useProxyProtocol")
+	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
+	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
+		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	configAuthKey = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
+	dryRun        = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed by passing -promscrape.config.strictParse=false command-line flag")
 )
@@ -104,26 +117,26 @@ func main() {
 	remotewrite.Init()
 	common.StartUnmarshalWorkers()
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, func(r io.Reader) error {
+		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, func(r io.Reader) error {
 			return influx.InsertHandlerForReader(r, false)
 		})
 	}
 	if len(*graphiteListenAddr) > 0 {
-		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
 	if len(*opentsdbListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, httpInsertHandler)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, httpInsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
 		httpInsertHandler := getOpenTSDBHTTPInsertHandler()
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, httpInsertHandler)
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, httpInsertHandler)
 	}
 
 	promscrape.Init(remotewrite.Push)
 
 	if len(*httpListenAddr) > 0 {
-		go httpserver.Serve(*httpListenAddr, requestHandler)
+		go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	}
 	logger.Infof("started vmagent in %.3f seconds", time.Since(startTime).Seconds())
 
@@ -223,7 +236,14 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
-		w.WriteHeader(http.StatusNoContent)
+		statusCode := http.StatusNoContent
+		if strings.HasPrefix(path, "/prometheus/api/v1/import/prometheus/metrics/job/") ||
+			strings.HasPrefix(path, "/api/v1/import/prometheus/metrics/job/") {
+			// Return 200 status code for pushgateway requests.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3636
+			statusCode = http.StatusOK
+		}
+		w.WriteHeader(statusCode)
 		return true
 	}
 	if strings.HasPrefix(path, "datadog/") {

app/vmagent/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmagent/prometheusimport/request_handler.go

@@ -1,12 +1,12 @@
 package prometheusimport
 
 import (
-	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -33,14 +33,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
-	}, nil)
-}
-
-// InsertHandlerForReader processes metrics from given reader with optional gzip format
-func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return parser.ParseStream(r, 0, isGzipped, func(rows []parser.Row) error {
-		return insertRows(nil, rows, nil)
-	}, nil)
+	}, func(s string) {
+		httpserver.LogError(req, s)
+	})
 }
 
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {

app/vmagent/prometheusimport/request_handler_test.go

@@ -0,0 +1,60 @@
+package prometheusimport
+
+import (
+	"bytes"
+	"flag"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
+)
+
+var (
+	srv        *httptest.Server
+	testOutput *bytes.Buffer
+)
+
+func TestInsertHandler(t *testing.T) {
+	setUp()
+	defer tearDown()
+	req := httptest.NewRequest("POST", "/insert/0/api/v1/import/prometheus", bytes.NewBufferString(`{"foo":"bar"}
+go_memstats_alloc_bytes_total 1`))
+	if err := InsertHandler(nil, req); err != nil {
+		t.Errorf("unxepected error %s", err)
+	}
+	expectedMsg := "cannot unmarshal Prometheus line"
+	if !strings.Contains(testOutput.String(), expectedMsg) {
+		t.Errorf("output %q should contain %q", testOutput.String(), expectedMsg)
+	}
+}
+
+func setUp() {
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(204)
+	}))
+	flag.Parse()
+	remoteWriteFlag := "remoteWrite.url"
+	if err := flag.Lookup(remoteWriteFlag).Value.Set(srv.URL); err != nil {
+		log.Fatalf("unable to set %q with value %q, err: %v", remoteWriteFlag, srv.URL, err)
+	}
+	logger.Init()
+	common.StartUnmarshalWorkers()
+	remotewrite.Init()
+	testOutput = &bytes.Buffer{}
+	logger.SetOutputForTests(testOutput)
+}
+
+func tearDown() {
+	common.StopUnmarshalWorkers()
+	srv.Close()
+	logger.ResetOutputForTest()
+	tmpDataDir := flag.Lookup("remoteWrite.tmpDataPath").Value.String()
+	fs.MustRemoveAll(tmpDataDir)
+
+}

app/vmagent/promremotewrite/request_handler.go

@@ -1,7 +1,6 @@
 package promremotewrite
 
 import (
-	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -33,13 +32,6 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
-// InsertHandlerForReader processes metrics from given reader
-func InsertHandlerForReader(at *auth.Token, r io.Reader) error {
-	return parser.ParseStream(r, func(tss []prompb.TimeSeries) error {
-		return insertRows(at, tss, nil)
-	})
-}
-
 func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmagent/remotewrite/remotewrite.go

@@ -62,10 +62,12 @@ var (
 
 	streamAggrConfig = flagutil.NewArrayString("remoteWrite.streamAggr.config", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
-		"See also -remoteWrite.streamAggr.keepInput")
+		"See also -remoteWrite.streamAggr.keepInput and -remoteWrite.streamAggr.dedupInterval")
 	streamAggrKeepInput = flagutil.NewArrayBool("remoteWrite.streamAggr.keepInput", "Whether to keep input samples after the aggregation with -remoteWrite.streamAggr.config. "+
 		"By default the input is dropped after the aggregation, so only the aggregate data is sent to the -remoteWrite.url. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html")
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", "Input samples are de-duplicated with this interval before being aggregated. "+
+		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
 
 var (
@@ -509,7 +511,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	// Initialize sas
 	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
 	if sasFile != "" {
-		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(argIdx, 0)
+		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 		if err != nil {
 			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggrFile=%q: %s", sasFile, err)
 		}

app/vmagent/vmimport/request_handler.go

@@ -1,7 +1,6 @@
 package vmimport
 
 import (
-	"io"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -36,13 +35,6 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 	})
 }
 
-// InsertHandlerForReader processes metrics from given reader
-func InsertHandlerForReader(r io.Reader, isGzipped bool) error {
-	return parser.ParseStream(r, isGzipped, func(rows []parser.Row) error {
-		return insertRows(nil, rows, nil)
-	})
-}
-
 func insertRows(at *auth.Token, rows []parser.Row, extraLabels []prompbmarshal.Label) error {
 	ctx := common.GetPushCtx()
 	defer common.PutPushCtx(ctx)

app/vmalert/README.md

@@ -9,6 +9,13 @@ protocol and require `-remoteWrite.url` to be configured.
 Vmalert is heavily inspired by [Prometheus](https://prometheus.io/docs/alerting/latest/overview/)
 implementation and aims to be compatible with its syntax.
 
+A [single-node](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#vmalert)
+or [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#vmalert)
+of VictoriaMetrics are capable of proxying requests to vmalert via `-vmalert.proxyURL` command-line flag. 
+Use this feature for the following cases:
+* for proxying requests from [Grafana Alerting UI](https://grafana.com/docs/grafana/latest/alerting/);
+* for accessing vmalert's UI through VictoriaMetrics Web interface.
+
 ## Features
 
 * Integration with [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) TSDB;
@@ -897,7 +904,9 @@ The shortlist of configuration flags is the following:
   -httpAuth.username string
      Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-     Address to listen for http connections (default ":8880")
+     Address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8880")
+  -httpListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -914,11 +923,6 @@ The shortlist of configuration flags is the following:
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-  -memory.allowedBytes size
-     Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
-     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
-  -memory.allowedPercent float
-     Allowed percent of system memory VictoriaMetrics caches may occupy. See also -memory.allowedBytes. Too low a value may increase cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache which will result in higher disk IO usage (default 60)
   -metricsAuthKey string
      Auth key for /metrics endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -notifier.basicAuth.password array
@@ -1008,7 +1012,7 @@ The shortlist of configuration flags is the following:
   -remoteRead.headers string
      Optional HTTP headers to send with each request to the corresponding -remoteRead.url. For example, -remoteRead.headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding -remoteRead.url. Multiple headers must be delimited by '^^': -remoteRead.headers='header1:value1^^header2:value2'
   -remoteRead.ignoreRestoreErrors
-     Whether to ignore errors from remote storage when restoring alerts state on startup. (default true)
+     Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases. (default true)
   -remoteRead.lookback duration
      Lookback defines how far to look into past for alerts timeseries. For example, if lookback=1h then range from now() to now()-1h will be scanned. (default 1h0m0s)
   -remoteRead.oauth2.clientID string
@@ -1086,7 +1090,7 @@ The shortlist of configuration flags is the following:
   -replay.disableProgressBar
      Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
   -replay.maxDatapointsPerQuery int
-     Max number of data points expected in one request. The higher the value, the less requests will be made during replay. (default 1000)
+     Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay. (default 1000)
   -replay.ruleRetryAttempts int
      Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
   -replay.rulesDelay duration

app/vmalert/alerting.go

@@ -421,7 +421,9 @@ func (ar *AlertingRule) UpdateWith(r Rule) error {
 	ar.Labels = nr.Labels
 	ar.Annotations = nr.Annotations
 	ar.EvalInterval = nr.EvalInterval
+	ar.Debug = nr.Debug
 	ar.q = nr.q
+	ar.state = nr.state
 	return nil
 }
 
@@ -498,6 +500,7 @@ func (ar *AlertingRule) ToAPI() APIRule {
 		LastSamples:    lastState.samples,
 		MaxUpdates:     ar.state.size(),
 		Updates:        ar.state.getAll(),
+		Debug:          ar.Debug,
 
 		// encode as strings to avoid rounding in JSON
 		ID:      fmt.Sprintf("%d", ar.ID()),
@@ -604,54 +607,59 @@ func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.Time
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
-// Restore restores the state of active alerts basing on previously written time series.
-// Restore restores only ActiveAt field. Field State will be always Pending and supposed
-// to be updated on next Exec, as well as Value field.
-// Only rules with For > 0 will be restored.
-func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, lookback time.Duration, labels map[string]string) error {
-	if q == nil {
-		return fmt.Errorf("querier is nil")
+// Restore restores the value of ActiveAt field for active alerts,
+// based on previously written time series `alertForStateMetricName`.
+// Only rules with For > 0 can be restored.
+func (ar *AlertingRule) Restore(ctx context.Context, q datasource.Querier, ts time.Time, lookback time.Duration) error {
+	if ar.For < 1 {
+		return nil
 	}
 
-	ts := time.Now()
-	qFn := func(query string) ([]datasource.Metric, error) {
-		res, _, err := ar.q.Query(ctx, query, ts)
-		return res, err
+	ar.alertsMu.Lock()
+	defer ar.alertsMu.Unlock()
+
+	if len(ar.alerts) < 1 {
+		return nil
 	}
 
-	// account for external labels in filter
-	var labelsFilter string
-	for k, v := range labels {
-		labelsFilter += fmt.Sprintf(",%s=%q", k, v)
-	}
-
-	expr := fmt.Sprintf("last_over_time(%s{alertname=%q%s}[%ds])",
-		alertForStateMetricName, ar.Name, labelsFilter, int(lookback.Seconds()))
-	qMetrics, _, err := q.Query(ctx, expr, ts)
-	if err != nil {
-		return err
-	}
-
-	for _, m := range qMetrics {
-		ls := &labelSet{
-			origin:    make(map[string]string, len(m.Labels)),
-			processed: make(map[string]string, len(m.Labels)),
+	for _, a := range ar.alerts {
+		if a.Restored || a.State != notifier.StatePending {
+			continue
 		}
-		for _, l := range m.Labels {
-			if l.Name == "__name__" {
-				continue
-			}
-			ls.origin[l.Name] = l.Value
-			ls.processed[l.Name] = l.Value
+
+		var labelsFilter []string
+		for k, v := range a.Labels {
+			labelsFilter = append(labelsFilter, fmt.Sprintf("%s=%q", k, v))
 		}
-		a, err := ar.newAlert(m, ls, time.Unix(int64(m.Values[0]), 0), qFn)
+		sort.Strings(labelsFilter)
+		expr := fmt.Sprintf("last_over_time(%s{%s}[%ds])",
+			alertForStateMetricName, strings.Join(labelsFilter, ","), int(lookback.Seconds()))
+
+		ar.logDebugf(ts, nil, "restoring alert state via query %q", expr)
+
+		qMetrics, _, err := q.Query(ctx, expr, ts)
 		if err != nil {
-			return fmt.Errorf("failed to create alert: %w", err)
+			return err
 		}
-		a.ID = hash(ls.processed)
-		a.State = notifier.StatePending
+
+		if len(qMetrics) < 1 {
+			ar.logDebugf(ts, nil, "no response was received from restore query")
+			continue
+		}
+
+		// only one series expected in response
+		m := qMetrics[0]
+		// __name__ supposed to be alertForStateMetricName
+		m.DelLabel("__name__")
+
+		// we assume that restore query contains all label matchers,
+		// so all received labels will match anyway if their number is equal.
+		if len(m.Labels) != len(a.Labels) {
+			ar.logDebugf(ts, nil, "state restore query returned not expected label-set %v", m.Labels)
+			continue
+		}
+		a.ActiveAt = time.Unix(int64(m.Values[0]), 0)
 		a.Restored = true
-		ar.alerts[a.ID] = a
 		logger.Infof("alert %q (%d) restored to state at %v", a.Name, a.ID, a.ActiveAt)
 	}
 	return nil

app/vmalert/alerting_test.go

@@ -6,12 +6,15 @@ import (
 	"reflect"
 	"sort"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestAlertingRule_ToTimeSeries(t *testing.T) {
@@ -502,118 +505,156 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 	}
 }
 
-func TestAlertingRule_Restore(t *testing.T) {
-	testCases := []struct {
-		rule      *AlertingRule
-		metrics   []datasource.Metric
-		expAlerts map[uint64]*notifier.Alert
-	}{
-		{
-			newTestRuleWithLabels("no extra labels"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(nil): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-			},
-		},
-		{
-			newTestRuleWithLabels("metric labels"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					alertNameLabel, "metric labels",
-					alertGroupNameLabel, "groupID",
-					"foo", "bar",
-					"namespace", "baz",
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{
-					alertNameLabel:      "metric labels",
-					alertGroupNameLabel: "groupID",
-					"foo":               "bar",
-					"namespace":         "baz",
-				}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-			},
-		},
-		{
-			newTestRuleWithLabels("rule labels", "source", "vm"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"foo", "bar",
-					"namespace", "baz",
-					// extra labels set by rule
-					"source", "vm",
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{
-					"foo":       "bar",
-					"namespace": "baz",
-					"source":    "vm",
-				}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-			},
-		},
-		{
-			newTestRuleWithLabels("multiple alerts"),
-			[]datasource.Metric{
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"host", "localhost-1",
-				),
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(2*time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"host", "localhost-2",
-				),
-				metricWithValueAndLabels(t, float64(time.Now().Truncate(3*time.Hour).Unix()),
-					"__name__", alertForStateMetricName,
-					"host", "localhost-3",
-				),
-			},
-			map[uint64]*notifier.Alert{
-				hash(map[string]string{"host": "localhost-1"}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(time.Hour)},
-				hash(map[string]string{"host": "localhost-2"}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(2 * time.Hour)},
-				hash(map[string]string{"host": "localhost-3"}): {State: notifier.StatePending,
-					ActiveAt: time.Now().Truncate(3 * time.Hour)},
-			},
-		},
+func TestGroup_Restore(t *testing.T) {
+	defaultTS := time.Now()
+	fqr := &fakeQuerierWithRegistry{}
+	fn := func(rules []config.Rule, expAlerts map[uint64]*notifier.Alert) {
+		t.Helper()
+		defer fqr.reset()
+
+		for _, r := range rules {
+			fqr.set(r.Expr, metricWithValueAndLabels(t, 0, "__name__", r.Alert))
+		}
+
+		fg := newGroup(config.Group{Name: "TestRestore", Rules: rules}, fqr, time.Second, nil)
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			nts := func() []notifier.Notifier { return []notifier.Notifier{&fakeNotifier{}} }
+			fg.start(context.Background(), nts, nil, fqr)
+			wg.Done()
+		}()
+		fg.close()
+		wg.Wait()
+
+		gotAlerts := make(map[uint64]*notifier.Alert)
+		for _, rs := range fg.Rules {
+			alerts := rs.(*AlertingRule).alerts
+			for k, v := range alerts {
+				if !v.Restored {
+					// set not restored alerts to predictable timestamp
+					v.ActiveAt = defaultTS
+				}
+				gotAlerts[k] = v
+			}
+		}
+
+		if len(gotAlerts) != len(expAlerts) {
+			t.Fatalf("expected %d alerts; got %d", len(expAlerts), len(gotAlerts))
+		}
+		for key, exp := range expAlerts {
+			got, ok := gotAlerts[key]
+			if !ok {
+				t.Fatalf("expected to have key %d", key)
+			}
+			if got.State != notifier.StatePending {
+				t.Fatalf("expected state %d; got %d", notifier.StatePending, got.State)
+			}
+			if got.ActiveAt != exp.ActiveAt {
+				t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
+			}
+		}
 	}
-	fakeGroup := Group{Name: "TestRule_Exec"}
-	for _, tc := range testCases {
-		t.Run(tc.rule.Name, func(t *testing.T) {
-			fq := &fakeQuerier{}
-			tc.rule.GroupID = fakeGroup.ID()
-			tc.rule.q = fq
-			fq.add(tc.metrics...)
-			if err := tc.rule.Restore(context.TODO(), fq, time.Hour, nil); err != nil {
-				t.Fatalf("unexpected err: %s", err)
-			}
-			if len(tc.rule.alerts) != len(tc.expAlerts) {
-				t.Fatalf("expected %d alerts; got %d", len(tc.expAlerts), len(tc.rule.alerts))
-			}
-			for key, exp := range tc.expAlerts {
-				got, ok := tc.rule.alerts[key]
-				if !ok {
-					t.Fatalf("expected to have key %d", key)
-				}
-				if got.State != exp.State {
-					t.Fatalf("expected state %d; got %d", exp.State, got.State)
-				}
-				if got.ActiveAt != exp.ActiveAt {
-					t.Fatalf("expected ActiveAt %v; got %v", exp.ActiveAt, got.ActiveAt)
-				}
-			}
+
+	stateMetric := func(name string, value time.Time, labels ...string) datasource.Metric {
+		labels = append(labels, "__name__", alertForStateMetricName)
+		labels = append(labels, alertNameLabel, name)
+		labels = append(labels, alertGroupNameLabel, "TestRestore")
+		return metricWithValueAndLabels(t, float64(value.Unix()), labels...)
+	}
+
+	// one active alert, no previous state
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+	fqr.reset()
+
+	// one active alert with state restore
+	ts := time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+		stateMetric("foo", ts))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// two rules, two active alerts, one with state restored
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("foo", ts))
+	fn(
+		[]config.Rule{
+			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
+			{Alert: "bar", Expr: "bar", For: promutils.NewDuration(time.Second)},
+		},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// two rules, two active alerts, two with state restored
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo"}[3600s])`,
+		stateMetric("foo", ts))
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="bar"}[3600s])`,
+		stateMetric("bar", ts))
+	fn(
+		[]config.Rule{
+			{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)},
+			{Alert: "bar", Expr: "bar", For: promutils.NewDuration(time.Second)},
+		},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts,
+			},
+			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: ts},
+		})
+
+	// one active alert but wrong state restore
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertname="bar",alertgroup="TestRestore"}[3600s])`,
+		stateMetric("wrong alert", ts))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
+				ActiveAt: defaultTS,
+			},
+		})
+
+	// one active alert with labels
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+		stateMetric("foo", ts, "env", "dev"))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
+				ActiveAt: ts,
+			},
+		})
+
+	// one active alert with restore labels missmatch
+	ts = time.Now().Truncate(time.Hour)
+	fqr.set(`last_over_time(ALERTS_FOR_STATE{alertgroup="TestRestore",alertname="foo",env="dev"}[3600s])`,
+		stateMetric("foo", ts, "env", "dev", "team", "foo"))
+	fn(
+		[]config.Rule{{Alert: "foo", Expr: "foo", Labels: map[string]string{"env": "dev"}, For: promutils.NewDuration(time.Second)}},
+		map[uint64]*notifier.Alert{
+			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore", "env": "dev"}): {
+				ActiveAt: defaultTS,
+			},
 		})
-	}
 }
 
 func TestAlertingRule_Exec_Negative(t *testing.T) {

app/vmalert/datasource/datasource.go

@@ -72,6 +72,15 @@ func (m *Metric) AddLabel(key, value string) {
 	m.Labels = append(m.Labels, Label{Name: key, Value: value})
 }
 
+// DelLabel deletes the given label from the label set
+func (m *Metric) DelLabel(key string) {
+	for i, l := range m.Labels {
+		if l.Name == key {
+			m.Labels = append(m.Labels[:i], m.Labels[i+1:]...)
+		}
+	}
+}
+
 // Label returns the given label value.
 // If label is missing empty string will be returned
 func (m *Metric) Label(key string) string {

app/vmalert/group.go

@@ -158,23 +158,23 @@ func (g *Group) ID() uint64 {
 }
 
 // Restore restores alerts state for group rules
-func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookback time.Duration, labels map[string]string) error {
-	labels = mergeLabels(g.Name, "", labels, g.Labels)
+func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, ts time.Time, lookback time.Duration) error {
 	for _, rule := range g.Rules {
-		rr, ok := rule.(*AlertingRule)
+		ar, ok := rule.(*AlertingRule)
 		if !ok {
 			continue
 		}
-		if rr.For < 1 {
+		if ar.For < 1 {
 			continue
 		}
-		// ignore QueryParams on purpose, because they could contain
-		// query filters. This may affect the restore procedure.
 		q := qb.BuildWithParams(datasource.QuerierParams{
-			DataSourceType: g.Type.String(),
-			Headers:        g.Headers,
+			DataSourceType:     g.Type.String(),
+			EvaluationInterval: g.Interval,
+			QueryParams:        g.Params,
+			Headers:            g.Headers,
+			Debug:              ar.Debug,
 		})
-		if err := rr.Restore(ctx, q, lookback, labels); err != nil {
+		if err := ar.Restore(ctx, q, ts, lookback); err != nil {
 			return fmt.Errorf("error while restoring rule %q: %w", rule, err)
 		}
 	}
@@ -251,7 +251,7 @@ func (g *Group) close() {
 
 var skipRandSleepOnGroupStart bool
 
-func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client) {
+func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
 	e := &executor{
@@ -259,26 +259,6 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		notifiers:                nts,
 		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label)}
 
-	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
-	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
-		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
-		if randSleep < sleepOffset {
-			randSleep += uint64(g.Interval)
-		}
-		randSleep -= sleepOffset
-		sleepTimer := time.NewTimer(time.Duration(randSleep))
-		select {
-		case <-ctx.Done():
-			sleepTimer.Stop()
-			return
-		case <-g.doneCh:
-			sleepTimer.Stop()
-			return
-		case <-sleepTimer.C:
-		}
-	}
-
 	evalTS := time.Now()
 
 	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
@@ -309,6 +289,16 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 
 	t := time.NewTicker(g.Interval)
 	defer t.Stop()
+
+	// restore the rules state after the first evaluation
+	// so only active alerts can be restored.
+	if rr != nil {
+		err := g.Restore(ctx, rr, evalTS, *remoteReadLookBack)
+		if err != nil {
+			logger.Errorf("error while restoring ruleState for group %q: %s", g.Name, err)
+		}
+	}
+
 	for {
 		select {
 		case <-ctx.Done():

app/vmalert/group_test.go

@@ -209,7 +209,7 @@ func TestGroupStart(t *testing.T) {
 	fs.add(m1)
 	fs.add(m2)
 	go func() {
-		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil)
+		g.start(context.Background(), func() []notifier.Notifier { return []notifier.Notifier{fn} }, nil, fs)
 		close(finished)
 	}()
 

app/vmalert/helpers_test.go

@@ -61,6 +61,49 @@ func (fq *fakeQuerier) Query(_ context.Context, _ string, _ time.Time) ([]dataso
 	return cp, req, nil
 }
 
+type fakeQuerierWithRegistry struct {
+	sync.Mutex
+	registry map[string][]datasource.Metric
+}
+
+func (fqr *fakeQuerierWithRegistry) set(key string, metrics ...datasource.Metric) {
+	fqr.Lock()
+	if fqr.registry == nil {
+		fqr.registry = make(map[string][]datasource.Metric)
+	}
+	fqr.registry[key] = metrics
+	fqr.Unlock()
+}
+
+func (fqr *fakeQuerierWithRegistry) reset() {
+	fqr.Lock()
+	fqr.registry = nil
+	fqr.Unlock()
+}
+
+func (fqr *fakeQuerierWithRegistry) BuildWithParams(_ datasource.QuerierParams) datasource.Querier {
+	return fqr
+}
+
+func (fqr *fakeQuerierWithRegistry) QueryRange(ctx context.Context, q string, _, _ time.Time) ([]datasource.Metric, error) {
+	req, _, err := fqr.Query(ctx, q, time.Now())
+	return req, err
+}
+
+func (fqr *fakeQuerierWithRegistry) Query(_ context.Context, expr string, _ time.Time) ([]datasource.Metric, *http.Request, error) {
+	fqr.Lock()
+	defer fqr.Unlock()
+
+	req, _ := http.NewRequest(http.MethodPost, "foo.com", nil)
+	metrics, ok := fqr.registry[expr]
+	if !ok {
+		return nil, req, nil
+	}
+	cp := make([]datasource.Metric, len(metrics))
+	copy(cp, metrics)
+	return cp, req, nil
+}
+
 type fakeNotifier struct {
 	sync.Mutex
 	alerts []notifier.Alert

app/vmalert/main.go

@@ -49,7 +49,9 @@ absolute path to all .tpl files in root.`)
 	configCheckInterval = flag.Duration("configCheckInterval", 0, "Interval for checking for changes in '-rule' or '-notifier.config' files. "+
 		"By default the checking is disabled. Send SIGHUP signal in order to force config check for changes.")
 
-	httpListenAddr     = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+	httpListenAddr   = flag.String("httpListenAddr", ":8880", "Address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	evaluationInterval = flag.Duration("evaluationInterval", time.Minute, "How often to evaluate the rules")
 
 	validateTemplates   = flag.Bool("rule.validateTemplates", true, "Whether to validate annotation and label templates")
@@ -71,7 +73,7 @@ absolute path to all .tpl files in root.`)
 
 	remoteReadLookBack = flag.Duration("remoteRead.lookback", time.Hour, "Lookback defines how far to look into past for alerts timeseries."+
 		" For example, if lookback=1h then range from now() to now()-1h will be scanned.")
-	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup.")
+	remoteReadIgnoreRestoreErrors = flag.Bool("remoteRead.ignoreRestoreErrors", true, "Whether to ignore errors from remote storage when restoring alerts state on startup. DEPRECATED - this flag has no effect and will be removed in the next releases.")
 
 	disableAlertGroupLabel = flag.Bool("disableAlertgroupLabel", false, "Whether to disable adding group's Name as label to generated alerts and time series.")
 
@@ -92,6 +94,10 @@ func main() {
 	logger.Init()
 	pushmetrics.Init()
 
+	if !*remoteReadIgnoreRestoreErrors {
+		logger.Warnf("flag `remoteRead.ignoreRestoreErrors` is deprecated and will be removed in next releases.")
+	}
+
 	err := templates.Load(*ruleTemplatesPath, true)
 	if err != nil {
 		logger.Fatalf("failed to parse %q: %s", *ruleTemplatesPath, err)
@@ -170,7 +176,7 @@ func main() {
 	go configReload(ctx, manager, groupsCfg, sighupCh)
 
 	rh := &requestHandler{m: manager}
-	go httpserver.Serve(*httpListenAddr, rh.handler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, rh.handler)
 
 	sig := procutil.WaitForSigterm()
 	logger.Infof("service received signal %s", sig)

app/vmalert/manager.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"sort"
 	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
@@ -82,24 +83,38 @@ func (m *manager) close() {
 	m.wg.Wait()
 }
 
-func (m *manager) startGroup(ctx context.Context, group *Group, restore bool) error {
-	if restore && m.rr != nil {
-		err := group.Restore(ctx, m.rr, *remoteReadLookBack, m.labels)
-		if err != nil {
-			if !*remoteReadIgnoreRestoreErrors {
-				return fmt.Errorf("failed to restore ruleState for group %q: %w", group.Name, err)
-			}
-			logger.Errorf("error while restoring ruleState for group %q: %s", group.Name, err)
-		}
-	}
-
+func (m *manager) startGroup(ctx context.Context, g *Group, restore bool) error {
 	m.wg.Add(1)
-	id := group.ID()
+	id := g.ID()
 	go func() {
-		group.start(ctx, m.notifiers, m.rw)
+		// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+		if !skipRandSleepOnGroupStart {
+			randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
+			sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
+			if randSleep < sleepOffset {
+				randSleep += uint64(g.Interval)
+			}
+			randSleep -= sleepOffset
+			sleepTimer := time.NewTimer(time.Duration(randSleep))
+			select {
+			case <-ctx.Done():
+				sleepTimer.Stop()
+				return
+			case <-g.doneCh:
+				sleepTimer.Stop()
+				return
+			case <-sleepTimer.C:
+			}
+		}
+		if restore {
+			g.start(ctx, m.notifiers, m.rw, m.rr)
+		} else {
+			g.start(ctx, m.notifiers, m.rw, nil)
+		}
+
 		m.wg.Done()
 	}()
-	m.groups[id] = group
+	m.groups[id] = g
 	return nil
 }
 

app/vmalert/manager_test.go

@@ -64,17 +64,18 @@ func TestManagerUpdateConcurrent(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(workers)
 	for i := 0; i < workers; i++ {
-		go func() {
+		go func(n int) {
 			defer wg.Done()
+			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
-				rnd := rand.Intn(len(paths))
+				rnd := r.Intn(len(paths))
 				cfg, err := config.Parse([]string{paths[rnd]}, notifier.ValidateTemplates, true)
 				if err != nil { // update can fail and this is expected
 					continue
 				}
 				_ = m.update(context.Background(), cfg, false)
 			}
-		}()
+		}(i)
 	}
 	wg.Wait()
 }

app/vmalert/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmalert/notifier/config_watcher_test.go

@@ -162,14 +162,15 @@ consul_sd_configs:
 	wg := sync.WaitGroup{}
 	wg.Add(workers)
 	for i := 0; i < workers; i++ {
-		go func() {
+		go func(n int) {
 			defer wg.Done()
+			r := rand.New(rand.NewSource(int64(n)))
 			for i := 0; i < iterations; i++ {
-				rnd := rand.Intn(len(paths))
+				rnd := r.Intn(len(paths))
 				_ = cw.reload(paths[rnd]) // update can fail and this is expected
 				_ = cw.notifiers()
 			}
-		}()
+		}(i)
 	}
 	wg.Wait()
 }

app/vmalert/remotewrite/remotewrite_test.go

@@ -27,12 +27,13 @@ func TestClient_Push(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to create client: %s", err)
 	}
+	r := rand.New(rand.NewSource(1))
 	const rowsN = 1e4
 	var sent int
 	for i := 0; i < rowsN; i++ {
 		s := prompbmarshal.TimeSeries{
 			Samples: []prompbmarshal.Sample{{
-				Value:     rand.Float64(),
+				Value:     r.Float64(),
 				Timestamp: time.Now().Unix(),
 			}},
 		}

app/vmalert/replay.go

@@ -26,7 +26,7 @@ var (
 			"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
 			"Keep it equal or bigger than -remoteWrite.flushInterval.")
 	replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
-		"Max number of data points expected in one request. The higher the value, the less requests will be made during replay.")
+		"Max number of data points expected in one request. It affects the max time range for every `/query_range` request during the replay. The higher the value, the less requests will be made during replay.")
 	replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
 		"Defines how many retries to make before giving up on rule if request for it returns an error.")
 	disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+

app/vmalert/rule.go

@@ -37,8 +37,6 @@ type ruleState struct {
 	sync.RWMutex
 	entries []ruleStateEntry
 	cur     int
-	// disabled defines whether ruleState tracks ruleStateEntry
-	disabled bool
 }
 
 type ruleStateEntry struct {
@@ -61,7 +59,7 @@ type ruleStateEntry struct {
 
 func newRuleState(size int) *ruleState {
 	if size < 1 {
-		return &ruleState{disabled: true}
+		size = 1
 	}
 	return &ruleState{
 		entries: make([]ruleStateEntry, size),
@@ -69,10 +67,6 @@ func newRuleState(size int) *ruleState {
 }
 
 func (s *ruleState) getLast() ruleStateEntry {
-	if s.disabled {
-		return ruleStateEntry{}
-	}
-
 	s.RLock()
 	defer s.RUnlock()
 	return s.entries[s.cur]
@@ -85,10 +79,6 @@ func (s *ruleState) size() int {
 }
 
 func (s *ruleState) getAll() []ruleStateEntry {
-	if s.disabled {
-		return nil
-	}
-
 	entries := make([]ruleStateEntry, 0)
 
 	s.RLock()
@@ -111,10 +101,6 @@ func (s *ruleState) getAll() []ruleStateEntry {
 }
 
 func (s *ruleState) add(e ruleStateEntry) {
-	if s.disabled {
-		return
-	}
-
 	s.Lock()
 	defer s.Unlock()
 

app/vmalert/rule_test.go

@@ -14,13 +14,13 @@ func TestRule_stateDisabled(t *testing.T) {
 	}
 
 	state.add(ruleStateEntry{at: time.Now()})
-	if !e.at.IsZero() {
-		t.Fatalf("expected entry to be zero")
-	}
+	state.add(ruleStateEntry{at: time.Now()})
+	state.add(ruleStateEntry{at: time.Now()})
 
-	if len(state.getAll()) != 0 {
+	if len(state.getAll()) != 1 {
+		// state should store at least one update at any circumstances
 		t.Fatalf("expected for state to have %d entries; got %d",
-			0, len(state.getAll()),
+			1, len(state.getAll()),
 		)
 	}
 }

app/vmalert/web.qtpl

@@ -40,9 +40,9 @@
             for _, g := range groups {
                 for _, r := range g.Rules {
                     if r.LastError != "" {
-                        rNotOk[g.Name]++
+                        rNotOk[g.ID]++
                     } else {
-                        rOk[g.Name]++
+                        rOk[g.ID]++
                     }
                 }
             }
@@ -50,11 +50,11 @@
          <a class="btn btn-primary" role="button" onclick="collapseAll()">Collapse All</a>
          <a class="btn btn-primary" role="button" onclick="expandAll()">Expand All</a>
         {% for _, g := range groups  %}
-              <div class="group-heading{% if rNotOk[g.Name] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
+              <div class="group-heading{% if rNotOk[g.ID] > 0 %} alert-danger{% endif %}"  data-bs-target="rules-{%s g.ID %}">
                 <span class="anchor" id="group-{%s g.ID %}"></span>
                 <a href="#group-{%s g.ID %}">{%s g.Name %}{% if g.Type != "prometheus" %} ({%s g.Type %}){% endif %} (every {%f.0 g.Interval %}s)</a>
-                 {% if rNotOk[g.Name] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.Name] %}</span> {% endif %}
-                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.Name] %}</span>
+                 {% if rNotOk[g.ID] > 0 %}<span class="badge bg-danger" title="Number of rules with status Error">{%d rNotOk[g.ID] %}</span> {% endif %}
+                <span class="badge bg-success" title="Number of rules withs status Ok">{%d rOk[g.ID] %}</span>
                 <p class="fs-6 fw-lighter">{%s g.File %}</p>
                 {% if len(g.Params) > 0 %}
                     <div class="fs-6 fw-lighter">Extra params
@@ -427,6 +427,16 @@
         </div>
       </div>
     </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Debug
+        </div>
+        <div class="col">
+           {%v rule.Debug %}
+        </div>
+      </div>
+    </div>
     {% endif %}
     <div class="container border-bottom p-2">
       <div class="row">

app/vmalert/web.qtpl.go

@@ -171,9 +171,9 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 		for _, g := range groups {
 			for _, r := range g.Rules {
 				if r.LastError != "" {
-					rNotOk[g.Name]++
+					rNotOk[g.ID]++
 				} else {
-					rOk[g.Name]++
+					rOk[g.ID]++
 				}
 			}
 		}
@@ -189,7 +189,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 			qw422016.N().S(`
               <div class="group-heading`)
 //line app/vmalert/web.qtpl:53
-			if rNotOk[g.Name] > 0 {
+			if rNotOk[g.ID] > 0 {
 //line app/vmalert/web.qtpl:53
 				qw422016.N().S(` alert-danger`)
 //line app/vmalert/web.qtpl:53
@@ -230,11 +230,11 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 			qw422016.N().S(`s)</a>
                  `)
 //line app/vmalert/web.qtpl:56
-			if rNotOk[g.Name] > 0 {
+			if rNotOk[g.ID] > 0 {
 //line app/vmalert/web.qtpl:56
 				qw422016.N().S(`<span class="badge bg-danger" title="Number of rules with status Error">`)
 //line app/vmalert/web.qtpl:56
-				qw422016.N().D(rNotOk[g.Name])
+				qw422016.N().D(rNotOk[g.ID])
 //line app/vmalert/web.qtpl:56
 				qw422016.N().S(`</span> `)
 //line app/vmalert/web.qtpl:56
@@ -243,7 +243,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []APIGr
 			qw422016.N().S(`
                 <span class="badge bg-success" title="Number of rules withs status Ok">`)
 //line app/vmalert/web.qtpl:57
-			qw422016.N().D(rOk[g.Name])
+			qw422016.N().D(rOk[g.ID])
 //line app/vmalert/web.qtpl:57
 			qw422016.N().S(`</span>
                 <p class="fs-6 fw-lighter">`)
@@ -1313,10 +1313,24 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
         </div>
       </div>
     </div>
+    <div class="container border-bottom p-2">
+      <div class="row">
+        <div class="col-2">
+          Debug
+        </div>
+        <div class="col">
+           `)
+//line app/vmalert/web.qtpl:436
+		qw422016.E().V(rule.Debug)
+//line app/vmalert/web.qtpl:436
+		qw422016.N().S(`
+        </div>
+      </div>
+    </div>
     `)
-//line app/vmalert/web.qtpl:430
+//line app/vmalert/web.qtpl:440
 	}
-//line app/vmalert/web.qtpl:430
+//line app/vmalert/web.qtpl:440
 	qw422016.N().S(`
     <div class="container border-bottom p-2">
       <div class="row">
@@ -1325,17 +1339,17 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
         </div>
         <div class="col">
            <a target="_blank" href="`)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.E().S(prefix)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.N().S(`groups#group-`)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.E().S(rule.GroupID)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.E().S(rule.GroupID)
-//line app/vmalert/web.qtpl:437
+//line app/vmalert/web.qtpl:447
 	qw422016.N().S(`</a>
         </div>
       </div>
@@ -1343,13 +1357,13 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
 
     <br>
     <div class="display-6 pb-3">Last `)
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().D(len(rule.Updates))
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().S(`/`)
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().D(rule.MaxUpdates)
-//line app/vmalert/web.qtpl:443
+//line app/vmalert/web.qtpl:453
 	qw422016.N().S(` updates</span>:</div>
         <table class="table table-striped table-hover table-sm">
             <thead>
@@ -1364,201 +1378,201 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule APIRule)
             <tbody>
 
      `)
-//line app/vmalert/web.qtpl:456
+//line app/vmalert/web.qtpl:466
 	for _, u := range rule.Updates {
-//line app/vmalert/web.qtpl:456
+//line app/vmalert/web.qtpl:466
 		qw422016.N().S(`
              <tr`)
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 		if u.err != nil {
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 			qw422016.N().S(` class="alert-danger"`)
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 		}
-//line app/vmalert/web.qtpl:457
+//line app/vmalert/web.qtpl:467
 		qw422016.N().S(`>
                  <td>
                     <span class="badge bg-primary rounded-pill me-3" title="Updated at">`)
-//line app/vmalert/web.qtpl:459
+//line app/vmalert/web.qtpl:469
 		qw422016.E().S(u.time.Format(time.RFC3339))
-//line app/vmalert/web.qtpl:459
+//line app/vmalert/web.qtpl:469
 		qw422016.N().S(`</span>
                  </td>
                  <td class="text-center" wi>`)
-//line app/vmalert/web.qtpl:461
+//line app/vmalert/web.qtpl:471
 		qw422016.N().D(u.samples)
-//line app/vmalert/web.qtpl:461
+//line app/vmalert/web.qtpl:471
 		qw422016.N().S(`</td>
                  <td class="text-center">`)
-//line app/vmalert/web.qtpl:462
+//line app/vmalert/web.qtpl:472
 		qw422016.N().FPrec(u.duration.Seconds(), 3)
-//line app/vmalert/web.qtpl:462
+//line app/vmalert/web.qtpl:472
 		qw422016.N().S(`s</td>
                  <td class="text-center">`)
-//line app/vmalert/web.qtpl:463
+//line app/vmalert/web.qtpl:473
 		qw422016.E().S(u.at.Format(time.RFC3339))
-//line app/vmalert/web.qtpl:463
+//line app/vmalert/web.qtpl:473
 		qw422016.N().S(`</td>
                  <td>
                     <textarea class="curl-area" rows="1" onclick="this.focus();this.select()">`)
-//line app/vmalert/web.qtpl:465
+//line app/vmalert/web.qtpl:475
 		qw422016.E().S(u.curl)
-//line app/vmalert/web.qtpl:465
+//line app/vmalert/web.qtpl:475
 		qw422016.N().S(`</textarea>
                 </td>
              </tr>
           </li>
           `)
-//line app/vmalert/web.qtpl:469
+//line app/vmalert/web.qtpl:479
 		if u.err != nil {
-//line app/vmalert/web.qtpl:469
+//line app/vmalert/web.qtpl:479
 			qw422016.N().S(`
              <tr`)
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 			if u.err != nil {
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 				qw422016.N().S(` class="alert-danger"`)
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 			}
-//line app/vmalert/web.qtpl:470
+//line app/vmalert/web.qtpl:480
 			qw422016.N().S(`>
                <td colspan="5">
                    <span class="alert-danger">`)
-//line app/vmalert/web.qtpl:472
+//line app/vmalert/web.qtpl:482
 			qw422016.E().V(u.err)
-//line app/vmalert/web.qtpl:472
+//line app/vmalert/web.qtpl:482
 			qw422016.N().S(`</span>
                </td>
              </tr>
           `)
-//line app/vmalert/web.qtpl:475
+//line app/vmalert/web.qtpl:485
 		}
-//line app/vmalert/web.qtpl:475
+//line app/vmalert/web.qtpl:485
 		qw422016.N().S(`
      `)
-//line app/vmalert/web.qtpl:476
+//line app/vmalert/web.qtpl:486
 	}
-//line app/vmalert/web.qtpl:476
+//line app/vmalert/web.qtpl:486
 	qw422016.N().S(`
 
     `)
-//line app/vmalert/web.qtpl:478
+//line app/vmalert/web.qtpl:488
 	tpl.StreamFooter(qw422016, r)
-//line app/vmalert/web.qtpl:478
+//line app/vmalert/web.qtpl:488
 	qw422016.N().S(`
 `)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 }
 
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 func WriteRuleDetails(qq422016 qtio422016.Writer, r *http.Request, rule APIRule) {
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	StreamRuleDetails(qw422016, r, rule)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 }
 
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 func RuleDetails(r *http.Request, rule APIRule) string {
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	WriteRuleDetails(qb422016, r, rule)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 	return qs422016
-//line app/vmalert/web.qtpl:479
+//line app/vmalert/web.qtpl:489
 }
 
-//line app/vmalert/web.qtpl:483
+//line app/vmalert/web.qtpl:493
 func streambadgeState(qw422016 *qt422016.Writer, state string) {
-//line app/vmalert/web.qtpl:483
+//line app/vmalert/web.qtpl:493
 	qw422016.N().S(`
 `)
-//line app/vmalert/web.qtpl:485
+//line app/vmalert/web.qtpl:495
 	badgeClass := "bg-warning text-dark"
 	if state == "firing" {
 		badgeClass = "bg-danger"
 	}
 
-//line app/vmalert/web.qtpl:489
+//line app/vmalert/web.qtpl:499
 	qw422016.N().S(`
 <span class="badge `)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.E().S(badgeClass)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.N().S(`">`)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.E().S(state)
-//line app/vmalert/web.qtpl:490
+//line app/vmalert/web.qtpl:500
 	qw422016.N().S(`</span>
 `)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 }
 
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 func writebadgeState(qq422016 qtio422016.Writer, state string) {
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	streambadgeState(qw422016, state)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 }
 
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 func badgeState(state string) string {
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	writebadgeState(qb422016, state)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 	return qs422016
-//line app/vmalert/web.qtpl:491
+//line app/vmalert/web.qtpl:501
 }
 
-//line app/vmalert/web.qtpl:493
+//line app/vmalert/web.qtpl:503
 func streambadgeRestored(qw422016 *qt422016.Writer) {
-//line app/vmalert/web.qtpl:493
+//line app/vmalert/web.qtpl:503
 	qw422016.N().S(`
 <span class="badge bg-warning text-dark" title="Alert state was restored after the service restart from remote storage">restored</span>
 `)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 }
 
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 func writebadgeRestored(qq422016 qtio422016.Writer) {
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	streambadgeRestored(qw422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 }
 
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 func badgeRestored() string {
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	writebadgeRestored(qb422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qs422016 := string(qb422016.B)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 	return qs422016
-//line app/vmalert/web.qtpl:495
+//line app/vmalert/web.qtpl:505
 }

app/vmalert/web_types.go

@@ -113,18 +113,20 @@ type APIRule struct {
 
 	// Additional fields
 
-	// Type of the rule: recording or alerting
+	// DatasourceType of the rule: prometheus or graphite
 	DatasourceType string `json:"datasourceType"`
 	LastSamples    int    `json:"lastSamples"`
 	// ID is a unique Alert's ID within a group
 	ID string `json:"id"`
 	// GroupID is an unique Group's ID
 	GroupID string `json:"group_id"`
+	// Debug shows whether debug mode is enabled
+	Debug bool `json:"debug"`
 
 	// MaxUpdates is the max number of recorded ruleStateEntry objects
 	MaxUpdates int `json:"max_updates_entries"`
 	// Updates contains the ordered list of recorded ruleStateEntry objects
-	Updates []ruleStateEntry `json:"updates"`
+	Updates []ruleStateEntry `json:"-"`
 }
 
 // WebLink returns a link to the alert which can be used in UI.

app/vmauth/README.md

@@ -261,7 +261,11 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -httpAuth.username string
      Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-     TCP address to listen for http connections (default ":8427")
+     TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8427")
+  -httpListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+  -internStringMaxLen int
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
   -logInvalidAuthTokens
      Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
@@ -280,8 +284,10 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
   -loggerWarnsPerSecondLimit int
      Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
+  -maxConcurrentRequests int
+     The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend (default 1000)
   -maxIdleConnsPerBackend int
-     The maximum number of idle connections vmauth can open per each backend host (default 100)
+     The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -301,6 +307,8 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      Supports an array of values separated by comma or specified via multiple flags.
   -reloadAuthKey string
      Auth key for /-/reload http endpoint. It must be passed as authKey=...
+  -responseTimeout duration
+     The timeout for receiving a response from backend (default 5m0s)
   -tls
      Whether to enable TLS for incoming HTTP requests at -httpListenAddr (aka https). -tlsCertFile and -tlsKeyFile must be set if -tls is set
   -tlsCertFile string

app/vmauth/auth_config.go

@@ -37,7 +37,7 @@ type UserInfo struct {
 	Username    string     `yaml:"username,omitempty"`
 	Password    string     `yaml:"password,omitempty"`
 	URLPrefix   *URLPrefix `yaml:"url_prefix,omitempty"`
-	URLMap      []URLMap   `yaml:"url_map,omitempty"`
+	URLMaps     []URLMap   `yaml:"url_map,omitempty"`
 	Headers     []Header   `yaml:"headers,omitempty"`
 
 	requests *metrics.Counter
@@ -284,7 +284,7 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 				return nil, err
 			}
 		}
-		for _, e := range ui.URLMap {
+		for _, e := range ui.URLMaps {
 			if len(e.SrcPaths) == 0 {
 				return nil, fmt.Errorf("missing `src_paths` in `url_map`")
 			}
@@ -295,7 +295,7 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) {
 				return nil, err
 			}
 		}
-		if len(ui.URLMap) == 0 && ui.URLPrefix == nil {
+		if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
 			return nil, fmt.Errorf("missing `url_prefix`")
 		}
 		if ui.BearerToken != "" {

app/vmauth/auth_config_test.go

@@ -278,7 +278,7 @@ users:
 `, map[string]*UserInfo{
 		getAuthToken("foo", "", ""): {
 			BearerToken: "foo",
-			URLMap: []URLMap{
+			URLMaps: []URLMap{
 				{
 					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
@@ -304,7 +304,7 @@ users:
 		},
 		getAuthToken("", "foo", ""): {
 			BearerToken: "foo",
-			URLMap: []URLMap{
+			URLMaps: []URLMap{
 				{
 					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),

app/vmauth/main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"flag"
 	"fmt"
+	"io"
+	"net"
 	"net/http"
-	"net/http/httputil"
+	"net/textproto"
 	"net/url"
 	"os"
 	"strings"
@@ -12,20 +14,28 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
-	httpListenAddr         = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections")
-	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host")
-	reloadAuthKey          = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
-	logInvalidAuthTokens   = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
+	httpListenAddr   = flag.String("httpListenAddr", ":8427", "TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol")
+	useProxyProtocol = flag.Bool("httpListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -httpListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	maxIdleConnsPerBackend = flag.Int("maxIdleConnsPerBackend", 100, "The maximum number of idle connections vmauth can open per each backend host. "+
+		"See also -maxConcurrentRequests")
+	responseTimeout       = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend")
+	maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+
+		"'429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend")
+	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
+	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
 )
 
@@ -41,7 +51,7 @@ func main() {
 	logger.Infof("starting vmauth at %q...", *httpListenAddr)
 	startTime := time.Now()
 	initAuthConfig()
-	go httpserver.Serve(*httpListenAddr, requestHandler)
+	go httpserver.Serve(*httpListenAddr, *useProxyProtocol, requestHandler)
 	logger.Infof("started vmauth in %.3f seconds", time.Since(startTime).Seconds())
 
 	sig := procutil.WaitForSigterm()
@@ -79,15 +89,20 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		// See https://docs.influxdata.com/influxdb/v2.0/api/
 		authToken = strings.Replace(authToken, "Token", "Bearer", 1)
 	}
+
 	ac := authConfig.Load().(map[string]*UserInfo)
 	ui := ac[authToken]
 	if ui == nil {
 		invalidAuthTokenRequests.Inc()
+		err := fmt.Errorf("cannot find the provided auth token %q in config", authToken)
 		if *logInvalidAuthTokens {
-			httpserver.Errorf(w, r, "cannot find the provided auth token %q in config", authToken)
+			err = &httpserver.ErrorWithStatusCode{
+				Err:        err,
+				StatusCode: http.StatusUnauthorized,
+			}
+			httpserver.Errorf(w, r, "%s", err)
 		} else {
-			errStr := fmt.Sprintf("cannot find the provided auth token %q in config", authToken)
-			http.Error(w, errStr, http.StatusBadRequest)
+			http.Error(w, err.Error(), http.StatusUnauthorized)
 		}
 		return true
 	}
@@ -97,26 +112,121 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
 		return true
 	}
-	r.Header.Set("vm-target-url", targetURL.String())
-	for _, h := range headers {
-		r.Header.Set(h.Name, h.Value)
+
+	// Limit the concurrency of requests to backends
+	concurrencyLimitOnce.Do(concurrencyLimitInit)
+	select {
+	case concurrencyLimitCh <- struct{}{}:
+	default:
+		concurrentRequestsLimitReachedTotal.Inc()
+		w.Header().Add("Retry-After", "10")
+		err := &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh)),
+			StatusCode: http.StatusTooManyRequests,
+		}
+		httpserver.Errorf(w, r, "%s", err)
+		return true
 	}
-	proxyRequest(w, r)
+	processRequest(w, r, targetURL, headers)
+	<-concurrencyLimitCh
 	return true
 }
 
-func proxyRequest(w http.ResponseWriter, r *http.Request) {
-	defer func() {
-		err := recover()
-		if err == nil || err == http.ErrAbortHandler {
-			// Suppress http.ErrAbortHandler panic.
-			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1353
-			return
+func processRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) {
+	// This code has been copied from net/http/httputil/reverseproxy.go
+	req := sanitizeRequestHeaders(r)
+	req.URL = targetURL
+	for _, h := range headers {
+		req.Header.Set(h.Name, h.Value)
+	}
+	transportOnce.Do(transportInit)
+	res, err := transport.RoundTrip(req)
+	if err != nil {
+		err = &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf("error when proxying the request to %q: %s", targetURL, err),
+			StatusCode: http.StatusBadGateway,
 		}
-		// Forward other panics to the caller.
-		panic(err)
-	}()
-	getReverseProxy().ServeHTTP(w, r)
+		httpserver.Errorf(w, r, "%s", err)
+		return
+	}
+	removeHopHeaders(res.Header)
+	copyHeader(w.Header(), res.Header)
+	w.WriteHeader(res.StatusCode)
+
+	copyBuf := copyBufPool.Get()
+	copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
+	_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
+	copyBufPool.Put(copyBuf)
+	_ = res.Body.Close()
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		requestURI := httpserver.GetRequestURI(r)
+		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+		return
+	}
+}
+
+var copyBufPool bytesutil.ByteBufferPool
+
+func copyHeader(dst, src http.Header) {
+	for k, vv := range src {
+		for _, v := range vv {
+			dst.Add(k, v)
+		}
+	}
+}
+
+func sanitizeRequestHeaders(r *http.Request) *http.Request {
+	// This code has been copied from net/http/httputil/reverseproxy.go
+	req := r.Clone(r.Context())
+	removeHopHeaders(req.Header)
+	if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
+		// If we aren't the first proxy retain prior
+		// X-Forwarded-For information as a comma+space
+		// separated list and fold multiple headers into one.
+		prior := req.Header["X-Forwarded-For"]
+		if len(prior) > 0 {
+			clientIP = strings.Join(prior, ", ") + ", " + clientIP
+		}
+		req.Header.Set("X-Forwarded-For", clientIP)
+	}
+	return req
+}
+
+func removeHopHeaders(h http.Header) {
+	// remove hop-by-hop headers listed in the "Connection" header of h.
+	// See RFC 7230, section 6.1
+	for _, f := range h["Connection"] {
+		for _, sf := range strings.Split(f, ",") {
+			if sf = textproto.TrimString(sf); sf != "" {
+				h.Del(sf)
+			}
+		}
+	}
+
+	// Remove hop-by-hop headers to the backend. Especially
+	// important is "Connection" because we want a persistent
+	// connection, regardless of what the client sent to us.
+	for _, key := range hopHeaders {
+		h.Del(key)
+	}
+}
+
+// Hop-by-hop headers. These are removed when sent to the backend.
+// As of RFC 7230, hop-by-hop headers are required to appear in the
+// Connection header field. These are the headers defined by the
+// obsoleted RFC 2616 (section 13.5.1) and are used for backward
+// compatibility.
+var hopHeaders = []string{
+	"Connection",
+	"Proxy-Connection", // non-standard but still sent by libcurl and rejected by e.g. google
+	"Keep-Alive",
+	"Proxy-Authenticate",
+	"Proxy-Authorization",
+	"Te",      // canonicalized version of "TE"
+	"Trailer", // not Trailers per URL above; https://www.rfc-editor.org/errata_search.php?eid=4522
+	"Transfer-Encoding",
+	"Upgrade",
 }
 
 var (
@@ -126,43 +236,41 @@ var (
 )
 
 var (
-	reverseProxy     *httputil.ReverseProxy
-	reverseProxyOnce sync.Once
+	transport     *http.Transport
+	transportOnce sync.Once
 )
 
-func getReverseProxy() *httputil.ReverseProxy {
-	reverseProxyOnce.Do(initReverseProxy)
-	return reverseProxy
+func transportInit() {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.ResponseHeaderTimeout = *responseTimeout
+	// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
+	tr.DisableCompression = true
+	// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
+	tr.ForceAttemptHTTP2 = false
+	tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
+	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
+		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
+	}
+	transport = tr
 }
 
-// initReverseProxy must be called after flag.Parse(), since it uses command-line flags.
-func initReverseProxy() {
-	reverseProxy = &httputil.ReverseProxy{
-		Director: func(r *http.Request) {
-			targetURL := r.Header.Get("vm-target-url")
-			target, err := url.Parse(targetURL)
-			if err != nil {
-				logger.Panicf("BUG: unexpected error when parsing targetURL=%q: %s", targetURL, err)
-			}
-			r.URL = target
-		},
-		Transport: func() *http.Transport {
-			tr := http.DefaultTransport.(*http.Transport).Clone()
-			// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
-			tr.DisableCompression = true
-			// Disable HTTP/2.0, since VictoriaMetrics components don't support HTTP/2.0 (because there is no sense in this).
-			tr.ForceAttemptHTTP2 = false
-			tr.MaxIdleConnsPerHost = *maxIdleConnsPerBackend
-			if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
-				tr.MaxIdleConns = tr.MaxIdleConnsPerHost
-			}
-			return tr
-		}(),
-		FlushInterval: time.Second,
-		ErrorLog:      logger.StdErrorLogger(),
-	}
+var (
+	concurrencyLimitCh   chan struct{}
+	concurrencyLimitOnce sync.Once
+)
+
+func concurrencyLimitInit() {
+	concurrencyLimitCh = make(chan struct{}, *maxConcurrentRequests)
+	_ = metrics.NewGauge("vmauth_concurrent_requests_capacity", func() float64 {
+		return float64(*maxConcurrentRequests)
+	})
+	_ = metrics.NewGauge("vmauth_concurrent_requests_current", func() float64 {
+		return float64(len(concurrencyLimitCh))
+	})
 }
 
+var concurrentRequestsLimitReachedTotal = metrics.NewCounter("vmauth_concurrent_requests_limit_reached_total")
+
 func usage() {
 	const s = `
 vmauth authenticates and authorizes incoming requests and proxies them to VictoriaMetrics.

app/vmauth/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmauth/target_url.go

@@ -52,7 +52,7 @@ func createTargetURL(ui *UserInfo, uOrig *url.URL) (*url.URL, []Header, error) {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1554
 		u.Path = ""
 	}
-	for _, e := range ui.URLMap {
+	for _, e := range ui.URLMaps {
 		for _, sp := range e.SrcPaths {
 			if sp.match(u.Path) {
 				return e.URLPrefix.mergeURLs(&u), e.Headers, nil

app/vmauth/target_url_test.go

@@ -54,7 +54,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 
 	// Complex routing with `url_map`
 	ui := &UserInfo{
-		URLMap: []URLMap{
+		URLMaps: []URLMap{
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
@@ -86,7 +86,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 
 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{
-		URLMap: []URLMap{
+		URLMaps: []URLMap{
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
@@ -132,7 +132,7 @@ func TestCreateTargetURLFailure(t *testing.T) {
 	}
 	f(&UserInfo{}, "/foo/bar")
 	f(&UserInfo{
-		URLMap: []URLMap{
+		URLMaps: []URLMap{
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://foobar/baz"),

app/vmbackup/main.go

@@ -65,7 +65,7 @@ func main() {
 				logger.Fatalf("Failed to set snapshot.deleteURL flag: %v", err)
 			}
 		}
-		deleteUrl, err := url.Parse(*snapshotCreateURL)
+		deleteUrl, err := url.Parse(*snapshotDeleteURL)
 		if err != nil {
 			logger.Fatalf("cannot parse snapshotDeleteURL: %s", err)
 		}
@@ -93,7 +93,7 @@ func main() {
 		logger.Fatalf("invalid -snapshotName=%q: %s", *snapshotName, err)
 	}
 
-	go httpserver.Serve(*httpListenAddr, nil)
+	go httpserver.Serve(*httpListenAddr, false, nil)
 
 	srcFS, err := newSrcFS()
 	if err != nil {

app/vmbackup/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmctl/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmctl/prometheus.go

@@ -102,6 +102,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 	if err != nil {
 		return fmt.Errorf("failed to read block: %s", err)
 	}
+	var it chunkenc.Iterator
 	for ss.Next() {
 		var name string
 		var labels []vm.LabelPair
@@ -123,7 +124,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
 
 		var timestamps []int64
 		var values []float64
-		it := series.Iterator()
+		it = series.Iterator(it)
 		for {
 			typ := it.Next()
 			if typ == chunkenc.ValNone {

app/vmctl/testdata/servers_integration_test/remote_read_server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
+	"github.com/prometheus/prometheus/tsdb/chunks"
 )
 
 const (
@@ -154,7 +155,7 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 			t.Fatalf("error unmarshal read request: %s", err)
 		}
 
-		var chunks []prompb.Chunk
+		var chks []prompb.Chunk
 		ctx := context.Background()
 		for idx, r := range req.Queries {
 			startTs := r.StartTimestampMs
@@ -171,9 +172,10 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 			}
 
 			ss := q.Select(false, nil, matchers...)
+			var iter chunks.Iterator
 			for ss.Next() {
 				series := ss.At()
-				iter := series.Iterator()
+				iter = series.Iterator(iter)
 				labels := remote.MergeLabels(labelsToLabelsProto(series.Labels()), nil)
 
 				frameBytesLeft := maxBytesInFrame
@@ -190,14 +192,14 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 						t.Fatalf("error found not populated chunk returned by SeriesSet at ref: %v", chunk.Ref)
 					}
 
-					chunks = append(chunks, prompb.Chunk{
+					chks = append(chks, prompb.Chunk{
 						MinTimeMs: chunk.MinTime,
 						MaxTimeMs: chunk.MaxTime,
 						Type:      prompb.Chunk_Encoding(chunk.Chunk.Encoding()),
 						Data:      chunk.Chunk.Bytes(),
 					})
 
-					frameBytesLeft -= chunks[len(chunks)-1].Size()
+					frameBytesLeft -= chks[len(chks)-1].Size()
 
 					// We are fine with minor inaccuracy of max bytes per frame. The inaccuracy will be max of full chunk size.
 					isNext = iter.Next()
@@ -207,7 +209,7 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 
 					resp := &prompb.ChunkedReadResponse{
 						ChunkedSeries: []*prompb.ChunkedSeries{
-							{Labels: labels, Chunks: chunks},
+							{Labels: labels, Chunks: chks},
 						},
 						QueryIndex: int64(idx),
 					}
@@ -220,7 +222,7 @@ func (rrs *RemoteReadServer) getStreamReadHandler(t *testing.T) http.Handler {
 					if _, err := stream.Write(b); err != nil {
 						t.Fatalf("error write to stream: %s", err)
 					}
-					chunks = chunks[:0]
+					chks = chks[:0]
 					rrs.storage.Reset()
 				}
 				if err := iter.Err(); err != nil {

app/vmgateway/README.md

@@ -304,7 +304,11 @@ The shortlist of configuration flags include the following:
   -httpAuth.username string
      Username for HTTP Basic Auth. The authentication is disabled if empty. See also -httpAuth.password
   -httpListenAddr string
-     TCP address to listen for http connections (default ":8431")
+     TCP address to listen for http connections. See also -httpListenAddr.useProxyProtocol (default ":8431")
+  -httpListenAddr.useProxyProtocol
+     Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
+  -internStringMaxLen int
+     The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300)
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int

app/vminsert/common/streamaggr.go

@@ -16,10 +16,12 @@ import (
 var (
 	streamAggrConfig = flag.String("streamAggr.config", "", "Optional path to file with stream aggregation config. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html . "+
-		"See also -remoteWrite.streamAggr.keepInput")
+		"See also -remoteWrite.streamAggr.keepInput and -streamAggr.dedupInterval")
 	streamAggrKeepInput = flag.Bool("streamAggr.keepInput", false, "Whether to keep input samples after the aggregation with -streamAggr.config. "+
 		"By default the input is dropped after the aggregation, so only the aggregate data is stored. "+
 		"See https://docs.victoriametrics.com/stream-aggregation.html")
+	streamAggrDedupInterval = flag.Duration("streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
+		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
 
 // InitStreamAggr must be called after flag.Parse and before using the common package.
@@ -30,7 +32,7 @@ func InitStreamAggr() {
 		// Nothing to initialize
 		return
 	}
-	a, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries)
+	a, err := streamaggr.LoadFromFile(*streamAggrConfig, pushAggregateSeries, *streamAggrDedupInterval)
 	if err != nil {
 		logger.Fatalf("cannot load -streamAggr.config=%q: %s", *streamAggrConfig, err)
 	}

app/vminsert/main.go

@@ -39,13 +39,25 @@ import (
 )
 
 var (
-	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty")
-	influxListenAddr   = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
-		"This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write")
+	graphiteListenAddr = flag.String("graphiteListenAddr", "", "TCP and UDP address to listen for Graphite plaintext data. Usually :2003 must be set. Doesn't work if empty. "+
+		"See also -graphiteListenAddr.useProxyProtocol")
+	graphiteUseProxyProtocol = flag.Bool("graphiteListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -graphiteListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	influxListenAddr = flag.String("influxListenAddr", "", "TCP and UDP address to listen for InfluxDB line protocol data. Usually :8089 must be set. Doesn't work if empty. "+
+		"This flag isn't needed when ingesting data over HTTP - just send it to http://<victoriametrics>:8428/write . "+
+		"See also -influxListenAddr.useProxyProtocol")
+	influxUseProxyProtocol = flag.Bool("influxListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -influxListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	opentsdbListenAddr = flag.String("opentsdbListenAddr", "", "TCP and UDP address to listen for OpentTSDB metrics. "+
 		"Telnet put messages and HTTP /api/put messages are simultaneously served on TCP port. "+
-		"Usually :4242 must be set. Doesn't work if empty")
-	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
+		"Usually :4242 must be set. Doesn't work if empty. "+
+		"See also -opentsdbListenAddr.useProxyProtocol")
+	opentsdbUseProxyProtocol = flag.Bool("opentsdbListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted at -opentsdbListenAddr . "+
+		"See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
+	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty. "+
+		"See also -opentsdbHTTPListenAddr.useProxyProtocol")
+	opentsdbHTTPUseProxyProtocol = flag.Bool("opentsdbHTTPListenAddr.useProxyProtocol", false, "Whether to use proxy protocol for connections accepted "+
+		"at -opentsdbHTTPListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt")
 	configAuthKey          = flag.String("configAuthKey", "", "Authorization key for accessing /config page. It must be passed via authKey query arg")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped. In this case the vm_metrics_with_dropped_labels_total metric at /metrics page is incremented")
 	maxLabelValueLen       = flag.Int("maxLabelValueLen", 16*1024, "The maximum length of label values in the accepted time series. Longer label values are truncated. In this case the vm_too_long_label_values_total metric at /metrics page is incremented")
@@ -71,16 +83,16 @@ func Init() {
 	storage.SetMaxLabelValueLen(*maxLabelValueLen)
 	common.StartUnmarshalWorkers()
 	if len(*graphiteListenAddr) > 0 {
-		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, graphite.InsertHandler)
+		graphiteServer = graphiteserver.MustStart(*graphiteListenAddr, *graphiteUseProxyProtocol, graphite.InsertHandler)
 	}
 	if len(*influxListenAddr) > 0 {
-		influxServer = influxserver.MustStart(*influxListenAddr, influx.InsertHandlerForReader)
+		influxServer = influxserver.MustStart(*influxListenAddr, *influxUseProxyProtocol, influx.InsertHandlerForReader)
 	}
 	if len(*opentsdbListenAddr) > 0 {
-		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
+		opentsdbServer = opentsdbserver.MustStart(*opentsdbListenAddr, *opentsdbUseProxyProtocol, opentsdb.InsertHandler, opentsdbhttp.InsertHandler)
 	}
 	if len(*opentsdbHTTPListenAddr) > 0 {
-		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, opentsdbhttp.InsertHandler)
+		opentsdbhttpServer = opentsdbhttpserver.MustStart(*opentsdbHTTPListenAddr, *opentsdbHTTPUseProxyProtocol, opentsdbhttp.InsertHandler)
 	}
 	promscrape.Init(func(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		prompush.Push(wr)

app/vminsert/prometheusimport/request_handler.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	parserCommon "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
@@ -29,7 +30,9 @@ func InsertHandler(req *http.Request) error {
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
 	return parser.ParseStream(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
-	}, nil)
+	}, func(s string) {
+		httpserver.LogError(req, s)
+	})
 }
 
 func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {

app/vmrestore/main.go

@@ -38,7 +38,7 @@ func main() {
 	logger.Init()
 	pushmetrics.Init()
 
-	go httpserver.Serve(*httpListenAddr, nil)
+	go httpserver.Serve(*httpListenAddr, false, nil)
 
 	srcFS, err := newSrcFS()
 	if err != nil {

app/vmrestore/multiarch/Dockerfile

@@ -2,7 +2,7 @@
 ARG certs_image
 ARG root_image
 FROM $certs_image as certs
-RUN apk --update --no-cache add ca-certificates
+RUN apk update && apk upgrade && apk --update --no-cache add ca-certificates
 
 FROM $root_image
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

app/vmselect/bufferedwriter/bufferedwriter.go

@@ -4,8 +4,9 @@ import (
 	"bufio"
 	"fmt"
 	"io"
-	"strings"
 	"sync"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 )
 
 // Get returns buffered writer for the given w.
@@ -63,7 +64,7 @@ func (bw *Writer) Write(p []byte) (int, error) {
 		return 0, bw.err
 	}
 	n, err := bw.bw.Write(p)
-	if err != nil && !isTrivialNetworkError(err) {
+	if err != nil && !netutil.IsTrivialNetworkError(err) {
 		bw.err = fmt.Errorf("cannot send %d bytes to client: %w", len(p), err)
 	}
 	return n, bw.err
@@ -76,7 +77,7 @@ func (bw *Writer) Flush() error {
 	if bw.err != nil {
 		return bw.err
 	}
-	if err := bw.bw.Flush(); err != nil && !isTrivialNetworkError(err) {
+	if err := bw.bw.Flush(); err != nil && !netutil.IsTrivialNetworkError(err) {
 		bw.err = fmt.Errorf("cannot flush data to client: %w", err)
 	}
 	return bw.err
@@ -88,12 +89,3 @@ func (bw *Writer) Error() error {
 	defer bw.lock.Unlock()
 	return bw.err
 }
-
-func isTrivialNetworkError(err error) bool {
-	// Suppress trivial network errors, which could occur at remote side.
-	s := err.Error()
-	if strings.Contains(s, "broken pipe") || strings.Contains(s, "reset by peer") {
-		return true
-	}
-	return false
-}

app/vmselect/promql/exec_test.go

@@ -6122,7 +6122,7 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() < 1300)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1000, 1200, 1200, 1200, 1200, 1200},
+			Values:     []float64{1000, 1200, nan, nan, nan, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}
@@ -6133,7 +6133,18 @@ func TestExecSuccess(t *testing.T) {
 		q := `interpolate(time() > 1500)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{1600, 1600, 1600, 1600, 1800, 2000},
+			Values:     []float64{nan, nan, nan, 1600, 1800, 2000},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r1}
+		f(q, resultExpected)
+	})
+	t.Run(`interpolate(tail_head_and_middle)`, func(t *testing.T) {
+		t.Parallel()
+		q := `interpolate(time() > 1100 and time() < 1300 default time() > 1700 and time() < 1900)`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{nan, 1200, 1400, 1600, 1800, nan},
 			Timestamps: timestampsExpected,
 		}
 		resultExpected := []netstorage.Result{r1}

app/vmselect/promql/rollup_timing_test.go

@@ -35,9 +35,10 @@ var (
 )
 
 var benchValues = func() []float64 {
+	r := rand.New(rand.NewSource(1))
 	values := make([]float64, 1000)
 	for i := range values {
-		values[i] = rand.Float64() * 100
+		values[i] = r.Float64() * 100
 	}
 	return values
 }()

app/vmselect/promql/transform.go

@@ -1165,7 +1165,8 @@ func transformInterpolate(tfa *transformFuncArg) ([]*timeseries, error) {
 	}
 	rvs := args[0]
 	for _, ts := range rvs {
-		values := ts.Values
+		values := skipLeadingNaNs(ts.Values)
+		values = skipTrailingNaNs(values)
 		if len(values) == 0 {
 			continue
 		}

app/vmselect/vmui/asset-manifest.json

@@ -1,12 +1,14 @@
 {
   "files": {
-    "main.css": "./static/css/main.e8e53cbf.css",
-    "main.js": "./static/js/main.58fe7908.js",
+    "main.css": "./static/css/main.b9c2d13c.css",
+    "main.js": "./static/js/main.40a4969a.js",
     "static/js/27.c1ccfd29.chunk.js": "./static/js/27.c1ccfd29.chunk.js",
+    "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
+    "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.e8e53cbf.css",
-    "static/js/main.58fe7908.js"
+    "static/css/main.b9c2d13c.css",
+    "static/js/main.40a4969a.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet"><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.58fe7908.js"></script><link href="./static/css/main.e8e53cbf.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.40a4969a.js"></script><link href="./static/css/main.b9c2d13c.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmselect/vmui/static/js/main.40a4969a.js.LICENSE.txt

@@ -7,7 +7,7 @@
 /*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */
 
 /**
- * @remix-run/router v1.0.5
+ * @remix-run/router v1.3.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -18,7 +18,7 @@
  */
 
 /**
- * React Router DOM v6.4.5
+ * React Router DOM v6.7.0
  *
  * Copyright (c) Remix Software Inc.
  *
@@ -29,7 +29,7 @@
  */
 
 /**
- * React Router v6.4.5
+ * React Router v6.7.0
  *
  * Copyright (c) Remix Software Inc.
  *

app/vmstorage/main.go

@@ -84,14 +84,6 @@ func CheckTimeRange(tr storage.TimeRange) error {
 
 // Init initializes vmstorage.
 func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
-	InitWithoutMetrics(resetCacheIfNeeded)
-	registerStorageMetrics(Storage)
-}
-
-// InitWithoutMetrics must be called instead of Init inside tests.
-//
-// This allows multiple Init / Stop cycles.
-func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
 		logger.Fatalf("invalid `-precisionBits`: %s", err)
 	}
@@ -130,6 +122,7 @@ func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
 	logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
 		*DataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
+	registerStorageMetrics(Storage)
 }
 
 // Storage is a storage.

app/vmui/Dockerfile-build

@@ -1,4 +1,4 @@
-FROM node:18-alpine3.15
+FROM node:18-alpine3.17
 
 RUN apk update && apk upgrade
 RUN apk add --no-cache bash bash-doc bash-completion libtool autoconf automake nasm pkgconfig libpng gcc make g++ zlib-dev gawk

app/vmui/Dockerfile-web

@@ -1,4 +1,4 @@
-FROM golang:1.19.5 as build-web-stage
+FROM golang:1.20.1 as build-web-stage
 COPY build /build
 
 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
     GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
 
-FROM alpine:3.17.1
+FROM alpine:3.17.2
 USER root
 
 COPY --from=build-web-stage /build/web-amd64 /app/web

app/vmui/packages/vmui/README.md

@@ -60,7 +60,7 @@ VMUI can be used to paste into other applications
 | Name                    |   Default   |                                                                           Description |
 |:------------------------|:-----------:|--------------------------------------------------------------------------------------:|
 | serverURL               | domain name |                                                          Can't be changed from the UI |
-| inputTenantID           |      -      |                            If the flag is present, the "Tenant ID" field is displayed |
+| useTenantID             |      -      |                           If the flag is present, the "Tenant ID" select is displayed |
 | headerStyles.background |  `#FFFFFF`  |                                                               Header background color |
 | headerStyles.color      |  `#3F51B5`  |                                                                     Header font color |
 | palette.primary         |  `#3F51B5`  |                               used to represent primary interface elements for a user |
@@ -74,7 +74,7 @@ VMUI can be used to paste into other applications
 ```json
 {
   "serverURL": "http://localhost:8428",
-  "inputTenantID": "true",
+  "useTenantID": true,
   "headerStyles": {
     "background": "#FFFFFF",
     "color": "#538DE8"
@@ -93,7 +93,7 @@ VMUI can be used to paste into other applications
 
 #### HTML example:
 ```html
-<div id="root" data-params='{"serverURL":"http://localhost:8428","inputTenantID":"true","headerStyles":{"background":"#FFFFFF","color":"#538DE8"},"palette":{"primary":"#538DE8","secondary":"#F76F8E","error":"#FD151B","warning":"#FFB30F","success":"#7BE622","info":"#0F5BFF"}}'></div>
+<div id="root" data-params='{"serverURL":"http://localhost:8428","useTenantID":true,"headerStyles":{"background":"#FFFFFF","color":"#538DE8"},"palette":{"primary":"#538DE8","secondary":"#F76F8E","error":"#FD151B","warning":"#FFB30F","success":"#7BE622","info":"#0F5BFF"}}'></div>
 ```
 
 

app/vmui/packages/vmui/public/index.html

@@ -3,11 +3,11 @@
   <head>
     <meta charset="utf-8" />
     <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
     <meta name="theme-color" content="#000000" />
     <meta
       name="description"
-      content="VM-UI is a metric explorer for Victoria Metrics"
+      content="UI for VictoriaMetrics"
     />
     <link rel="apple-touch-icon" href="%PUBLIC_URL%/apple-touch-icon.png" />
     <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
@@ -26,10 +26,18 @@
       Learn how to configure a non-root public URL by running `npm run build`.
     -->
     <title>VM UI</title>
-    <link rel="preconnect" href="https://fonts.googleapis.com">
-    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
-    <link href="https://fonts.googleapis.com/css2?family=JetBrains+Mono&family=Lato:wght@300;400;700&display=swap" rel="stylesheet">
     <script src="%PUBLIC_URL%/dashboards/index.js" type="module"></script>
+
+    <meta name="twitter:card" content="summary_large_image">
+    <meta name="twitter:image" content="%PUBLIC_URL%/preview.jpg">
+    <meta name="twitter:title" content="UI for VictoriaMetrics">
+    <meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data">
+    <meta name="twitter:site" content="@VictoriaMetrics">
+
+    <meta property="og:title" content="Metric explorer for VictoriaMetrics">
+    <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
+    <meta property="og:image" content="%PUBLIC_URL%/preview.jpg">
+    <meta property="og:type" content="website">
   </head>
   <body>
     <noscript>You need to enable JavaScript to run this app.</noscript>

app/vmui/packages/vmui/src/App.tsx

@@ -8,60 +8,57 @@ import DashboardsLayout from "./pages/PredefinedPanels";
 import CardinalityPanel from "./pages/CardinalityPanel";
 import TopQueries from "./pages/TopQueries";
 import ThemeProvider from "./components/Main/ThemeProvider/ThemeProvider";
-import Spinner from "./components/Main/Spinner/Spinner";
 import TracePage from "./pages/TracePage";
 import ExploreMetrics from "./pages/ExploreMetrics";
 import PreviewIcons from "./components/Main/Icons/PreviewIcons";
 
 const App: FC = () => {
 
-  const [loadingTheme, setLoadingTheme] = useState(true);
-
-  if (loadingTheme) return (
-    <>
-      <Spinner/>
-      <ThemeProvider setLoadingTheme={setLoadingTheme}/>;
-    </>
-  );
+  const [loadedTheme, setLoadedTheme] = useState(false);
 
   return <>
     <HashRouter>
       <AppContextProvider>
-        <Routes>
-          <Route
-            path={"/"}
-            element={<Layout/>}
-          >
-            <Route
-              path={router.home}
-              element={<CustomPanel/>}
-            />
-            <Route
-              path={router.metrics}
-              element={<ExploreMetrics/>}
-            />
-            <Route
-              path={router.cardinality}
-              element={<CardinalityPanel/>}
-            />
-            <Route
-              path={router.topQueries}
-              element={<TopQueries/>}
-            />
-            <Route
-              path={router.trace}
-              element={<TracePage/>}
-            />
-            <Route
-              path={router.dashboards}
-              element={<DashboardsLayout/>}
-            />
-            <Route
-              path={router.icons}
-              element={<PreviewIcons/>}
-            />
-          </Route>
-        </Routes>
+        <>
+          <ThemeProvider onLoaded={setLoadedTheme}/>
+          {loadedTheme && (
+            <Routes>
+              <Route
+                path={"/"}
+                element={<Layout/>}
+              >
+                <Route
+                  path={router.home}
+                  element={<CustomPanel/>}
+                />
+                <Route
+                  path={router.metrics}
+                  element={<ExploreMetrics/>}
+                />
+                <Route
+                  path={router.cardinality}
+                  element={<CardinalityPanel/>}
+                />
+                <Route
+                  path={router.topQueries}
+                  element={<TopQueries/>}
+                />
+                <Route
+                  path={router.trace}
+                  element={<TracePage/>}
+                />
+                <Route
+                  path={router.dashboards}
+                  element={<DashboardsLayout/>}
+                />
+                <Route
+                  path={router.icons}
+                  element={<PreviewIcons/>}
+                />
+              </Route>
+            </Routes>
+          )}
+        </>
       </AppContextProvider>
     </HashRouter>
   </>;

app/vmui/packages/vmui/src/api/accountId.ts

@@ -0,0 +1,2 @@
+export const getAccountIds = (server: string) =>
+  `${server.replace(/^(.+)(\/select.+)/, "$1")}/admin/tenants`;

app/vmui/packages/vmui/src/components/Chart/BarChart/BarChart.tsx

@@ -3,11 +3,13 @@ import uPlot, { Options as uPlotOptions } from "uplot";
 import useResize from "../../../hooks/useResize";
 import { BarChartProps } from "./types";
 import "./style.scss";
+import { useAppState } from "../../../state/common/StateContext";
 
 const BarChart: FC<BarChartProps> = ({
   data,
   container,
   configs }) => {
+  const { isDarkTheme } = useAppState();
 
   const uPlotRef = useRef<HTMLDivElement>(null);
   const [uPlotInst, setUPlotInst] = useState<uPlot>();
@@ -28,7 +30,7 @@ const BarChart: FC<BarChartProps> = ({
     const u = new uPlot(options, data, uPlotRef.current);
     setUPlotInst(u);
     return u.destroy;
-  }, [uPlotRef.current, layoutSize]);
+  }, [uPlotRef.current, layoutSize, isDarkTheme]);
 
   useEffect(() => updateChart(), [data]);
 

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/ChartTooltip.tsx

@@ -1,5 +1,5 @@
 import React, { FC, useEffect, useMemo, useRef, useState } from "preact/compat";
-import uPlot, { Series } from "uplot";
+import uPlot from "uplot";
 import { MetricResult } from "../../../api/types";
 import { formatPrettyNumber } from "../../../utils/uplot/helpers";
 import dayjs from "dayjs";
@@ -11,12 +11,13 @@ import { CloseIcon, DragIcon } from "../../Main/Icons";
 import classNames from "classnames";
 import { MouseEvent as ReactMouseEvent } from "react";
 import "./style.scss";
+import { SeriesItem } from "../../../utils/uplot/series";
 
 export interface ChartTooltipProps {
   id: string,
   u: uPlot,
   metrics: MetricResult[],
-  series: Series[],
+  series: SeriesItem[],
   yRange: number[];
   unit?: string,
   isSticky?: boolean,
@@ -55,15 +56,16 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
 
   const color = series[seriesIdx]?.stroke+"";
 
+  const calculations = series[seriesIdx]?.calculations || {};
+
   const groups = new Set(metrics.map(m => m.group));
   const showQueryNum = groups.size > 1;
   const group = metrics[seriesIdx-1]?.group || 0;
 
   const metric = metrics[seriesIdx-1]?.metric || {};
-  const labelNames = Object.keys(metric).filter(x => x != "__name__");
-  const metricName = metric["__name__"] || "value";
 
   const fields = useMemo(() => {
+    const labelNames = Object.keys(metric).filter(x => x != "__name__");
     return labelNames.map(key => `${key}=${JSON.stringify(metric[key])}`);
   }, [metrics, seriesIdx]);
 
@@ -100,10 +102,15 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
     const overflowX = leftOnChart + tooltipWidth >= width ? tooltipWidth + (2 * margin) : 0;
     const overflowY = topOnChart + tooltipHeight >= height ? tooltipHeight + (2 * margin) : 0;
 
-    setPosition({
+    const position = {
       top: topOnChart + tooltipOffset.top + margin - overflowY,
       left: leftOnChart + tooltipOffset.left + margin - overflowX
-    });
+    };
+
+    if (position.left < 0) position.left = 20;
+    if (position.top < 0) position.top = 20;
+
+    setPosition(position);
   };
 
   useEffect(calcPosition, [u, value, dataTime, seriesIdx, tooltipOffset, tooltipRef]);
@@ -169,19 +176,21 @@ const ChartTooltip: FC<ChartTooltipProps> = ({
           className="vm-chart-tooltip-data__marker"
           style={{ background: color }}
         />
-        <p>
-          {metricName}: 
-          <b className="vm-chart-tooltip-data__value">{valueFormat}</b>
-          {unit}
-        </p>
-      </div>
-      {!!fields.length && (
-        <div className="vm-chart-tooltip-info">
-          {fields.map((f, i) => (
-            <div key={`${f}_${i}`}>{f}</div>
-          ))}
+        <div>
+          curr:<b>{valueFormat}{unit}</b>, avg:<b>{calculations.avg}</b><br/>
+          min:<b>{calculations.min}</b>, max:<b>{calculations.max}</b>, last:<b>{calculations.last}</b>
         </div>
-      )}
+      </div>
+      <div className="vm-chart-tooltip-info">
+        {metric["__name__"]}
+        &#123;
+        {fields.map((f, i) => (
+          <span key="{i}">
+            {f}{i +1 < fields.length && ","}
+          </span>
+        ))}
+        &#125;
+      </div>
     </div>
   ), targetPortal);
 };

app/vmui/packages/vmui/src/components/Chart/ChartTooltip/style.scss

@@ -25,7 +25,6 @@ $chart-tooltip-y: -1 * ($padding-small + $chart-tooltip-half-icon);
   pointer-events: none;
 
   &_sticky {
-    background-color: $color-dove-gray;
     pointer-events: auto;
     z-index: 99;
   }
@@ -62,11 +61,6 @@ $chart-tooltip-y: -1 * ($padding-small + $chart-tooltip-half-icon);
     word-break: break-all;
     line-height: 12px;
 
-    &__value {
-      padding: 4px;
-      font-weight: bold;
-    }
-
     &__marker {
       width: 12px;
       height: 12px;

app/vmui/packages/vmui/src/components/Chart/Legend/LegendItem/LegendItem.tsx

@@ -14,6 +14,7 @@ interface LegendItemProps {
 const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
   const [copiedValue, setCopiedValue] = useState("");
   const freeFormFields = useMemo(() => getFreeFields(legend), [legend]);
+  const calculations = legend.calculations;
 
   const handleClickFreeField = async (val: string, id: string) => {
     await navigator.clipboard.writeText(val);
@@ -30,11 +31,11 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
     handleClickFreeField(freeField, id);
   };
 
-
   return (
     <div
       className={classNames({
         "vm-legend-item": true,
+        "vm-legend-row": true,
         "vm-legend-item_hide": !legend.checked,
       })}
       onClick={createHandlerClick(legend)}
@@ -45,30 +46,30 @@ const LegendItem: FC<LegendItemProps> = ({ legend, onChange }) => {
       />
       <div className="vm-legend-item-info">
         <span className="vm-legend-item-info__label">
-          {legend.freeFormFields["__name__"] || (freeFormFields.length == 0 ? "{}" : "")}
-        </span>
-        {freeFormFields.length > 0 &&
-          <span>
-            &#123;
-            {freeFormFields.map(f => (
-              <Tooltip
-                key={f.id}
-                open={copiedValue === f.id}
-                title={"Copied!"}
-                placement="top-center"
+          {legend.freeFormFields["__name__"]}
+          &#123;
+          {freeFormFields.map((f, i) => (
+            <Tooltip
+              key={f.id}
+              open={copiedValue === f.id}
+              title={"copied!"}
+              placement="top-center"
+            >
+              <span
+                className="vm-legend-item-info__free-fields"
+                key={f.key}
+                onClick={createHandlerCopy(f.freeField, f.id)}
+                title="copy to clipboard"
               >
-                <span
-                  className="vm-legend-item-info__free-fields"
-                  key={f.key}
-                  onClick={createHandlerCopy(f.freeField, f.id)}
-                >
-                  {f.freeField}
-                </span>
-              </Tooltip>
-            ))}
-            &#125;
-          </span>
-        }
+                {f.freeField}{i + 1 < freeFormFields.length && ","}
+              </span>
+            </Tooltip>
+          ))}
+          &#125;
+        </span>
+      </div>
+      <div className="vm-legend-item-values">
+        avg:{calculations.avg}, min:{calculations.min}, max:{calculations.max}, last:{calculations.last}
       </div>
     </div>
   );

app/vmui/packages/vmui/src/components/Chart/Legend/LegendItem/style.scss

@@ -6,10 +6,11 @@
   grid-gap: $padding-small;
   align-items: start;
   justify-content: start;
-  padding: $padding-small $padding-large $padding-small $padding-small;
+  padding: $padding-small;
   background-color: $color-background-block;
   cursor: pointer;
   transition: 0.2s ease;
+  margin-bottom: $padding-small;
 
   &:hover {
     background-color: rgba(0, 0, 0, 0.1);
@@ -30,22 +31,26 @@
 
   &-info {
     font-weight: normal;
+    word-break: break-all;
 
     &__label {
-
+      margin-right: 2px;
     }
 
     &__free-fields {
-      padding: 3px;
+      padding: 2px;
       cursor: pointer;
 
       &:hover {
         text-decoration: underline;
       }
-
-      &:not(:last-child):after {
-        content: ",";
-      }
     }
   }
+
+  &-values {
+    grid-column: 2;
+    display: flex;
+    align-items: center;
+    gap: $padding-small;
+  }
 }

app/vmui/packages/vmui/src/components/Chart/Legend/style.scss

@@ -9,12 +9,13 @@
 
   &-group {
     min-width: 23%;
+    width: 100%;
     margin: 0 $padding-global $padding-global 0;
 
     &-title {
       display: flex;
       align-items: center;
-      padding: 0 $padding-small $padding-small;
+      padding: $padding-small;
       margin-bottom: 1px;
       border-bottom: $border-divider;
 

app/vmui/packages/vmui/src/components/Chart/LineChart/LineChart.tsx

@@ -21,6 +21,8 @@ import "./style.scss";
 import classNames from "classnames";
 import ChartTooltip, { ChartTooltipProps } from "../ChartTooltip/ChartTooltip";
 import dayjs from "dayjs";
+import { useAppState } from "../../../state/common/StateContext";
+import { SeriesItem } from "../../../utils/uplot/series";
 
 export interface LineChartProps {
   metrics: MetricResult[];
@@ -47,11 +49,14 @@ const LineChart: FC<LineChartProps> = ({
   container,
   height
 }) => {
+  const { isDarkTheme } = useAppState();
+
   const uPlotRef = useRef<HTMLDivElement>(null);
   const [isPanning, setPanning] = useState(false);
   const [xRange, setXRange] = useState({ min: period.start, max: period.end });
   const [yRange, setYRange] = useState([0, 1]);
   const [uPlotInst, setUPlotInst] = useState<uPlot>();
+  const [startTouchDistance, setStartTouchDistance] = useState(0);
   const layoutSize = useResize(container);
 
   const [showTooltip, setShowTooltip] = useState(false);
@@ -81,6 +86,7 @@ const LineChart: FC<LineChartProps> = ({
       left: parseFloat(u.over.style.left),
       top: parseFloat(u.over.style.top)
     });
+
     u.over.addEventListener("mousedown", e => {
       const { ctrlKey, metaKey, button } = e;
       const leftClick = button === 0;
@@ -91,6 +97,10 @@ const LineChart: FC<LineChartProps> = ({
       }
     });
 
+    u.over.addEventListener("touchstart", e => {
+      dragChart({ u, e, setPanning, setPlotScale, factor });
+    });
+
     u.over.addEventListener("wheel", e => {
       if (!e.ctrlKey && !e.metaKey) return;
       e.preventDefault();
@@ -222,7 +232,7 @@ const LineChart: FC<LineChartProps> = ({
     setUPlotInst(u);
     setXRange({ min: period.start, max: period.end });
     return u.destroy;
-  }, [uPlotRef.current, series, layoutSize, height]);
+  }, [uPlotRef.current, series, layoutSize, height, isDarkTheme]);
 
   useEffect(() => {
     window.addEventListener("keydown", handleKeyDown);
@@ -232,6 +242,47 @@ const LineChart: FC<LineChartProps> = ({
     };
   }, [xRange]);
 
+  const handleTouchStart = (e: TouchEvent) => {
+    if (e.touches.length !== 2) return;
+    e.preventDefault();
+
+    const dx = e.touches[0].clientX - e.touches[1].clientX;
+    const dy = e.touches[0].clientY - e.touches[1].clientY;
+    setStartTouchDistance(Math.sqrt(dx * dx + dy * dy));
+  };
+
+  const handleTouchMove = (e: TouchEvent) => {
+    if (e.touches.length !== 2 || !uPlotInst) return;
+    e.preventDefault();
+
+    const dx = e.touches[0].clientX - e.touches[1].clientX;
+    const dy = e.touches[0].clientY - e.touches[1].clientY;
+    const endTouchDistance = Math.sqrt(dx * dx + dy * dy);
+    const diffDistance = startTouchDistance - endTouchDistance;
+
+    const max = (uPlotInst.scales.x.max || xRange.max);
+    const min = (uPlotInst.scales.x.min || xRange.min);
+    const dur = max - min;
+    const dir = (diffDistance > 0 ? -1 : 1);
+
+    const zoomFactor = dur / 50 * dir;
+    uPlotInst.batch(() => setPlotScale({
+      u: uPlotInst,
+      min: min + zoomFactor,
+      max: max - zoomFactor
+    }));
+  };
+
+  useEffect(() => {
+    window.addEventListener("touchmove", handleTouchMove);
+    window.addEventListener("touchstart", handleTouchStart);
+
+    return () => {
+      window.removeEventListener("touchmove", handleTouchMove);
+      window.removeEventListener("touchstart", handleTouchStart);
+    };
+  }, [uPlotInst, startTouchDistance]);
+
   useEffect(() => updateChart(typeChartUpdate.data), [data]);
   useEffect(() => updateChart(typeChartUpdate.xRange), [xRange]);
   useEffect(() => updateChart(typeChartUpdate.yRange), [yaxis]);
@@ -253,6 +304,10 @@ const LineChart: FC<LineChartProps> = ({
         "vm-line-chart": true,
         "vm-line-chart_panning": isPanning
       })}
+      style={{
+        minWidth: `${layoutSize.width || 400}px`,
+        minHeight: `${height || 500}px`
+      }}
     >
       <div
         className="vm-line-chart__u-plot"
@@ -262,7 +317,7 @@ const LineChart: FC<LineChartProps> = ({
         <ChartTooltip
           unit={unit}
           u={uPlotInst}
-          series={series}
+          series={series as SeriesItem[]}
           metrics={metrics}
           yRange={yRange}
           tooltipIdx={tooltipIdx}

app/vmui/packages/vmui/src/components/Configurators/AdditionalSettings/AdditionalSettings.tsx

@@ -1,6 +1,4 @@
 import React, { FC } from "preact/compat";
-import { getAppModeParams } from "../../../utils/app-mode";
-import TenantsConfiguration from "../TenantsConfiguration/TenantsConfiguration";
 import { useCustomPanelDispatch, useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
 import { useQueryDispatch, useQueryState } from "../../../state/query/QueryStateContext";
 import "./style.scss";
@@ -8,8 +6,6 @@ import Switch from "../../Main/Switch/Switch";
 
 const AdditionalSettings: FC = () => {
 
-  const { inputTenantID } = getAppModeParams();
-
   const { autocomplete } = useQueryState();
   const queryDispatch = useQueryDispatch();
 
@@ -44,11 +40,6 @@ const AdditionalSettings: FC = () => {
       value={isTracingEnabled}
       onChange={onChangeQueryTracing}
     />
-    {!!inputTenantID && (
-      <div className="vm-additional-settings__input">
-        <TenantsConfiguration/>
-      </div>
-    )}
   </div>;
 };
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/GlobalSettings.tsx

@@ -1,4 +1,4 @@
-import React, { FC, useState } from "preact/compat";
+import React, { FC, useEffect, useState } from "preact/compat";
 import ServerConfigurator from "./ServerConfigurator/ServerConfigurator";
 import { useAppDispatch, useAppState } from "../../../state/common/StateContext";
 import { SettingsIcon } from "../../Main/Icons";
@@ -13,10 +13,13 @@ import { getAppModeEnable } from "../../../utils/app-mode";
 import classNames from "classnames";
 import Timezones from "./Timezones/Timezones";
 import { useTimeDispatch, useTimeState } from "../../../state/time/TimeStateContext";
+import ThemeControl from "../ThemeControl/ThemeControl";
+import useDeviceDetect from "../../../hooks/useDeviceDetect";
 
 const title = "Settings";
 
-const GlobalSettings: FC = () => {
+const GlobalSettings: FC<{showTitle?: boolean}> = ({ showTitle }) => {
+  const { isMobile } = useDeviceDetect();
 
   const appModeEnable = getAppModeEnable();
   const { serverUrl: stateServerUrl } = useAppState();
@@ -42,8 +45,16 @@ const GlobalSettings: FC = () => {
     handleClose();
   };
 
+  useEffect(() => {
+    if (stateServerUrl === serverUrl) return;
+    setServerUrl(stateServerUrl);
+  }, [stateServerUrl]);
+
   return <>
-    <Tooltip title={title}>
+    <Tooltip
+      open={showTitle === true ? false : undefined}
+      title={title}
+    >
       <Button
         className={classNames({
           "vm-header-button": !appModeEnable
@@ -52,14 +63,21 @@ const GlobalSettings: FC = () => {
         color="primary"
         startIcon={<SettingsIcon/>}
         onClick={handleOpen}
-      />
+      >
+        {showTitle && title}
+      </Button>
     </Tooltip>
     {open && (
       <Modal
         title={title}
         onClose={handleClose}
       >
-        <div className="vm-server-configurator">
+        <div
+          className={classNames({
+            "vm-server-configurator": true,
+            "vm-server-configurator_mobile": isMobile
+          })}
+        >
           {!appModeEnable && (
             <div className="vm-server-configurator__input">
               <ServerConfigurator
@@ -82,19 +100,24 @@ const GlobalSettings: FC = () => {
               onChange={setTimezone}
             />
           </div>
+          {!appModeEnable && (
+            <div className="vm-server-configurator__input">
+              <ThemeControl/>
+            </div>
+          )}
           <div className="vm-server-configurator__footer">
             <Button
               variant="outlined"
               color="error"
               onClick={handleClose}
             >
-                Cancel
+              Cancel
             </Button>
             <Button
               variant="contained"
               onClick={handlerApply}
             >
-                apply
+              apply
             </Button>
           </div>
         </div>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/LimitsConfigurator.tsx

@@ -70,15 +70,16 @@ const LimitsConfigurator: FC<ServerConfiguratorProps> = ({ limits, onChange , on
       </div>
       <div className="vm-limits-configurator__inputs">
         {fields.map(f => (
-          <TextField
-            key={f.type}
-            label={f.label}
-            value={limits[f.type]}
-            error={error[f.type]}
-            onChange={createChangeHandler(f.type)}
-            onEnter={onEnter}
-            type="number"
-          />
+          <div key={f.type}>
+            <TextField
+              label={f.label}
+              value={limits[f.type]}
+              error={error[f.type]}
+              onChange={createChangeHandler(f.type)}
+              onEnter={onEnter}
+              type="number"
+            />
+          </div>
         ))}
       </div>
     </div>

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/LimitsConfigurator/style.scss

@@ -12,10 +12,14 @@
   }
 
   &__inputs {
-    display: grid;
-    grid-template-columns: repeat(3, 1fr);
+    display: flex;
+    flex-wrap: wrap;
     align-items: center;
     justify-content: space-between;
     gap: $padding-global;
+
+    div {
+      flex-grow: 1;
+    }
   }
 }

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/ServerConfigurator/ServerConfigurator.tsx

@@ -22,18 +22,14 @@ const ServerConfigurator: FC<ServerConfiguratorProps> = ({ serverUrl, onChange ,
   };
 
   return (
-    <div>
-      <div className="vm-server-configurator__title">
-        Server URL
-      </div>
-      <TextField
-        autofocus
-        value={serverUrl}
-        error={error}
-        onChange={onChangeServer}
-        onEnter={onEnter}
-      />
-    </div>
+    <TextField
+      autofocus
+      label="Server URL"
+      value={serverUrl}
+      error={error}
+      onChange={onChangeServer}
+      onEnter={onEnter}
+    />
   );
 };
 

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/TenantsConfiguration.tsx

@@ -0,0 +1,139 @@
+import React, { FC, useState, useRef, useEffect, useMemo } from "preact/compat";
+import { useAppDispatch, useAppState } from "../../../../state/common/StateContext";
+import { useTimeDispatch } from "../../../../state/time/TimeStateContext";
+import { ArrowDownIcon, StorageIcon } from "../../../Main/Icons";
+import Button from "../../../Main/Button/Button";
+import "./style.scss";
+import { replaceTenantId } from "../../../../utils/default-server-url";
+import classNames from "classnames";
+import Popper from "../../../Main/Popper/Popper";
+import { getAppModeEnable } from "../../../../utils/app-mode";
+import Tooltip from "../../../Main/Tooltip/Tooltip";
+import useDeviceDetect from "../../../../hooks/useDeviceDetect";
+import TextField from "../../../Main/TextField/TextField";
+
+const TenantsConfiguration: FC<{accountIds: string[]}> = ({ accountIds }) => {
+  const appModeEnable = getAppModeEnable();
+  const { isMobile } = useDeviceDetect();
+
+  const { tenantId: tenantIdState, serverUrl } = useAppState();
+  const dispatch = useAppDispatch();
+  const timeDispatch = useTimeDispatch();
+
+  const [search, setSearch] = useState("");
+  const [openOptions, setOpenOptions] = useState(false);
+  const optionsButtonRef = useRef<HTMLDivElement>(null);
+
+  const accountIdsFiltered = useMemo(() => {
+    if (!search) return accountIds;
+    try {
+      const regexp = new RegExp(search, "i");
+      const found = accountIds.filter((item) => regexp.test(item));
+      return found.sort((a,b) => (a.match(regexp)?.index || 0) - (b.match(regexp)?.index || 0));
+    } catch (e) {
+      return [];
+    }
+  }, [search, accountIds]);
+
+  const getTenantIdFromUrl = (url: string) => {
+    const regexp = /(\/select\/)(\d+|\d.+)(\/)(.+)/;
+    return (url.match(regexp) || [])[2];
+  };
+
+  const showTenantSelector = useMemo(() => {
+    const id = getTenantIdFromUrl(serverUrl);
+    return accountIds.length > 1 && id;
+  }, [accountIds, serverUrl]);
+
+  const toggleOpenOptions = () => {
+    setOpenOptions(prev => !prev);
+  };
+
+  const handleCloseOptions = () => {
+    setOpenOptions(false);
+  };
+
+  const createHandlerChange = (value: string) => () => {
+    const tenant = value;
+    dispatch({ type: "SET_TENANT_ID", payload: tenant });
+    if (serverUrl) {
+      const updateServerUrl = replaceTenantId(serverUrl, tenant);
+      if (updateServerUrl === serverUrl) return;
+      dispatch({ type: "SET_SERVER", payload: updateServerUrl });
+      timeDispatch({ type: "RUN_QUERY" });
+    }
+    handleCloseOptions();
+  };
+
+  useEffect(() => {
+    const id = getTenantIdFromUrl(serverUrl);
+
+    if (tenantIdState && tenantIdState !== id) {
+      createHandlerChange(tenantIdState)();
+    } else {
+      createHandlerChange(id)();
+    }
+  }, [serverUrl]);
+
+  if (!showTenantSelector) return null;
+
+  return (
+    <div className="vm-tenant-input">
+      <Tooltip title="Define Tenant ID if you need request to another storage">
+        <div ref={optionsButtonRef}>
+          <Button
+            className={appModeEnable ? "" : "vm-header-button"}
+            variant="contained"
+            color="primary"
+            fullWidth
+            startIcon={<StorageIcon/>}
+            endIcon={!isMobile ? (
+              <div
+                className={classNames({
+                  "vm-execution-controls-buttons__arrow": true,
+                  "vm-execution-controls-buttons__arrow_open": openOptions,
+                })}
+              >
+                <ArrowDownIcon/>
+              </div>
+            ) : undefined}
+            onClick={toggleOpenOptions}
+          >
+            {!isMobile && tenantIdState}
+          </Button>
+        </div>
+      </Tooltip>
+      <Popper
+        open={openOptions}
+        placement="bottom-right"
+        onClose={handleCloseOptions}
+        buttonRef={optionsButtonRef}
+      >
+        <div className="vm-list vm-tenant-input-list">
+          <div className="vm-tenant-input-list__search">
+            <TextField
+              autofocus
+              label="Search"
+              value={search}
+              onChange={setSearch}
+            />
+          </div>
+          {accountIdsFiltered.map(id => (
+            <div
+              className={classNames({
+                "vm-list-item": true,
+                "vm-list-item_active": id === tenantIdState
+              })}
+              key={id}
+              onClick={createHandlerChange(id)}
+            >
+              {id}
+            </div>
+          ))}
+        </div>
+      </Popper>
+    </div>
+  );
+};
+
+export default TenantsConfiguration;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/hooks/useFetchAccountIds.ts

@@ -0,0 +1,44 @@
+import { useAppState } from "../../../../../state/common/StateContext";
+import { useEffect, useMemo, useState } from "preact/compat";
+import { ErrorTypes } from "../../../../../types";
+import { getAccountIds } from "../../../../../api/accountId";
+import { getAppModeParams } from "../../../../../utils/app-mode";
+
+export const useFetchAccountIds = () => {
+  const { useTenantID } = getAppModeParams();
+  const { serverUrl } = useAppState();
+
+  const [isLoading, setIsLoading] = useState(false);
+  const [error, setError] = useState<ErrorTypes | string>();
+  const [accountIds, setAccountIds] = useState<string[]>([]);
+
+  const fetchUrl = useMemo(() => getAccountIds(serverUrl), [serverUrl]);
+
+  useEffect(() => {
+    if (!useTenantID) return;
+    const fetchData = async () => {
+      setIsLoading(true);
+      try {
+        const response = await fetch(fetchUrl);
+        const resp = await response.json();
+        const data = (resp.data || []) as string[];
+        setAccountIds(data.sort((a, b) => a.localeCompare(b)));
+
+        if (response.ok) {
+          setError(undefined);
+        } else {
+          setError(`${resp.errorType}\r\n${resp?.error}`);
+        }
+      } catch (e) {
+        if (e instanceof Error) {
+          setError(`${e.name}: ${e.message}`);
+        }
+      }
+      setIsLoading(false);
+    };
+
+    fetchData().catch(console.error);
+  }, [fetchUrl]);
+
+  return { accountIds, isLoading, error };
+};

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/TenantsConfiguration/style.scss

@@ -0,0 +1,19 @@
+@use "../../../../styles/variables" as *;
+
+.vm-tenant-input {
+  position: relative;
+
+  &-list {
+    max-height: 300px;
+    overflow: auto;
+    overscroll-behavior: none;
+    border-radius: $border-radius-medium;
+
+    &__search {
+      position: sticky;
+      top: 0;
+      padding: $padding-small;
+      background-color: $color-background-block;
+    }
+  }
+}

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/Timezones.tsx

@@ -91,6 +91,7 @@ const Timezones: FC<TimezonesProps> = ({ timezoneState, onChange }) => {
         buttonRef={targetRef}
         placement="bottom-left"
         onClose={handleCloseList}
+        fullWidth
       >
         <div className="vm-timezones-list">
           <div className="vm-timezones-list-header">

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/Timezones/style.scss

@@ -23,7 +23,7 @@
       display: inline-flex;
       align-items: center;
       justify-content: center;
-      background-color: rgba($color-black, 0.06);
+      background-color: $color-hover-black;
       padding: calc($padding-small/2);
       border-radius: $border-radius-small;
     }
@@ -46,7 +46,6 @@
   }
 
   &-list {
-    min-width: 600px;
     max-height: 200px;
     background-color: $color-background-block;
     border-radius: $border-radius-medium;

app/vmui/packages/vmui/src/components/Configurators/GlobalSettings/style.scss

@@ -1,16 +1,35 @@
 @use "src/styles/variables" as *;
 
 .vm-server-configurator {
-  display: grid;
+  display: flex;
+  flex-direction: column;
   align-items: center;
   gap: $padding-medium;
   width: 600px;
 
-  &__input {
+  &_mobile {
+    grid-auto-rows: min-content;
+    align-items: flex-start;
+    height: 100%;
+    width: 100%;
+  }
 
+  @media (max-width: 768px) {
+    width: 100%;
+  }
+
+  &__input {
+    width: 100%;
+
+    &_server {
+      display: grid;
+      grid-template-columns: 1fr auto;
+      gap: 0 $padding-small;
+    }
   }
 
   &__title {
+    grid-column: auto / span 2;
     display: flex;
     align-items: center;
     justify-content: flex-start;
@@ -28,4 +47,10 @@
     margin-left: auto;
     margin-right: 0;
   }
+
+  &_mobile &__footer {
+    align-items: flex-end;
+    flex-grow: 1;
+    width: 100%;
+  }
 }

app/vmui/packages/vmui/src/components/Configurators/GraphSettings/GraphSettings.tsx

@@ -57,6 +57,7 @@ const GraphSettings: FC<GraphSettingsProps> = ({ yaxis, setYaxisLimits, toggleEn
             </h3>
             <Button
               size="small"
+              variant="text"
               startIcon={<CloseIcon/>}
               onClick={handleClose}
             />