docs/CHANGELOG.md: cut v1.94.0

See https://alpinelinux.org/posts/Alpine-3.18.4-released.html

.github/workflows/check-licenses.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@main
         with:
-          go-version: 1.20.7
+          go-version: 1.21.1
         id: go
       - name: Code checkout
         uses: actions/checkout@master

.github/workflows/codeql-analysis-js.yml

@@ -33,7 +33,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/codeql-analysis.yml

@@ -52,12 +52,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.7
+          go-version: 1.21.1
           check-latest: true
           cache: true
         if: ${{ matrix.language == 'go' }}

.github/workflows/main.yml

@@ -27,12 +27,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.7
+          go-version: 1.21.1
           check-latest: true
           cache: true
 
@@ -51,12 +51,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.7
+          go-version: 1.21.1
           check-latest: true
           cache: true
 
@@ -75,13 +75,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Go
         id: go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.7
+          go-version: 1.21.1
           check-latest: true
           cache: true
 

.github/workflows/sync-docs.yml

@@ -15,11 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: main
       - name: Checkout private code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: VictoriaMetrics/vmdocs
           token: ${{ secrets.VM_BOT_GH_TOKEN }}

.github/workflows/update-sandbox.yml

@@ -1,80 +0,0 @@
-name: sandbox-release
-on:
-  release:
-    types: [published]
-permissions:
-  contents: write
-jobs:
-  deploy-sandbox:
-    runs-on: ubuntu-latest
-    steps:
-      - name: check inputs
-        if: github.event.release.tag_name == ''
-        run: exit 1
-    
-      - name: Check out code
-        uses: actions/checkout@v3
-        with:
-          repository: VictoriaMetrics/ops
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-
-      - name: Import GPG key
-        id: import-gpg
-        uses: crazy-max/ghaction-import-gpg@v5
-        with:
-          gpg_private_key: ${{ secrets.VM_BOT_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.VM_BOT_PASSPHRASE }}
-          git_user_signingkey: true
-          git_commit_gpgsign: true
-
-      - name: update image tag
-        uses: fjogeleit/yaml-update-action@main
-        with:
-          valueFile: 'gcp-test/sandbox/manifests/benchmark-vm/vmcluster.yaml'
-          commitChange: false
-          createPR: false
-          changes: |
-            {
-              "gcp-test/sandbox/manifests/benchmark-vm/vmcluster.yaml": {
-                "spec.vminsert.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
-                "spec.vmselect.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
-                "spec.vmstorage.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster"
-              },
-              "gcp-test/sandbox/manifests/benchmark-vm/vmsingle.yaml": {
-                "spec.image.tag": "${{ github.event.release.tag_name }}-enterprise"
-              },
-              "gcp-test/sandbox/manifests/monitoring/monitoring-vmagent.yaml": {
-                "spec.image.tag": "${{ github.event.release.tag_name }}"
-              },
-              "gcp-test/sandbox/manifests/monitoring/monitoring-vmcluster.yaml": {
-                "spec.vminsert.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
-                "spec.vmselect.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster",
-                "spec.vmstorage.image.tag": "${{ github.event.release.tag_name }}-enterprise-cluster"
-              },
-              "gcp-test/sandbox/manifests/monitoring/vmalert.yaml": {
-                "spec.image.tag": "${{ github.event.release.tag_name }}-enterprise"
-              }
-            }
-
-      - name: commit changes
-        run: |
-          git config --global user.name "${{ steps.import-gpg.outputs.email }}"
-          git config --global user.email "${{ steps.import-gpg.outputs.email }}"
-          git add .
-          git commit -S -m "Deploy image tag ${RELEASE_TAG} to sandbox"
-        env:
-          RELEASE_TAG: ${{ github.event.release.tag_name }}
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
-        with:
-          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-          branch: release-automation
-          token: ${{ secrets.VM_BOT_GH_TOKEN }}
-          delete-branch: true
-          title: "release ${{ github.event.release.tag_name }}"
-          body: |
-            Release [${{ github.event.release.tag_name }}](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/${{ github.event.release.tag_name }}) to sandbox
-        
-            > Auto-generated by `Github Actions Bot`
-    

Makefile

@@ -16,6 +16,7 @@ GO_BUILDINFO = -X '$(PKG_PREFIX)/lib/buildinfo.Version=$(APP_NAME)-$(DATEINFO_TA
 
 include app/*/Makefile
 include deployment/*/Makefile
+include dashboards/Makefile
 include snap/local/Makefile
 include package/release/Makefile
 
@@ -34,7 +35,6 @@ clean:
 
 publish: package-base \
 	publish-victoria-metrics \
-	publish-victoria-logs \
 	publish-vmagent \
 	publish-vmalert \
 	publish-vmauth \
@@ -174,6 +174,7 @@ vmutils-crossbuild: \
 	vmutils-windows-amd64
 
 publish-release:
+	rm -rf bin/*
 	git checkout $(TAG) && LATEST_TAG=stable $(MAKE) release publish && \
 		git checkout $(TAG)-cluster && LATEST_TAG=cluster-stable $(MAKE) release publish && \
 		git checkout $(TAG)-enterprise && LATEST_TAG=enterprise-stable $(MAKE) release publish && \
@@ -181,7 +182,6 @@ publish-release:
 
 release: \
 	release-victoria-metrics \
-	release-victoria-logs \
 	release-vmutils
 
 release-victoria-metrics: \
@@ -437,7 +437,7 @@ benchmark-pure:
 vendor-update:
 	go get -u -d ./lib/...
 	go get -u -d ./app/...
-	go mod tidy -compat=1.19
+	go mod tidy -compat=1.20
 	go mod vendor
 
 app-local:
@@ -463,7 +463,7 @@ golangci-lint: install-golangci-lint
 	golangci-lint run
 
 install-golangci-lint:
-	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2
+	which golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.54.2
 
 govulncheck: install-govulncheck
 	govulncheck ./...

README.md

@@ -110,6 +110,7 @@ Case studies:
 * [Brandwatch](https://docs.victoriametrics.com/CaseStudies.html#brandwatch)
 * [CERN](https://docs.victoriametrics.com/CaseStudies.html#cern)
 * [COLOPL](https://docs.victoriametrics.com/CaseStudies.html#colopl)
+* [Criteo](https://docs.victoriametrics.com/CaseStudies.html#criteo)
 * [Dig Security](https://docs.victoriametrics.com/CaseStudies.html#dig-security)
 * [Fly.io](https://docs.victoriametrics.com/CaseStudies.html#flyio)
 * [German Research Center for Artificial Intelligence](https://docs.victoriametrics.com/CaseStudies.html#german-research-center-for-artificial-intelligence)
@@ -152,7 +153,7 @@ VictoriaMetrics can also be installed via these installation methods:
 The following command-line flags are used the most:
 
 * `-storageDataPath` - VictoriaMetrics stores all the data in this directory. Default path is `victoria-metrics-data` in the current working directory.
-* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month. The minimum retention period is 24h or 1d. See [the Retention section](#retention) for more details.
+* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 1 month (31 days). The minimum retention period is 24h or 1d. See [these docs](#retention) for more details.
 
 Other flags have good enough default values, so set them only if you really need this. Pass `-help` to see [all the available flags with description and default values](#list-of-command-line-flags).
 
@@ -173,7 +174,8 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c
 
 ### Environment variables
 
-All the VictoriaMetrics components allow referring environment variables in command-line flags via `%{ENV_VAR}` syntax.
+All the VictoriaMetrics components allow referring environment variables in `yaml` configuration files (such as `-promscrape.config`)
+and in command-line flags via `%{ENV_VAR}` syntax.
 For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret`
 if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup.
 This expansion is performed by VictoriaMetrics itself.
@@ -363,6 +365,8 @@ See the [example VMUI at VictoriaMetrics playground](https://play.victoriametric
 * queries with the biggest average execution duration;
 * queries that took the most summary time for execution.
 
+This information is obtained from the `/api/v1/status/top_queries` HTTP endpoint.
+
 ## Active queries
 
 [VMUI](#vmui) provides `active queries` tab, which shows currently execute queries.
@@ -372,6 +376,8 @@ It provides the following information per each query:
 - The duration of the query execution.
 - The client address, who initiated the query execution.
 
+This information is obtained from the `/api/v1/status/active_queries` HTTP endpoint.
+
 ## Metrics explorer
 
 [VMUI](#vmui) provides an ability to explore metrics exported by a particular `job` / `instance` in the following way:
@@ -403,14 +409,16 @@ matching the specified [series selector](https://prometheus.io/docs/prometheus/l
 
 Cardinality explorer is built on top of [/api/v1/status/tsdb](#tsdb-stats).
 
+See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
+See the example of using the cardinality explorer [here](https://victoriametrics.com/blog/cardinality-explorer/).
+
+## Cardinality explorer statistic inaccuracy
+
 In [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) each vmstorage tracks the stored time series individually.
 vmselect requests stats via [/api/v1/status/tsdb](#tsdb-stats) API from each vmstorage node and merges the results by summing per-series stats.
 This may lead to inflated values when samples for the same time series are spread across multiple vmstorage nodes
 due to [replication](#replication) or [rerouting](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html?highlight=re-routes#cluster-availability).
 
-See [cardinality explorer playground](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/prometheus/graph/#/cardinality).
-See the example of using the cardinality explorer [here](https://victoriametrics.com/blog/cardinality-explorer/).
-
 ## How to apply new config to VictoriaMetrics
 
 VictoriaMetrics is configured via command-line flags, so it must be restarted when new command-line flags should be applied:
@@ -615,6 +623,28 @@ Some plugins for Telegraf such as [fluentd](https://github.com/fangli/fluent-plu
 or [Juniper/jitmon](https://github.com/Juniper/jtimon) send `SHOW DATABASES` query to `/query` and expect a particular database name in the response.
 Comma-separated list of expected databases can be passed to VictoriaMetrics via `-influx.databaseNames` command-line flag.
 
+### How to send data in InfluxDB v2 format
+
+VictoriaMetrics exposes endpoint for InfluxDB v2 HTTP API at `/influx/api/v2/write` and `/api/v2/write`.
+
+
+In order to write data with InfluxDB line protocol to local VictoriaMetrics using `curl`:
+
+<div class="with-copy" markdown="1">
+
+```console
+curl -d 'measurement,tag1=value1,tag2=value2 field1=123,field2=1.23' -X POST 'http://localhost:8428/api/v2/write'
+```
+
+</div>
+
+The `/api/v1/export` endpoint should return the following response:
+
+```json
+{"metric":{"__name__":"measurement_field1","tag1":"value1","tag2":"value2"},"values":[123],"timestamps":[1695902762311]}
+{"metric":{"__name__":"measurement_field2","tag1":"value1","tag2":"value2"},"values":[1.23],"timestamps":[1695902762311]}
+```
+
 ## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
 
 Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,
@@ -829,7 +859,7 @@ Additionally, VictoriaMetrics provides the following handlers:
 * `/api/v1/series/count` - returns the total number of time series in the database. Some notes:
   * the handler scans all the inverted index, so it can be slow if the database contains tens of millions of time series;
   * the handler may count [deleted time series](#how-to-delete-time-series) additionally to normal time series due to internal implementation restrictions;
-* `/api/v1/status/active_queries` - returns a list of currently running queries.
+* `/api/v1/status/active_queries` - returns the list of currently running queries. This list is also available at [`active queries` page at VMUI](#active-queries).
 * `/api/v1/status/top_queries` - returns the following query lists:
   * the most frequently executed queries - `topByCount`
   * queries with the biggest average execution duration - `topByAvgDuration`
@@ -839,6 +869,8 @@ Additionally, VictoriaMetrics provides the following handlers:
   For example, request to `/api/v1/status/top_queries?topN=5&maxLifetime=30s` would return up to 5 queries per list, which were executed during the last 30 seconds.
   VictoriaMetrics tracks the last `-search.queryStats.lastQueriesCount` queries with durations at least `-search.queryStats.minQueryDuration`.
 
+  See also [`top queries` page at VMUI](#top-queries).
+
 ### Timestamp formats
 
 VictoriaMetrics accepts the following formats for `time`, `start` and `end` query args
@@ -914,7 +946,7 @@ to your needs or when testing bugfixes.
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make victoria-metrics` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics` binary and puts it into the `bin` folder.
 
@@ -930,7 +962,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make victoria-metrics-linux-arm` or `make victoria-metrics-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-linux-arm` or `victoria-metrics-linux-arm64` binary respectively and puts it into the `bin` folder.
 
@@ -944,7 +976,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 `Pure Go` mode builds only Go code without [cgo](https://golang.org/cmd/cgo/) dependencies.
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make victoria-metrics-pure` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `victoria-metrics-pure` binary and puts it into the `bin` folder.
 
@@ -1609,8 +1641,8 @@ See also [how to work with snapshots](#how-to-work-with-snapshots).
 ## Retention
 
 Retention is configured with the `-retentionPeriod` command-line flag, which takes a number followed by a time unit 
-character - `h(ours)`, `d(ays)`, `w(eeks)`, `y(ears)`. If the time unit is not specified, a month is assumed. 
-For instance, `-retentionPeriod=3` means that the data will be stored for 3 months and then deleted. 
+character - `h(ours)`, `d(ays)`, `w(eeks)`, `y(ears)`. If the time unit is not specified, a month (31 days) is assumed.
+For instance, `-retentionPeriod=3` means that the data will be stored for 3 months (93 days) and then deleted.
 The default retention period is one month. The **minimum retention** period is 24h or 1d.
 
 Data is split in per-month partitions inside `<-storageDataPath>/data/{small,big}` folders.
@@ -1653,9 +1685,10 @@ See [these docs](https://docs.victoriametrics.com/guides/guide-vmcluster-multipl
 which allow configuring multiple retentions for distinct sets of time series matching the configured [series filters](https://docs.victoriametrics.com/keyConcepts.html#filtering)
 via `-retentionFilter` command-line flag. This flag accepts `filter:duration` options, where `filter` must be
 a valid [series filter](https://docs.victoriametrics.com/keyConcepts.html#filtering), while the `duration`
-must contain valid [retention](#retention) for time series matching the given `filter`. If series doesn't match
-any configured `-retentionFilter`, then the retention configured via [-retentionPeriod](#retention) command-line flag is applied to it.
-If series matches multiple configured retention filters, then the smallest retention is applied.
+must contain valid [retention](#retention) for time series matching the given `filter`. 
+The `duration` of the `-retentionFilter` must be lower or equal to [-retentionPeriod](#retention) flag value.
+If series doesn't match any configured `-retentionFilter`, then the retention configured via [-retentionPeriod](#retention) 
+command-line flag is applied to it. If series matches multiple configured retention filters, then the smallest retention is applied.
 
 For example, the following config sets 3 days retention for time series with `team="juniors"` label,
 30 days retention for time series with `env="dev"` or `env="staging"` label and 1 year retention for the remaining time series:
@@ -1693,6 +1726,10 @@ Downsampling happens during [background merges](https://docs.victoriametrics.com
 and can't be performed if there is not enough of free disk space or if vmstorage 
 is in [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode).
 
+Please, note that intervals of `-downsampling.period` must be multiples of each other. 
+In case [deduplication](https://docs.victoriametrics.com/#deduplication) is enabled value of `-dedup.minScrapeInterval` must also be multiple of `-downsampling.period` intervals.
+This is required to ensure consistency of deduplication and downsampling results.
+
 The downsampling can be evaluated for free by downloading and using enterprise binaries from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
 
 ## Multi-tenancy
@@ -1785,9 +1822,9 @@ Graphs on the dashboards contain useful hints - hover the `i` icon in the top le
 We recommend setting up [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts)
 via [vmalert](https://docs.victoriametrics.com/vmalert.html) or via Prometheus.
 
-VictoriaMetrics exposes currently running queries and their execution times at `/api/v1/status/active_queries` page.
+VictoriaMetrics exposes currently running queries and their execution times at [`active queries` page](#active-queries).
 
-VictoriaMetrics exposes queries, which take the most time to execute, at `/api/v1/status/top_queries` page.
+VictoriaMetrics exposes queries, which take the most time to execute, at [`top queries` page](#top-queries).
 
 See also [VictoriaMetrics Monitoring](https://victoriametrics.com/blog/victoriametrics-monitoring/)
 and [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html).
@@ -1932,9 +1969,6 @@ and [cardinality explorer docs](#cardinality-explorer).
   has at least 20% of free space. The remaining amount of free space
   can be [monitored](#monitoring) via `vm_free_disk_space_bytes` metric. The total size of data
   stored on the disk can be monitored via sum of `vm_data_size_bytes` metrics.
-  See also `vm_merge_need_free_disk_space` metrics, which are set to values higher than 0
-  if background merge cannot be initiated due to free disk space shortage. The value shows the number of per-month partitions,
-  which would start background merge if they had more free disk space.
 
 * VictoriaMetrics buffers incoming data in memory for up to a few seconds before flushing it to persistent storage.
   This may lead to the following "issues":
@@ -2293,7 +2327,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -finalMergeDelay duration
      The delay before starting final merge for per-month partition after no new data is ingested into it. Final merge may require additional disk IO and CPU resources. Final merge may increase query speed and reduce disk space usage in some cases. Zero value disables final merge
   -flagsAuthKey string
@@ -2363,6 +2397,12 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -logNewSeries
      Whether to log new series. This option is for debug purposes only. It can lead to performance issues when big number of new series are ingested into VictoriaMetrics
   -loggerDisableTimestamps

SECURITY.md

@@ -5,8 +5,8 @@
 | Version | Supported          |
 |---------|--------------------|
 | [latest release](https://docs.victoriametrics.com/CHANGELOG.html) | :white_check_mark: |
+| v1.93.x LTS release | :white_check_mark: |
 | v1.87.x LTS release | :white_check_mark: |
-| v1.79.x LTS release | :white_check_mark: |
 | other releases  | :x:                |
 
 ## Reporting a Vulnerability

app/vlinsert/elasticsearch/elasticsearch.go

@@ -3,6 +3,7 @@ package elasticsearch
 import (
 	"bufio"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"math"
@@ -11,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
@@ -21,7 +24,10 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
-	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	elasticsearchVersion = flag.String("elasticsearch.version", "8.9.0", "Elasticsearch version to report to client")
 )
 
 // RequestHandler processes Elasticsearch insert requests
@@ -60,9 +66,9 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			// See the latest available version for Elasticsearch at https://github.com/elastic/elasticsearch/releases
 			fmt.Fprintf(w, `{
 			"version": {
-				"number": "8.8.0"
+				"number": %q
 			}
-		}`)
+		}`, *elasticsearchVersion)
 		case http.MethodHead:
 			// Return empty response for Logstash ping request.
 		}
@@ -88,22 +94,32 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
+		if err := vlstorage.CanWriteData(); err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
 		processLogMessage := cp.GetProcessLogMessageFunc(lr)
 		isGzip := r.Header.Get("Content-Encoding") == "gzip"
 		n, err := readBulkRequest(r.Body, isGzip, cp.TimeField, cp.MsgField, processLogMessage)
+		vlstorage.MustAddRows(lr)
+		logstorage.PutLogRows(lr)
 		if err != nil {
 			logger.Warnf("cannot decode log message #%d in /_bulk request: %s", n, err)
 			return true
 		}
-		vlstorage.MustAddRows(lr)
-		logstorage.PutLogRows(lr)
 
 		tookMs := time.Since(startTime).Milliseconds()
 		bw := bufferedwriter.Get(w)
 		defer bufferedwriter.Put(bw)
 		WriteBulkResponse(bw, n, tookMs)
 		_ = bw.Flush()
+
+		// update bulkRequestDuration only for successfully parsed requests
+		// There is no need in updating bulkRequestDuration for request errors,
+		// since their timings are usually much smaller than the timing for successful request parsing.
+		bulkRequestDuration.UpdateDuration(startTime)
+
 		return true
 	default:
 		return false
@@ -111,7 +127,9 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
 }
 
 var (
-	bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
+	bulkRequestsTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
+	rowsIngestedTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
+	bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
 )
 
 func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
@@ -157,8 +175,6 @@ func readBulkRequest(r io.Reader, isGzip bool, timeField, msgField string,
 
 var lineBufferPool bytesutil.ByteBufferPool
 
-var rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="elasticsearch_bulk"}`)
-
 func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
 	processLogMessage func(timestamp int64, fields []logstorage.Field),
 ) (bool, error) {
@@ -209,6 +225,7 @@ func readBulkLine(sc *bufio.Scanner, timeField, msgField string,
 	p.RenameField(msgField, "_msg")
 	processLogMessage(ts, p.Fields)
 	logjson.PutParser(p)
+
 	return true, nil
 }
 

app/vlinsert/insertutils/common_params.go

@@ -3,12 +3,13 @@ package insertutils
 import (
 	"net/http"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 // CommonParams contains common HTTP parameters used by log ingestion APIs.
@@ -73,12 +74,19 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
 // GetProcessLogMessageFunc returns a function, which adds parsed log messages to lr.
 func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(timestamp int64, fields []logstorage.Field) {
 	return func(timestamp int64, fields []logstorage.Field) {
+		if len(fields) > *MaxFieldsPerLine {
+			rf := logstorage.RowFormatter(fields)
+			logger.Warnf("dropping log line with %d fields; it exceeds -insert.maxFieldsPerLine=%d; %s", len(fields), *MaxFieldsPerLine, rf)
+			rowsDroppedTotalTooManyFields.Inc()
+			return
+		}
+
 		lr.MustAdd(cp.TenantID, timestamp, fields)
 		if cp.Debug {
 			s := lr.GetRowString(0)
 			lr.ResetKeepSettings()
 			logger.Infof("remoteAddr=%s; requestURI=%s; ignoring log entry because of `debug` query arg: %s", cp.DebugRemoteAddr, cp.DebugRequestURI, s)
-			rowsDroppedTotal.Inc()
+			rowsDroppedTotalDebug.Inc()
 			return
 		}
 		if lr.NeedFlush() {
@@ -88,4 +96,5 @@ func (cp *CommonParams) GetProcessLogMessageFunc(lr *logstorage.LogRows) func(ti
 	}
 }
 
-var rowsDroppedTotal = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+var rowsDroppedTotalDebug = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
+var rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)

app/vlinsert/insertutils/flags.go

@@ -1,10 +1,15 @@
 package insertutils
 
 import (
+	"flag"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 var (
 	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
 	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")
+
+	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
+	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
 )

app/vlinsert/jsonline/jsonline.go

@@ -21,6 +21,7 @@ import (
 
 // RequestHandler processes jsonline insert requests
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
+	startTime := time.Now()
 	w.Header().Add("Content-Type", "application/json")
 
 	if r.Method != "POST" {
@@ -35,6 +36,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		httpserver.Errorf(w, r, "%s", err)
 		return true
 	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
+	}
 	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
 	processLogMessage := cp.GetProcessLogMessageFunc(lr)
 
@@ -77,6 +82,11 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	vlstorage.MustAddRows(lr)
 	logstorage.PutLogRows(lr)
 
+	// update jsonlineRequestDuration only for successfully parsed requests.
+	// There is no need in updating jsonlineRequestDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	jsonlineRequestDuration.UpdateDuration(startTime)
+
 	return true
 }
 
@@ -109,6 +119,7 @@ func readLine(sc *bufio.Scanner, timeField, msgField string, processLogMessage f
 	p.RenameField(msgField, "_msg")
 	processLogMessage(ts, p.Fields)
 	logjson.PutParser(p)
+
 	return true, nil
 }
 
@@ -144,6 +155,7 @@ func parseISO8601Timestamp(s string) (int64, error) {
 var lineBufferPool bytesutil.ByteBufferPool
 
 var (
-	requestsTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
-	rowsIngestedTotal = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+	requestsTotal           = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
+	rowsIngestedTotal       = metrics.NewCounter(`vl_rows_ingested_total{type="jsonline"}`)
+	jsonlineRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
 )

app/vlinsert/loki/loki.go

@@ -5,29 +5,31 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/metrics"
-)
-
-var (
-	lokiRequestsJSONTotal     = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
-	lokiRequestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
 )
 
 // RequestHandler processes Loki insert requests
-//
-// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
 func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
-	if path != "/api/v1/push" {
+	switch path {
+	case "/api/v1/push":
+		return handleInsert(r, w)
+	case "/ready":
+		// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ready"))
+		return true
+	default:
 		return false
 	}
+}
+
+// See https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
+func handleInsert(r *http.Request, w http.ResponseWriter) bool {
 	contentType := r.Header.Get("Content-Type")
 	switch contentType {
 	case "application/json":
-		lokiRequestsJSONTotal.Inc()
 		return handleJSON(r, w)
 	default:
-		// Protobuf request body should be handled by default accoring to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
-		lokiRequestsProtobufTotal.Inc()
+		// Protobuf request body should be handled by default according to https://grafana.com/docs/loki/latest/api/#push-log-entries-to-loki
 		return handleProtobuf(r, w)
 	}
 }

app/vlinsert/loki/loki_json.go

@@ -18,12 +18,11 @@ import (
 	"github.com/valyala/fastjson"
 )
 
-var (
-	rowsIngestedJSONTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
-	parserPool            fastjson.ParserPool
-)
+var parserPool fastjson.ParserPool
 
 func handleJSON(r *http.Request, w http.ResponseWriter) bool {
+	startTime := time.Now()
+	lokiRequestsJSONTotal.Inc()
 	reader := r.Body
 	if r.Header.Get("Content-Encoding") == "gzip" {
 		zr, err := common.GetGzipReader(reader)
@@ -48,19 +47,36 @@ func handleJSON(r *http.Request, w http.ResponseWriter) bool {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return true
 	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
+	}
 	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
 	processLogMessage := cp.GetProcessLogMessageFunc(lr)
 	n, err := parseJSONRequest(data, processLogMessage)
 	vlstorage.MustAddRows(lr)
 	logstorage.PutLogRows(lr)
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse Loki request: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki json request: %s", err)
 		return true
 	}
+
 	rowsIngestedJSONTotal.Add(n)
+
+	// update lokiRequestJSONDuration only for successfully parsed requests
+	// There is no need in updating lokiRequestJSONDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	lokiRequestJSONDuration.UpdateDuration(startTime)
+
 	return true
 }
 
+var (
+	lokiRequestsJSONTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
+	rowsIngestedJSONTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="json"}`)
+	lokiRequestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
+)
+
 func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
 	p := parserPool.Get()
 	defer parserPool.Put(p)
@@ -155,7 +171,6 @@ func parseJSONRequest(data []byte, processLogMessage func(timestamp int64, field
 				Value: bytesutil.ToUnsafeString(msg),
 			})
 			processLogMessage(ts, fields)
-
 		}
 		rowsIngested += len(lines)
 	}

app/vlinsert/loki/loki_protobuf.go

@@ -19,12 +19,13 @@ import (
 )
 
 var (
-	rowsIngestedProtobufTotal = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
-	bytesBufPool              bytesutil.ByteBufferPool
-	pushReqsPool              sync.Pool
+	bytesBufPool bytesutil.ByteBufferPool
+	pushReqsPool sync.Pool
 )
 
 func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
+	startTime := time.Now()
+	lokiRequestsProtobufTotal.Inc()
 	wcr := writeconcurrencylimiter.GetReader(r.Body)
 	data, err := io.ReadAll(wcr)
 	writeconcurrencylimiter.PutReader(wcr)
@@ -38,19 +39,36 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) bool {
 		httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
 		return true
 	}
+	if err := vlstorage.CanWriteData(); err != nil {
+		httpserver.Errorf(w, r, "%s", err)
+		return true
+	}
 	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields)
 	processLogMessage := cp.GetProcessLogMessageFunc(lr)
 	n, err := parseProtobufRequest(data, processLogMessage)
 	vlstorage.MustAddRows(lr)
 	logstorage.PutLogRows(lr)
 	if err != nil {
-		httpserver.Errorf(w, r, "cannot parse loki request: %s", err)
+		httpserver.Errorf(w, r, "cannot parse Loki protobuf request: %s", err)
 		return true
 	}
+
 	rowsIngestedProtobufTotal.Add(n)
+
+	// update lokiRequestProtobufDuration only for successfully parsed requests
+	// There is no need in updating lokiRequestProtobufDuration for request errors,
+	// since their timings are usually much smaller than the timing for successful request parsing.
+	lokiRequestProtobufDuration.UpdateDuration(startTime)
+
 	return true
 }
 
+var (
+	lokiRequestsProtobufTotal   = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
+	rowsIngestedProtobufTotal   = metrics.NewCounter(`vl_rows_ingested_total{type="loki",format="protobuf"}`)
+	lokiRequestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
+)
+
 func parseProtobufRequest(data []byte, processLogMessage func(timestamp int64, fields []logstorage.Field)) (int, error) {
 	bb := bytesBufPool.Get()
 	defer bytesBufPool.Put(bb)

app/vlinsert/loki/loki_protobuf_timing_test.go

@@ -6,8 +6,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/golang/snappy"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 )
 
 func BenchmarkParseProtobufRequest(b *testing.B) {

app/vlselect/vmui/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
-    "main.css": "./static/css/main.5f461a27.css",
-    "main.js": "./static/js/main.7566144a.js",
+    "main.css": "./static/css/main.3258b6c4.css",
+    "main.js": "./static/js/main.41b816cc.js",
     "static/js/522.b5ae4365.chunk.js": "./static/js/522.b5ae4365.chunk.js",
     "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
     "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.5f461a27.css",
-    "static/js/main.7566144a.js"
+    "static/css/main.3258b6c4.css",
+    "static/js/main.41b816cc.js"
   ]
 }

app/vlselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.7566144a.js"></script><link href="./static/css/main.5f461a27.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.41b816cc.js"></script><link href="./static/css/main.3258b6c4.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vlstorage/main.go

@@ -3,14 +3,17 @@ package vlstorage
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -29,6 +32,8 @@ var (
 		"see https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#stream-fields ; see also -logIngestedRows")
 	logIngestedRows = flag.Bool("logIngestedRows", false, "Whether to log all the ingested log entries; this can be useful for debugging of data ingestion; "+
 		"see https://docs.victoriametrics.com/VictoriaLogs/data-ingestion/ ; see also -logNewStreams")
+	minFreeDiskSpaceBytes = flagutil.NewBytes("storage.minFreeDiskSpaceBytes", 10e6, "The minimum free disk space at -storageDataPath after which "+
+		"the storage stops accepting new data")
 )
 
 // Init initializes vlstorage.
@@ -39,15 +44,16 @@ func Init() {
 		logger.Panicf("BUG: Init() has been already called")
 	}
 
-	if retentionPeriod.Msecs < 24*3600*1000 {
+	if retentionPeriod.Duration() < 24*time.Hour {
 		logger.Fatalf("-retentionPeriod cannot be smaller than a day; got %s", retentionPeriod)
 	}
 	cfg := &logstorage.StorageConfig{
-		Retention:       time.Millisecond * time.Duration(retentionPeriod.Msecs),
-		FlushInterval:   *inmemoryDataFlushInterval,
-		FutureRetention: time.Millisecond * time.Duration(futureRetention.Msecs),
-		LogNewStreams:   *logNewStreams,
-		LogIngestedRows: *logIngestedRows,
+		Retention:             retentionPeriod.Duration(),
+		FlushInterval:         *inmemoryDataFlushInterval,
+		FutureRetention:       futureRetention.Duration(),
+		LogNewStreams:         *logNewStreams,
+		LogIngestedRows:       *logIngestedRows,
+		MinFreeDiskSpaceBytes: minFreeDiskSpaceBytes.N,
 	}
 	logger.Infof("opening storage at -storageDataPath=%s", *storageDataPath)
 	startTime := time.Now()
@@ -74,7 +80,21 @@ func Stop() {
 var strg *logstorage.Storage
 var storageMetrics *metrics.Set
 
+// CanWriteData returns non-nil error if it cannot write data to vlstorage.
+func CanWriteData() error {
+	if strg.IsReadOnly() {
+		return &httpserver.ErrorWithStatusCode{
+			Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
+				"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
+			StatusCode: http.StatusTooManyRequests,
+		}
+	}
+	return nil
+}
+
 // MustAddRows adds lr to vlstorage
+//
+// It is advised to call CanWriteData() before calling MustAddRows()
 func MustAddRows(lr *logstorage.LogRows) {
 	strg.MustAddRows(lr)
 }
@@ -107,6 +127,12 @@ func initStorageMetrics(strg *logstorage.Storage) *metrics.Set {
 	ms.NewGauge(fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), func() float64 {
 		return float64(fs.MustGetFreeSpace(*storageDataPath))
 	})
+	ms.NewGauge(fmt.Sprintf(`vl_storage_is_read_only{path=%q}`, *storageDataPath), func() float64 {
+		if m().IsReadOnly {
+			return 1
+		}
+		return 0
+	})
 
 	ms.NewGauge(`vl_active_merges{type="inmemory"}`, func() float64 {
 		return float64(m().InmemoryActiveMerges)

app/vmagent/README.md

@@ -3,7 +3,8 @@
 `vmagent` is a tiny agent which helps you collect metrics from various sources,
 [relabel and filter the collected metrics](#relabeling)
 and store them in [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics)
-or any other storage systems via Prometheus `remote_write` protocol.
+or any other storage systems via Prometheus `remote_write` protocol
+or via [VictoriaMetrics `remote_write` protocol](#victoriametrics-remote-write-protocol).
 
 See [Quick Start](#quick-start) for details.
 
@@ -754,10 +755,10 @@ as soon as it is parsed in stream parsing mode.
 
 A single `vmagent` instance can scrape tens of thousands of scrape targets. Sometimes this isn't enough due to limitations on CPU, network, RAM, etc.
 In this case scrape targets can be split among multiple `vmagent` instances (aka `vmagent` horizontal scaling, sharding and clustering).
-Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values.
-The flag value must be in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster.
-The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag. For example, the following commands
-spread scrape targets among a cluster of two `vmagent` instances:
+The number of `vmagent` instances in the cluster must be passed to `-promscrape.cluster.membersCount` command-line flag.
+Each `vmagent` instance in the cluster must use identical `-promscrape.config` files with distinct `-promscrape.cluster.memberNum` values
+in the range `0 ... N-1`, where `N` is the number of `vmagent` instances in the cluster specified via `-promscrape.cluster.membersCount`.
+For example, the following commands spread scrape targets among a cluster of two `vmagent` instances:
 
 ```
 /path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=0 -promscrape.config=/path/to/config.yml ...
@@ -765,7 +766,7 @@ spread scrape targets among a cluster of two `vmagent` instances:
 ```
 
 The `-promscrape.cluster.memberNum` can be set to a StatefulSet pod name when `vmagent` runs in Kubernetes.
-The pod name must end with a number in the range `0 ... promscrape.cluster.memberNum-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
+The pod name must end with a number in the range `0 ... promscrape.cluster.membersCount-1`. For example, `-promscrape.cluster.memberNum=vmagent-0`.
 
 By default, each scrape target is scraped only by a single `vmagent` instance in the cluster. If there is a need for replicating scrape targets among multiple `vmagent` instances,
 then `-promscrape.cluster.replicationFactor` command-line flag must be set to the desired number of replicas. For example, the following commands
@@ -781,6 +782,14 @@ If each target is scraped by multiple `vmagent` instances, then data deduplicati
 The `-dedup.minScrapeInterval` must be set to the `scrape_interval` configured at `-promscrape.config`.
 See [these docs](https://docs.victoriametrics.com/#deduplication) for details.
 
+The `-promscrape.cluster.memberLabel` command-line flag allows specifying a name for `member num` label to add to all the scraped metrics.
+The value of the `member num` label is set to `-promscrape.cluster.memberNum`. For example, the following config instructs adding `vmagent_instance="0"` label
+to all the metrics scraped by the given `vmagent` instance:
+
+```
+/path/to/vmagent -promscrape.cluster.membersCount=2 -promscrape.cluster.memberNum=0 -promscrape.cluster.memberLabel=vmagent_instance
+```
+
 See also [how to shard data among multiple remote storage systems](#sharding-among-remote-storages).
 
 
@@ -1130,7 +1139,7 @@ It may be needed to build `vmagent` from source code when developing or testing
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmagent` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds the `vmagent` binary and puts it into the `bin` folder.
 
@@ -1159,7 +1168,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmagent-linux-arm` or `make vmagent-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics)
    It builds `vmagent-linux-arm` or `vmagent-linux-arm64` binary respectively and puts it into the `bin` folder.
 
@@ -1236,7 +1245,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -flagsAuthKey string
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
@@ -1327,6 +1336,12 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
   -kafka.consumer.topic.options array
      Optional key=value;key1=value2 settings for topic consumer. See full configuration options at https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
      Supports an array of values separated by comma or specified via multiple flags.
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -1376,8 +1391,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
      Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
   -promscrape.azureSDCheckInterval duration
      Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#azure_sd_configs for details (default 1m0s)
+  -promscrape.cluster.memberLabel string
+     If non-empty, then the label with this name and the -promscrape.cluster.memberNum value is added to all the scraped metrics
   -promscrape.cluster.memberNum string
-     The number of number in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name (default "0")
+     The number of vmagent instance in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name. See also -promscrape.cluster.memberLabel (default "0")
   -promscrape.cluster.membersCount int
      The number of members in a cluster of scrapers. Each member must have a unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default, cluster scraping is disabled, i.e. a single scraper scrapes all the targets
   -promscrape.cluster.name string

app/vmagent/main.go

@@ -11,6 +11,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/csvimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/datadog"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/graphite"
@@ -39,7 +41,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -208,7 +209,7 @@ func getAuthTokenFromPath(path string) (*auth.Token, error) {
 	if p.Suffix != "opentsdb/api/put" {
 		return nil, fmt.Errorf("unsupported path requested: %q; expecting 'opentsdb/api/put'", p.Suffix)
 	}
-	return auth.NewToken(p.AuthToken)
+	return auth.NewTokenPossibleMultitenant(p.AuthToken)
 }
 
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
@@ -251,7 +252,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(statusCode)
 		return true
 	}
-	if strings.HasPrefix(path, "datadog/") {
+	if strings.HasPrefix(path, "/datadog/") {
 		// Trim suffix from paths starting from /datadog/ in order to support legacy DataDog agent.
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2670
 		path = strings.TrimSuffix(path, "/")

app/vmagent/opentelemetry/request_handler.go

@@ -1,6 +1,7 @@
 package opentelemetry
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/common"
@@ -26,6 +27,9 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	if req.Header.Get("Content-Type") == "application/json" {
+		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+	}
 	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(at, tss, extraLabels)
 	})

app/vmagent/prometheusimport/request_handler.go

@@ -32,7 +32,7 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
 		return insertRows(at, rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vmagent/remotewrite/client.go

@@ -2,6 +2,7 @@ package remotewrite
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -26,10 +27,10 @@ var (
 	forceVMProto = flagutil.NewArrayBool("remoteWrite.forceVMProto", "Whether to force VictoriaMetrics remote write protocol for sending data "+
 		"to the corresponding -remoteWrite.url . See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol")
 
-	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
+	rateLimit = flagutil.NewArrayInt("remoteWrite.rateLimit", 0, "Optional rate limit in bytes per second for data sent to the corresponding -remoteWrite.url. "+
 		"By default, the rate limit is disabled. It can be useful for limiting load on remote storage when big amounts of buffered data "+
 		"is sent after temporary unavailability of the remote storage")
-	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", "Timeout for sending a single block of data to the corresponding -remoteWrite.url (default 1m)")
+	sendTimeout = flagutil.NewArrayDuration("remoteWrite.sendTimeout", time.Minute, "Timeout for sending a single block of data to the corresponding -remoteWrite.url")
 	proxyURL    = flagutil.NewArrayString("remoteWrite.proxyURL", "Optional proxy URL for writing data to the corresponding -remoteWrite.url. "+
 		"Supported proxies: http, https, socks5. Example: -remoteWrite.proxyURL=socks5://proxy:1234")
 
@@ -134,7 +135,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 	}
 	hc := &http.Client{
 		Transport: tr,
-		Timeout:   sendTimeout.GetOptionalArgOrDefault(argIdx, time.Minute),
+		Timeout:   sendTimeout.GetOptionalArg(argIdx),
 	}
 	c := &client{
 		sanitizedURL:   sanitizedURL,
@@ -169,7 +170,7 @@ func newHTTPClient(argIdx int, remoteWriteURL, sanitizedURL string, fq *persiste
 }
 
 func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
-	if bytesPerSec := rateLimit.GetOptionalArgOrDefault(argIdx, 0); bytesPerSec > 0 {
+	if bytesPerSec := rateLimit.GetOptionalArg(argIdx); bytesPerSec > 0 {
 		logger.Infof("applying %d bytes per second rate limit for -remoteWrite.url=%q", bytesPerSec, sanitizedURL)
 		c.rl.perSecondLimit = int64(bytesPerSec)
 	}
@@ -178,7 +179,7 @@ func (c *client) init(argIdx, concurrency int, sanitizedURL string) {
 	c.bytesSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_bytes_sent_total{url=%q}`, c.sanitizedURL))
 	c.blocksSent = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_blocks_sent_total{url=%q}`, c.sanitizedURL))
 	c.rateLimit = metrics.GetOrCreateGauge(fmt.Sprintf(`vmagent_remotewrite_rate_limit{url=%q}`, c.sanitizedURL), func() float64 {
-		return float64(rateLimit.GetOptionalArgOrDefault(argIdx, 0))
+		return float64(rateLimit.GetOptionalArg(argIdx))
 	})
 	c.requestDuration = metrics.GetOrCreateHistogram(fmt.Sprintf(`vmagent_remotewrite_duration_seconds{url=%q}`, c.sanitizedURL))
 	c.requestsOKCount = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_requests_total{url=%q, status_code="2XX"}`, c.sanitizedURL))
@@ -322,6 +323,20 @@ func (c *client) runWorker() {
 }
 
 func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
+	req := c.newRequest(url, body)
+	resp, err := c.hc.Do(req)
+	if err != nil && errors.Is(err, io.EOF) {
+		// it is likely connection became stale.
+		// So we do one more attempt in hope request will succeed.
+		// If not, the error should be handled by the caller as usual.
+		// This should help with https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4139
+		req = c.newRequest(url, body)
+		resp, err = c.hc.Do(req)
+	}
+	return resp, err
+}
+
+func (c *client) newRequest(url string, body []byte) *http.Request {
 	reqBody := bytes.NewBuffer(body)
 	req, err := http.NewRequest(http.MethodPost, url, reqBody)
 	if err != nil {
@@ -345,7 +360,7 @@ func (c *client) doRequest(url string, body []byte) (*http.Response, error) {
 			logger.Warnf("cannot sign remoteWrite request with AWS sigv4: %s", err)
 		}
 	}
-	return c.hc.Do(req)
+	return req
 }
 
 // sendBlockHTTP sends the given block to c.remoteWriteURL.

app/vmagent/remotewrite/relabel.go

@@ -87,8 +87,8 @@ func initLabelsGlobal() {
 	}
 }
 
-func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
-	if len(extraLabels) == 0 && pcs.Len() == 0 && !*usePromCompatibleNaming {
+func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, pcs *promrelabel.ParsedConfigs) []prompbmarshal.TimeSeries {
+	if pcs.Len() == 0 && !*usePromCompatibleNaming {
 		// Nothing to change.
 		return tss
 	}
@@ -98,34 +98,15 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 		ts := &tss[i]
 		labelsLen := len(labels)
 		labels = append(labels, ts.Labels...)
-		// extraLabels must be added before applying relabeling according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write
-		for j := range extraLabels {
-			extraLabel := &extraLabels[j]
-			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
-			if tmp != nil {
-				tmp.Value = extraLabel.Value
-			} else {
-				labels = append(labels, *extraLabel)
-			}
-		}
-		if *usePromCompatibleNaming {
-			// Replace unsupported Prometheus chars in label names and metric names with underscores.
-			tmpLabels := labels[labelsLen:]
-			for j := range tmpLabels {
-				label := &tmpLabels[j]
-				if label.Name == "__name__" {
-					label.Value = promrelabel.SanitizeName(label.Value)
-				} else {
-					label.Name = promrelabel.SanitizeName(label.Name)
-				}
-			}
-		}
 		labels = pcs.Apply(labels, labelsLen)
 		labels = promrelabel.FinalizeLabels(labels[:labelsLen], labels[labelsLen:])
 		if len(labels) == labelsLen {
 			// Drop the current time series, since relabeling removed all the labels.
 			continue
 		}
+		if *usePromCompatibleNaming {
+			fixPromCompatibleNaming(labels[labelsLen:])
+		}
 		tssDst = append(tssDst, prompbmarshal.TimeSeries{
 			Labels:  labels[labelsLen:],
 			Samples: ts.Samples,
@@ -135,6 +116,29 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab
 	return tssDst
 }
 
+func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraLabels []prompbmarshal.Label) {
+	if len(extraLabels) == 0 {
+		return
+	}
+	labels := rctx.labels[:0]
+	for i := range tss {
+		ts := &tss[i]
+		labelsLen := len(labels)
+		labels = append(labels, ts.Labels...)
+		for j := range extraLabels {
+			extraLabel := extraLabels[j]
+			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
+			if tmp != nil {
+				tmp.Value = extraLabel.Value
+			} else {
+				labels = append(labels, extraLabel)
+			}
+		}
+		ts.Labels = labels[labelsLen:]
+	}
+	rctx.labels = labels
+}
+
 type relabelCtx struct {
 	// pool for labels, which are used during the relabeling.
 	labels []prompbmarshal.Label
@@ -159,3 +163,15 @@ func putRelabelCtx(rctx *relabelCtx) {
 	rctx.labels = rctx.labels[:0]
 	relabelCtxPool.Put(rctx)
 }
+
+func fixPromCompatibleNaming(labels []prompbmarshal.Label) {
+	// Replace unsupported Prometheus chars in label names and metric names with underscores.
+	for i := range labels {
+		label := &labels[i]
+		if label.Name == "__name__" {
+			label.Value = promrelabel.SanitizeMetricName(label.Value)
+		} else {
+			label.Name = promrelabel.SanitizeLabelName(label.Name)
+		}
+	}
+}

app/vmagent/remotewrite/relabel_test.go

@@ -10,18 +10,16 @@ import (
 )
 
 func TestApplyRelabeling(t *testing.T) {
-	f := func(extraLabels []prompbmarshal.Label, pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
+	f := func(pcs *promrelabel.ParsedConfigs, sTss, sExpTss string) {
 		rctx := &relabelCtx{}
 		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
-		gotTss := rctx.applyRelabeling(tss, extraLabels, pcs)
+		gotTss := rctx.applyRelabeling(tss, pcs)
 		if !reflect.DeepEqual(gotTss, expTss) {
 			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, gotTss)
 		}
 	}
 
-	f(nil, nil, "up", "up")
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, "up", `up{foo="bar"}`)
-	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, nil, `up{foo="baz"}`, `up{foo="bar"}`)
+	f(nil, "up", "up")
 
 	pcs, err := promrelabel.ParseRelabelConfigsData([]byte(`
 - target_label: "foo"
@@ -32,11 +30,33 @@ func TestApplyRelabeling(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
-	f(nil, pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
+	f(pcs, `up{foo="baz", env="prod"}`, `up{foo="aaa"}`)
 
 	oldVal := *usePromCompatibleNaming
 	*usePromCompatibleNaming = true
-	f(nil, nil, `foo.bar`, `foo_bar`)
+	f(nil, `foo.bar`, `foo_bar`)
+	*usePromCompatibleNaming = oldVal
+}
+
+func TestAppendExtraLabels(t *testing.T) {
+	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
+		t.Helper()
+		rctx := &relabelCtx{}
+		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
+		rctx.appendExtraLabels(tss, extraLabels)
+		if !reflect.DeepEqual(tss, expTss) {
+			t.Fatalf("expected to have: \n%v;\ngot: \n%v", expTss, tss)
+		}
+	}
+
+	f(nil, "up", "up")
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, "up", `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "foo", Value: "bar"}}, `up{foo="baz"}`, `up{foo="bar"}`)
+	f([]prompbmarshal.Label{{Name: "baz", Value: "qux"}}, `up{foo="baz"}`, `up{foo="baz",baz="qux"}`)
+
+	oldVal := *usePromCompatibleNaming
+	*usePromCompatibleNaming = true
+	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
 	*usePromCompatibleNaming = oldVal
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -10,8 +10,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/cespare/xxhash/v2"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -28,6 +26,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/streamaggr"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/tenantmetrics"
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 )
 
 var (
@@ -49,14 +48,15 @@ var (
 		"isn't enough for sending high volume of collected data to remote storage. Default value is 2 * numberOfAvailableCPUs")
 	showRemoteWriteURL = flag.Bool("remoteWrite.showURL", false, "Whether to show -remoteWrite.url in the exported metrics. "+
 		"It is hidden by default, since it can contain sensitive info such as auth key")
-	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
+	maxPendingBytesPerURL = flagutil.NewArrayBytes("remoteWrite.maxDiskUsagePerURL", 0, "The maximum file-based buffer size in bytes at -remoteWrite.tmpDataPath "+
 		"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
 		"Buffered data is stored in ~500MB chunks. It is recommended to set the value for this flag to a multiple of the block size 500MB. "+
 		"Disk usage is unlimited if the value is set to 0")
-	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", "The number of significant figures to leave in metric values before writing them "+
+	significantFigures = flagutil.NewArrayInt("remoteWrite.significantFigures", 0, "The number of significant figures to leave in metric values before writing them "+
 		"to remote storage. See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant figures. "+
 		"This option may be used for improving data compression for the stored metrics. See also -remoteWrite.roundDigits")
-	roundDigits = flagutil.NewArrayInt("remoteWrite.roundDigits", "Round metric values to this number of decimal digits after the point before writing them to remote storage. "+
+	roundDigits = flagutil.NewArrayInt("remoteWrite.roundDigits", 100, "Round metric values to this number of decimal digits after the point before "+
+		"writing them to remote storage. "+
 		"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
 		"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
 		"This option may be used for improving data compression for the stored metrics")
@@ -78,7 +78,7 @@ var (
 	streamAggrDropInput = flagutil.NewArrayBool("remoteWrite.streamAggr.dropInput", "Whether to drop all the input samples after the aggregation "+
 		"with -remoteWrite.streamAggr.config. By default, only aggregates samples are dropped, while the remaining samples "+
 		"are written to the corresponding -remoteWrite.url . See also -remoteWrite.streamAggr.keepInput and https://docs.victoriametrics.com/stream-aggregation.html")
-	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", "Input samples are de-duplicated with this interval before being aggregated. "+
+	streamAggrDedupInterval = flagutil.NewArrayDuration("remoteWrite.streamAggr.dedupInterval", 0, "Input samples are de-duplicated with this interval before being aggregated. "+
 		"Only the last sample per each time series per each interval is aggregated if the interval is greater than zero")
 )
 
@@ -258,7 +258,7 @@ func newRemoteWriteCtxs(at *auth.Token, urls []string) []*remoteWriteCtx {
 		if *showRemoteWriteURL {
 			sanitizedURL = fmt.Sprintf("%d:%s", i+1, remoteWriteURL)
 		}
-		rwctxs[i] = newRemoteWriteCtx(i, at, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
+		rwctxs[i] = newRemoteWriteCtx(i, remoteWriteURL, maxInmemoryBlocks, sanitizedURL)
 	}
 
 	if !*keepDanglingQueues {
@@ -355,7 +355,7 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 	var rctx *relabelCtx
 	rcs := allRelabelConfigs.Load()
 	pcsGlobal := rcs.global
-	if pcsGlobal.Len() > 0 || len(labelsGlobal) > 0 {
+	if pcsGlobal.Len() > 0 {
 		rctx = getRelabelCtx()
 	}
 	tss := wr.Timeseries
@@ -386,7 +386,7 @@ func Push(at *auth.Token, wr *prompbmarshal.WriteRequest) {
 		}
 		if rctx != nil {
 			rowsCountBeforeRelabel := getRowsCount(tssBlock)
-			tssBlock = rctx.applyRelabeling(tssBlock, labelsGlobal, pcsGlobal)
+			tssBlock = rctx.applyRelabeling(tssBlock, pcsGlobal)
 			rowsCountAfterRelabel := getRowsCount(tssBlock)
 			rowsDroppedByGlobalRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 		}
@@ -559,14 +559,14 @@ type remoteWriteCtx struct {
 	rowsDroppedByRelabel   *metrics.Counter
 }
 
-func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
+func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, maxInmemoryBlocks int, sanitizedURL string) *remoteWriteCtx {
 	// strip query params, otherwise changing params resets pq
 	pqURL := *remoteWriteURL
 	pqURL.RawQuery = ""
 	pqURL.Fragment = ""
 	h := xxhash.Sum64([]byte(pqURL.String()))
 	queuePath := filepath.Join(*tmpDataPath, persistentQueueDirname, fmt.Sprintf("%d_%016X", argIdx+1, h))
-	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArgOrDefault(argIdx, 0)
+	maxPendingBytes := maxPendingBytesPerURL.GetOptionalArg(argIdx)
 	if maxPendingBytes != 0 && maxPendingBytes < persistentqueue.DefaultChunkFileSize {
 		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195
 		logger.Warnf("rounding the -remoteWrite.maxDiskUsagePerURL=%d to the minimum supported value: %d", maxPendingBytes, persistentqueue.DefaultChunkFileSize)
@@ -590,8 +590,8 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	c.init(argIdx, *queues, sanitizedURL)
 
 	// Initialize pss
-	sf := significantFigures.GetOptionalArgOrDefault(argIdx, 0)
-	rd := roundDigits.GetOptionalArgOrDefault(argIdx, 100)
+	sf := significantFigures.GetOptionalArg(argIdx)
+	rd := roundDigits.GetOptionalArg(argIdx)
 	pssLen := *queues
 	if n := cgroup.AvailableCPUs(); pssLen > n {
 		// There is no sense in running more than availableCPUs concurrent pendingSeries,
@@ -616,7 +616,7 @@ func newRemoteWriteCtx(argIdx int, at *auth.Token, remoteWriteURL *url.URL, maxI
 	// Initialize sas
 	sasFile := streamAggrConfig.GetOptionalArg(argIdx)
 	if sasFile != "" {
-		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(argIdx, 0)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(argIdx)
 		sas, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 		if err != nil {
 			logger.Fatalf("cannot initialize stream aggregators from -remoteWrite.streamAggr.config=%q: %s", sasFile, err)
@@ -668,7 +668,7 @@ func (rwctx *remoteWriteCtx) Push(tss []prompbmarshal.TimeSeries) {
 		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
 		tss = append(*v, tss...)
 		rowsCountBeforeRelabel := getRowsCount(tss)
-		tss = rctx.applyRelabeling(tss, nil, pcs)
+		tss = rctx.applyRelabeling(tss, pcs)
 		rowsCountAfterRelabel := getRowsCount(tss)
 		rwctx.rowsDroppedByRelabel.Add(rowsCountBeforeRelabel - rowsCountAfterRelabel)
 	}
@@ -705,11 +705,13 @@ var matchIdxsPool bytesutil.ByteBufferPool
 
 func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, dropInput bool) []prompbmarshal.TimeSeries {
 	dst := src[:0]
-	for i, match := range matchIdxs {
-		if match == 0 {
-			continue
+	if !dropInput {
+		for i, match := range matchIdxs {
+			if match == 1 {
+				continue
+			}
+			dst = append(dst, src[i])
 		}
-		dst = append(dst, src[i])
 	}
 	tail := src[len(dst):]
 	_ = prompbmarshal.ResetTimeSeries(tail)
@@ -717,9 +719,26 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 }
 
 func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
+	var rctx *relabelCtx
+	var v *[]prompbmarshal.TimeSeries
+	if len(labelsGlobal) > 0 {
+		// Make a copy of tss before adding extra labels in order to prevent
+		// from affecting time series for other remoteWrite.url configs.
+		rctx = getRelabelCtx()
+		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
+		tss = append(*v, tss...)
+		rctx.appendExtraLabels(tss, labelsGlobal)
+	}
+
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 	pss[idx].Push(tss)
+
+	if rctx != nil {
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}
 }
 
 func (rwctx *remoteWriteCtx) reinitStreamAggr() {
@@ -732,7 +751,7 @@ func (rwctx *remoteWriteCtx) reinitStreamAggr() {
 	sasFile := streamAggrConfig.GetOptionalArg(rwctx.idx)
 	logger.Infof("reloading stream aggregation configs pointed by -remoteWrite.streamAggr.config=%q", sasFile)
 	metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_total{path=%q}`, sasFile)).Inc()
-	dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(rwctx.idx, 0)
+	dedupInterval := streamAggrDedupInterval.GetOptionalArg(rwctx.idx)
 	sasNew, err := streamaggr.LoadFromFile(sasFile, rwctx.pushInternal, dedupInterval)
 	if err != nil {
 		metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_streamaggr_config_reloads_errors_total{path=%q}`, sasFile)).Inc()
@@ -774,7 +793,7 @@ func CheckStreamAggrConfigs() error {
 		if sasFile == "" {
 			continue
 		}
-		dedupInterval := streamAggrDedupInterval.GetOptionalArgOrDefault(idx, 0)
+		dedupInterval := streamAggrDedupInterval.GetOptionalArg(idx)
 		sas, err := streamaggr.LoadFromFile(sasFile, pushNoop, dedupInterval)
 		if err != nil {
 			return fmt.Errorf("cannot load -remoteWrite.streamAggr.config=%q: %w", sasFile, err)

app/vmagent/remotewrite/statconn.go

@@ -27,7 +27,7 @@ var (
 	stdDialerOnce sync.Once
 )
 
-func statDial(ctx context.Context, networkUnused, addr string) (conn net.Conn, err error) {
+func statDial(ctx context.Context, _, addr string) (conn net.Conn, err error) {
 	network := netutil.GetTCPNetwork()
 	d := getStdDialer()
 	conn, err = d.DialContext(ctx, network, addr)

app/vmalert/README.md

@@ -112,16 +112,23 @@ name: <string>
 # How often rules in the group are evaluated.
 [ interval: <duration> | default = -evaluationInterval flag ]
 
+# Optional
+# Group will be evaluated at the exact offset in the range of [0...interval].
+# E.g. for Group with `interval: 1h` and `eval_offset: 5m` the evaluation will
+# start at 5th minute of the hour. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3409
+# `eval_offset` can't be bigger than `interval`.
+[ eval_offset: <duration> ]
+
 # Limit the number of alerts an alerting rule and series a recording
 # rule can produce. 0 is no limit.
 [ limit: <int> | default = 0 ]
 
 # How many rules execute at once within a group. Increasing concurrency may speed
-# up round execution speed.
+# up group's evaluation duration (exposed via `vmalert_iteration_duration_seconds` metric).
 [ concurrency: <integer> | default = 1 ]
 
 # Optional type for expressions inside the rules. Supported values: "graphite" and "prometheus".
-# By default "prometheus" type is used.
+# By default, "prometheus" type is used.
 [ type: <string> ]
 
 # Optional list of HTTP URL parameters
@@ -519,7 +526,7 @@ Alertmanagers.
 
 To avoid recording rules results and alerts state duplication in VictoriaMetrics server
 don't forget to configure [deduplication](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#deduplication).
-The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `evaluation_interval`.
+The recommended value for `-dedup.minScrapeInterval` must be multiple of vmalert's `-evaluationInterval`.
 If you observe inconsistent or "jumping" values in series produced by vmalert, try disabling `-datasource.queryTimeAlignment`
 command line flag. Because of alignment, two or more vmalert HA pairs will produce results with the same timestamps.
 But due of backfilling (data delivered to the datasource with some delay) values of such results may differ,
@@ -771,7 +778,7 @@ may get empty response from the datasource and produce empty recording rules or
 
 Try the following recommendations to reduce the chance of hitting the data delay issue:
 
-* Always configure group's `evaluationInterval` to be bigger or at least equal to 
+* Always configure group's `-evaluationInterval` to be bigger or at least equal to
 [time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution);
 * Ensure that `[duration]` value is at least twice bigger than 
 [time series resolution](https://docs.victoriametrics.com/keyConcepts.html#time-series-resolution). For example,
@@ -983,7 +990,7 @@ The shortlist of configuration flags is the following:
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -evaluationInterval duration
      How often to evaluate the rules (default 1m0s)
   -external.alert.source string
@@ -1023,6 +1030,12 @@ The shortlist of configuration flags is the following:
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -1494,7 +1507,7 @@ spec:
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmalert` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert` binary and puts it into the `bin` folder.
 
@@ -1510,7 +1523,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 ### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmalert-linux-arm` or `make vmalert-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmalert-linux-arm` or `vmalert-linux-arm64` binary respectively and puts it into the `bin` folder.
 

app/vmalert/alerting.go

@@ -72,6 +72,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
+			EvalOffset:         group.EvalOffset,
 			QueryParams:        group.Params,
 			Headers:            group.Headers,
 			Debug:              cfg.Debug,

app/vmalert/alerting_test.go

@@ -427,7 +427,8 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 			newTestAlertingRule("multi-series-for=>pending=>pending=>firing", 3*time.Second),
 			[]datasource.Metric{
 				{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
-				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+				{
+					Values: []float64{1, 1}, Timestamps: []int64{1, 5},
 					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
 				},
 			},
@@ -436,21 +437,26 @@ func TestAlertingRule_ExecRange(t *testing.T) {
 				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
 				{State: notifier.StateFiring, ActiveAt: time.Unix(1, 0)},
 				//
-				{State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
+				{
+					State: notifier.StatePending, ActiveAt: time.Unix(1, 0),
 					Labels: map[string]string{
 						"foo": "bar",
-					}},
-				{State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
+					},
+				},
+				{
+					State: notifier.StatePending, ActiveAt: time.Unix(5, 0),
 					Labels: map[string]string{
 						"foo": "bar",
-					}},
+					},
+				},
 			},
 		},
 		{
 			newTestRuleWithLabels("multi-series-firing", "source", "vm"),
 			[]datasource.Metric{
 				{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
-				{Values: []float64{1, 1}, Timestamps: []int64{1, 5},
+				{
+					Values: []float64{1, 1}, Timestamps: []int64{1, 5},
 					Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
 				},
 			},
@@ -586,7 +592,8 @@ func TestGroup_Restore(t *testing.T) {
 		[]config.Rule{{Alert: "foo", Expr: "foo", For: promutils.NewDuration(time.Second)}},
 		map[uint64]*notifier.Alert{
 			hash(map[string]string{alertNameLabel: "foo", alertGroupNameLabel: "TestRestore"}): {
-				ActiveAt: ts},
+				ActiveAt: ts,
+			},
 		})
 
 	// two rules, two active alerts, one with state restored
@@ -603,7 +610,8 @@ func TestGroup_Restore(t *testing.T) {
 				ActiveAt: defaultTS,
 			},
 			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
-				ActiveAt: ts},
+				ActiveAt: ts,
+			},
 		})
 
 	// two rules, two active alerts, two with state restored
@@ -622,7 +630,8 @@ func TestGroup_Restore(t *testing.T) {
 				ActiveAt: ts,
 			},
 			hash(map[string]string{alertNameLabel: "bar", alertGroupNameLabel: "TestRestore"}): {
-				ActiveAt: ts},
+				ActiveAt: ts,
+			},
 		})
 
 	// one active alert but wrong state restore
@@ -844,7 +853,8 @@ func TestAlertingRule_Template(t *testing.T) {
 				hash(map[string]string{
 					alertNameLabel:      "OriginLabels",
 					alertGroupNameLabel: "Testing",
-					"instance":          "foo"}): {
+					"instance":          "foo",
+				}): {
 					Labels: map[string]string{
 						alertNameLabel:      "OriginLabels",
 						alertGroupNameLabel: "Testing",
@@ -872,7 +882,6 @@ func TestAlertingRule_Template(t *testing.T) {
 				gotAlert := tc.rule.alerts[hash]
 				if gotAlert == nil {
 					t.Fatalf("alert %d is missing; labels: %v; annotations: %v", hash, expAlert.Labels, expAlert.Annotations)
-					break
 				}
 				if !reflect.DeepEqual(expAlert.Annotations, gotAlert.Annotations) {
 					t.Fatalf("expected to have annotations %#v; got %#v", expAlert.Annotations, gotAlert.Annotations)

app/vmalert/config/config.go

@@ -23,6 +23,7 @@ type Group struct {
 	File        string
 	Name        string              `yaml:"name"`
 	Interval    *promutils.Duration `yaml:"interval,omitempty"`
+	EvalOffset  *promutils.Duration `yaml:"eval_offset,omitempty"`
 	Limit       int                 `yaml:"limit,omitempty"`
 	Rules       []Rule              `yaml:"rules"`
 	Concurrency int                 `yaml:"concurrency"`
@@ -63,11 +64,27 @@ func (g *Group) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
-// Validate check for internal Group or Rule configuration errors
+// Validate checks configuration errors for group and internal rules
 func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool) error {
 	if g.Name == "" {
 		return fmt.Errorf("group name must be set")
 	}
+	if g.Interval.Duration() < 0 {
+		return fmt.Errorf("interval shouldn't be lower than 0")
+	}
+	if g.EvalOffset.Duration() < 0 {
+		return fmt.Errorf("eval_offset shouldn't be lower than 0")
+	}
+	// if `eval_offset` is set, interval won't use global evaluationInterval flag and must bigger than offset.
+	if g.EvalOffset.Duration() > g.Interval.Duration() {
+		return fmt.Errorf("eval_offset should be smaller than interval; now eval_offset: %v, interval: %v", g.EvalOffset.Duration(), g.Interval.Duration())
+	}
+	if g.Limit < 0 {
+		return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
+	}
+	if g.Concurrency < 0 {
+		return fmt.Errorf("invalid concurrency %d, shouldn't be less than 0", g.Concurrency)
+	}
 
 	uniqueRules := map[uint64]struct{}{}
 	for _, r := range g.Rules {
@@ -76,26 +93,26 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
 			ruleName = r.Alert
 		}
 		if _, ok := uniqueRules[r.ID]; ok {
-			return fmt.Errorf("%q is a duplicate within the group %q", r.String(), g.Name)
+			return fmt.Errorf("%q is a duplicate in group", r.String())
 		}
 		uniqueRules[r.ID] = struct{}{}
 		if err := r.Validate(); err != nil {
-			return fmt.Errorf("invalid rule %q.%q: %w", g.Name, ruleName, err)
+			return fmt.Errorf("invalid rule %q: %w", ruleName, err)
 		}
 		if validateExpressions {
 			// its needed only for tests.
 			// because correct types must be inherited after unmarshalling.
 			exprValidator := g.Type.ValidateExpr
 			if err := exprValidator(r.Expr); err != nil {
-				return fmt.Errorf("invalid expression for rule %q.%q: %w", g.Name, ruleName, err)
+				return fmt.Errorf("invalid expression for rule  %q: %w", ruleName, err)
 			}
 		}
 		if validateTplFn != nil {
 			if err := validateTplFn(r.Annotations); err != nil {
-				return fmt.Errorf("invalid annotations for rule %q.%q: %w", g.Name, ruleName, err)
+				return fmt.Errorf("invalid annotations for rule  %q: %w", ruleName, err)
 			}
 			if err := validateTplFn(r.Labels); err != nil {
-				return fmt.Errorf("invalid labels for rule %q.%q: %w", g.Name, ruleName, err)
+				return fmt.Errorf("invalid labels for rule  %q: %w", ruleName, err)
 			}
 		}
 	}

app/vmalert/config/config_test.go

@@ -68,6 +68,10 @@ func TestParseBad(t *testing.T) {
 		path   []string
 		expErr string
 	}{
+		{
+			[]string{"testdata/rules/rules_interval_bad.rules"},
+			"eval_offset should be smaller than interval",
+		},
 		{
 			[]string{"testdata/rules/rules0-bad.rules"},
 			"unexpected token",
@@ -141,6 +145,35 @@ func TestGroup_Validate(t *testing.T) {
 			group:  &Group{},
 			expErr: "group name must be set",
 		},
+		{
+			group: &Group{
+				Name:     "negative interval",
+				Interval: promutils.NewDuration(-1),
+			},
+			expErr: "interval shouldn't be lower than 0",
+		},
+		{
+			group: &Group{
+				Name:       "wrong eval_offset",
+				Interval:   promutils.NewDuration(time.Minute),
+				EvalOffset: promutils.NewDuration(2 * time.Minute),
+			},
+			expErr: "eval_offset should be smaller than interval",
+		},
+		{
+			group: &Group{
+				Name:  "wrong limit",
+				Limit: -1,
+			},
+			expErr: "invalid limit",
+		},
+		{
+			group: &Group{
+				Name:        "wrong concurrency",
+				Concurrency: -1,
+			},
+			expErr: "invalid concurrency",
+		},
 		{
 			group: &Group{
 				Name: "test",

app/vmalert/config/testdata/rules/rules_interval_bad.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: groupTest
+    ## default interval is 1min, eval_offset shouldn't be greater than interval
+    eval_offset: 2m
+    rules:
+      - alert: VMRows
+        for: 2s
+        expr: sum(rate(vm_http_request_errors_total[2s])) > 0
+        labels:
+          label: bar
+          host: "{{ $labels.instance }}"
+        annotations:
+          summary: "{{ $value }}"

app/vmalert/datasource/datasource.go

@@ -44,6 +44,7 @@ type QuerierBuilder interface {
 type QuerierParams struct {
 	DataSourceType     string
 	EvaluationInterval time.Duration
+	EvalOffset         *time.Duration
 	QueryParams        url.Values
 	Headers            map[string]string
 	Debug              bool

app/vmalert/datasource/vm.go

@@ -37,11 +37,20 @@ type VMStorage struct {
 	appendTypePrefix bool
 	lookBack         time.Duration
 	queryStep        time.Duration
+	dataSourceType   datasourceType
 
-	dataSourceType     datasourceType
+	// evaluationInterval will align the request's timestamp
+	// if `datasource.queryTimeAlignment` is enabled,
+	// will set request's `step` param as well.
 	evaluationInterval time.Duration
-	extraParams        url.Values
-	extraHeaders       []keyValue
+	// evaluationOffset shifts the request's timestamp, will be equal
+	// to the offset specified evaluationInterval.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4693
+	evaluationOffset *time.Duration
+	// extraParams contains params to be attached to each HTTP request
+	extraParams url.Values
+	// extraHeaders are headers to be attached to each HTTP request
+	extraHeaders []keyValue
 
 	// whether to print additional log messages
 	// for each sent request
@@ -86,13 +95,21 @@ func (s *VMStorage) Clone() *VMStorage {
 func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage {
 	s.dataSourceType = toDatasourceType(params.DataSourceType)
 	s.evaluationInterval = params.EvaluationInterval
+	s.evaluationOffset = params.EvalOffset
 	if params.QueryParams != nil {
 		if s.extraParams == nil {
 			s.extraParams = url.Values{}
 		}
 		for k, vl := range params.QueryParams {
-			for _, v := range vl { // custom query params are prior to default ones
-				s.extraParams.Set(k, v)
+			// custom query params are prior to default ones
+			if s.extraParams.Has(k) {
+				s.extraParams.Del(k)
+			}
+			for _, v := range vl {
+				// don't use .Set() instead of Del/Add since it is allowed
+				// for GET params to be duplicated
+				// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908
+				s.extraParams.Add(k, v)
 			}
 		}
 	}
@@ -127,21 +144,14 @@ func NewVMStorage(baseURL string, authCfg *promauth.Config, lookBack time.Durati
 
 // Query executes the given query and returns parsed response
 func (s *VMStorage) Query(ctx context.Context, query string, ts time.Time) (Result, *http.Request, error) {
-	req, err := s.newRequestPOST()
-	if err != nil {
-		return Result{}, nil, err
-	}
-
-	switch s.dataSourceType {
-	case "", datasourcePrometheus:
-		s.setPrometheusInstantReqParams(req, query, ts)
-	case datasourceGraphite:
-		s.setGraphiteReqParams(req, query, ts)
-	default:
-		return Result{}, nil, fmt.Errorf("engine not found: %q", s.dataSourceType)
-	}
-
+	req := s.newQueryRequest(query, ts)
 	resp, err := s.do(ctx, req)
+	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+		// something in the middle between client and datasource might be closing
+		// the connection. So we do a one more attempt in hope request will succeed.
+		req = s.newQueryRequest(query, ts)
+		resp, err = s.do(ctx, req)
+	}
 	if err != nil {
 		return Result{}, req, err
 	}
@@ -164,18 +174,20 @@ func (s *VMStorage) QueryRange(ctx context.Context, query string, start, end tim
 	if s.dataSourceType != datasourcePrometheus {
 		return res, fmt.Errorf("%q is not supported for QueryRange", s.dataSourceType)
 	}
-	req, err := s.newRequestPOST()
-	if err != nil {
-		return res, err
-	}
 	if start.IsZero() {
 		return res, fmt.Errorf("start param is missing")
 	}
 	if end.IsZero() {
 		return res, fmt.Errorf("end param is missing")
 	}
-	s.setPrometheusRangeReqParams(req, query, start, end)
+	req := s.newQueryRangeRequest(query, start, end)
 	resp, err := s.do(ctx, req)
+	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
+		// something in the middle between client and datasource might be closing
+		// the connection. So we do a one more attempt in hope request will succeed.
+		req = s.newQueryRangeRequest(query, start, end)
+		resp, err = s.do(ctx, req)
+	}
 	if err != nil {
 		return res, err
 	}
@@ -190,11 +202,6 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 		logger.Infof("DEBUG datasource request: executing %s request with params %q", req.Method, req.URL.RawQuery)
 	}
 	resp, err := s.c.Do(req.WithContext(ctx))
-	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
-		// something in the middle between client and datasource might be closing
-		// the connection. So we do a one more attempt in hope request will succeed.
-		resp, err = s.c.Do(req.WithContext(ctx))
-	}
 	if err != nil {
 		return nil, fmt.Errorf("error getting response from %s: %w", req.URL.Redacted(), err)
 	}
@@ -206,10 +213,29 @@ func (s *VMStorage) do(ctx context.Context, req *http.Request) (*http.Response,
 	return resp, nil
 }
 
-func (s *VMStorage) newRequestPOST() (*http.Request, error) {
+func (s *VMStorage) newQueryRangeRequest(query string, start, end time.Time) *http.Request {
+	req := s.newRequest()
+	s.setPrometheusRangeReqParams(req, query, start, end)
+	return req
+}
+
+func (s *VMStorage) newQueryRequest(query string, ts time.Time) *http.Request {
+	req := s.newRequest()
+	switch s.dataSourceType {
+	case "", datasourcePrometheus:
+		s.setPrometheusInstantReqParams(req, query, ts)
+	case datasourceGraphite:
+		s.setGraphiteReqParams(req, query, ts)
+	default:
+		logger.Panicf("BUG: engine not found: %q", s.dataSourceType)
+	}
+	return req
+}
+
+func (s *VMStorage) newRequest() *http.Request {
 	req, err := http.NewRequest(http.MethodPost, s.datasourceURL, nil)
 	if err != nil {
-		return nil, err
+		logger.Panicf("BUG: unexpected error from http.NewRequest(%q): %s", s.datasourceURL, err)
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
@@ -218,5 +244,5 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	for _, h := range s.extraHeaders {
 		req.Header.Set(h.key, h.value)
 	}
-	return req, nil
+	return req
 }

app/vmalert/datasource/vm_prom_api.go

@@ -161,13 +161,8 @@ func (s *VMStorage) setPrometheusInstantReqParams(r *http.Request, query string,
 		r.URL.Path += "/api/v1/query"
 	}
 	q := r.URL.Query()
-	if s.lookBack > 0 {
-		timestamp = timestamp.Add(-s.lookBack)
-	}
-	if *queryTimeAlignment && s.evaluationInterval > 0 {
-		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
-		timestamp = timestamp.Truncate(s.evaluationInterval)
-	}
+
+	timestamp = s.adjustReqTimestamp(timestamp)
 	q.Set("time", timestamp.Format(time.RFC3339))
 	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
@@ -191,6 +186,9 @@ func (s *VMStorage) setPrometheusRangeReqParams(r *http.Request, query string, s
 		r.URL.Path += "/api/v1/query_range"
 	}
 	q := r.URL.Query()
+	if s.evaluationOffset != nil {
+		start = start.Truncate(s.evaluationInterval).Add(*s.evaluationOffset)
+	}
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
 	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
@@ -215,3 +213,30 @@ func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string) {
 	q.Set("query", query)
 	r.URL.RawQuery = q.Encode()
 }
+
+func (s *VMStorage) adjustReqTimestamp(timestamp time.Time) time.Time {
+	if s.evaluationOffset != nil {
+		// calculate the min timestamp on the evaluationInterval
+		intervalStart := timestamp.Truncate(s.evaluationInterval)
+		ts := intervalStart.Add(*s.evaluationOffset)
+		if timestamp.Before(ts) {
+			// if passed timestamp is before the expected evaluation offset,
+			// then we should adjust it to the previous evaluation round.
+			// E.g. request with evaluationInterval=1h and evaluationOffset=30m
+			// was evaluated at 11:20. Then the timestamp should be adjusted
+			// to 10:30, to the previous evaluationInterval.
+			return ts.Add(-s.evaluationInterval)
+		}
+		// evaluationOffset shouldn't interfere with queryTimeAlignment or lookBack,
+		// so we return it immediately
+		return ts
+	}
+	if *queryTimeAlignment {
+		// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232
+		timestamp = timestamp.Truncate(s.evaluationInterval)
+	}
+	if s.lookBack > 0 {
+		timestamp = timestamp.Add(-s.lookBack)
+	}
+	return timestamp
+}

app/vmalert/datasource/vm_prom_api_test.go

@@ -3,6 +3,7 @@ package datasource
 import (
 	"encoding/json"
 	"testing"
+	"time"
 )
 
 func BenchmarkMetrics(b *testing.B) {
@@ -18,3 +19,74 @@ func BenchmarkMetrics(b *testing.B) {
 		}
 	})
 }
+
+func TestGetPrometheusReqTimestamp(t *testing.T) {
+	offset := 30 * time.Minute
+	testCases := []struct {
+		name               string
+		s                  *VMStorage
+		queryTimeAlignment bool
+		originTS, expTS    string
+	}{
+		{
+			"with eval_offset, find previous offset point",
+			&VMStorage{
+				evaluationOffset:   &offset,
+				evaluationInterval: time.Hour,
+				lookBack:           1 * time.Minute,
+			},
+			false,
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T10:30:00+00:00",
+		},
+		{
+			"with eval_offset",
+			&VMStorage{
+				evaluationOffset:   &offset,
+				evaluationInterval: time.Hour,
+			},
+			true,
+			"2023-08-28T11:41:00+00:00",
+			"2023-08-28T11:30:00+00:00",
+		},
+		{
+			"with query align",
+			&VMStorage{
+				evaluationInterval: time.Hour,
+			},
+			true,
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T11:00:00+00:00",
+		},
+		{
+			"with query align and lookback",
+			&VMStorage{
+				evaluationInterval: time.Hour,
+				lookBack:           1 * time.Minute,
+			},
+			true,
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T10:59:00+00:00",
+		},
+		{
+			"without query align",
+			&VMStorage{
+				evaluationInterval: time.Hour,
+			},
+			false,
+			"2023-08-28T11:11:00+00:00",
+			"2023-08-28T11:11:00+00:00",
+		},
+	}
+	for _, tc := range testCases {
+		oldAlignPara := *queryTimeAlignment
+		*queryTimeAlignment = tc.queryTimeAlignment
+		originT, _ := time.Parse(time.RFC3339, tc.originTS)
+		expT, _ := time.Parse(time.RFC3339, tc.expTS)
+		gotTS := tc.s.adjustReqTimestamp(originT)
+		if !gotTS.Equal(expT) {
+			t.Fatalf("get wrong prometheus request timestamp, expect %s, got %s", expT, gotTS)
+		}
+		*queryTimeAlignment = oldAlignPara
+	}
+}

app/vmalert/datasource/vm_test.go

@@ -596,6 +596,17 @@ func TestRequestParams(t *testing.T) {
 				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
 			},
 		},
+		{
+			"allow duplicates in query params",
+			false,
+			storage.Clone().ApplyParams(QuerierParams{
+				QueryParams: url.Values{"extra_labels": {"env=dev", "foo=bar"}},
+			}),
+			func(t *testing.T, r *http.Request) {
+				exp := url.Values{"query": {query}, "round_digits": {"10"}, "extra_labels": {"env=dev", "foo=bar"}, "time": {timestamp.Format(time.RFC3339)}}
+				checkEqualString(t, exp.Encode(), r.URL.RawQuery)
+			},
+		},
 		{
 			"graphite extra params",
 			false,
@@ -629,10 +640,7 @@ func TestRequestParams(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			req, err := tc.vm.newRequestPOST()
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
+			req := tc.vm.newRequest()
 			switch tc.vm.dataSourceType {
 			case "", datasourcePrometheus:
 				if tc.queryRange {
@@ -727,10 +735,7 @@ func TestHeaders(t *testing.T) {
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
 			vm := tt.vmFn()
-			req, err := vm.newRequestPOST()
-			if err != nil {
-				t.Fatalf("unexpected error: %s", err)
-			}
+			req := vm.newQueryRequest("foo", time.Now())
 			tt.checkFn(t, req)
 		})
 	}

app/vmalert/group.go

@@ -31,6 +31,7 @@ type Group struct {
 	Rules          []Rule
 	Type           config.Type
 	Interval       time.Duration
+	EvalOffset     *time.Duration
 	Limit          int
 	Concurrency    int
 	Checksum       string
@@ -116,6 +117,9 @@ func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
 	if g.Concurrency < 1 {
 		g.Concurrency = 1
 	}
+	if cfg.EvalOffset != nil {
+		g.EvalOffset = &cfg.EvalOffset.D
+	}
 	for _, h := range cfg.Headers {
 		g.Headers[h.Key] = h.Value
 	}
@@ -163,6 +167,10 @@ func (g *Group) ID() uint64 {
 	hash.Write([]byte("\xff"))
 	hash.Write([]byte(g.Name))
 	hash.Write([]byte(g.Type.Get()))
+	hash.Write([]byte(g.Interval.String()))
+	if g.EvalOffset != nil {
+		hash.Write([]byte(g.EvalOffset.String()))
+	}
 	return hash.Sum64()
 }
 
@@ -277,15 +285,13 @@ var skipRandSleepOnGroupStart bool
 func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *remotewrite.Client, rr datasource.QuerierBuilder) {
 	defer func() { close(g.finishedCh) }()
 
-	// Spread group rules evaluation over time in order to reduce load on VictoriaMetrics.
+	// sleep random duration to spread group rules evaluation
+	// over time in order to reduce load on datasource.
 	if !skipRandSleepOnGroupStart {
-		randSleep := uint64(float64(g.Interval) * (float64(g.ID()) / (1 << 64)))
-		sleepOffset := uint64(time.Now().UnixNano()) % uint64(g.Interval)
-		if randSleep < sleepOffset {
-			randSleep += uint64(g.Interval)
-		}
-		randSleep -= sleepOffset
-		sleepTimer := time.NewTimer(time.Duration(randSleep))
+		sleepBeforeStart := delayBeforeStart(time.Now(), g.ID(), g.Interval, g.EvalOffset)
+		g.infof("will start in %v", sleepBeforeStart)
+
+		sleepTimer := time.NewTimer(sleepBeforeStart)
 		select {
 		case <-ctx.Done():
 			sleepTimer.Stop()
@@ -297,6 +303,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		}
 	}
 
+	evalTS := time.Now()
+
 	e := &executor{
 		rw:                       rw,
 		notifiers:                nts,
@@ -304,9 +312,7 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 
-	evalTS := time.Now()
-
-	logger.Infof("group %q started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+	g.infof("started")
 
 	eval := func(ctx context.Context, ts time.Time) {
 		g.metrics.iterationTotal.Inc()
@@ -375,19 +381,12 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 				continue
 			}
 
-			// ensure that staleness is tracked or existing rules only
+			// ensure that staleness is tracked for existing rules only
 			e.purgeStaleSeries(g.Rules)
-
 			e.notifierHeaders = g.NotifierHeaders
-
-			if g.Interval != ng.Interval {
-				g.Interval = ng.Interval
-				t.Stop()
-				t = time.NewTicker(g.Interval)
-				evalTS = time.Now()
-			}
 			g.mu.Unlock()
-			logger.Infof("group %q re-started; interval=%v; concurrency=%d", g.Name, g.Interval, g.Concurrency)
+
+			g.infof("re-started")
 		case <-t.C:
 			missed := (time.Since(evalTS) / g.Interval) - 1
 			if missed < 0 {
@@ -405,6 +404,35 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 	}
 }
 
+// delayBeforeStart returns a duration on the interval between [ts..ts+interval].
+// delayBeforeStart accounts for `offset`, so returned duration should be always
+// bigger than the `offset`.
+func delayBeforeStart(ts time.Time, key uint64, interval time.Duration, offset *time.Duration) time.Duration {
+	var randSleep time.Duration
+	randSleep = time.Duration(float64(interval) * (float64(key) / (1 << 64)))
+	sleepOffset := time.Duration(ts.UnixNano() % interval.Nanoseconds())
+	if randSleep < sleepOffset {
+		randSleep += interval
+	}
+	randSleep -= sleepOffset
+	// check if `ts` after randSleep is before `offset`,
+	// if it is, add extra eval_offset to randSleep.
+	// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3409.
+	if offset != nil {
+		tmpEvalTS := ts.Add(randSleep)
+		if tmpEvalTS.Before(tmpEvalTS.Truncate(interval).Add(*offset)) {
+			randSleep += *offset
+		}
+	}
+	return randSleep.Truncate(time.Second)
+}
+
+func (g *Group) infof(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	logger.Infof("group %q %s; interval=%v; eval_offset=%v; concurrency=%d",
+		g.Name, msg, g.Interval, g.EvalOffset, g.Concurrency)
+}
+
 // getResolveDuration returns the duration after which firing alert
 // can be considered as resolved.
 func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Duration {

app/vmalert/group_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"math"
 	"reflect"
 	"sort"
 	"testing"
@@ -35,18 +36,19 @@ func TestUpdateWith(t *testing.T) {
 		},
 		{
 			"update alerting rule",
-			[]config.Rule{{
-				Alert: "foo",
-				Expr:  "up > 0",
-				For:   promutils.NewDuration(time.Second),
-				Labels: map[string]string{
-					"bar": "baz",
+			[]config.Rule{
+				{
+					Alert: "foo",
+					Expr:  "up > 0",
+					For:   promutils.NewDuration(time.Second),
+					Labels: map[string]string{
+						"bar": "baz",
+					},
+					Annotations: map[string]string{
+						"summary":     "{{ $value|humanize }}",
+						"description": "{{$labels}}",
+					},
 				},
-				Annotations: map[string]string{
-					"summary":     "{{ $value|humanize }}",
-					"description": "{{$labels}}",
-				},
-			},
 				{
 					Alert: "bar",
 					Expr:  "up > 0",
@@ -54,7 +56,8 @@ func TestUpdateWith(t *testing.T) {
 					Labels: map[string]string{
 						"bar": "baz",
 					},
-				}},
+				},
+			},
 			[]config.Rule{
 				{
 					Alert: "foo",
@@ -75,7 +78,8 @@ func TestUpdateWith(t *testing.T) {
 					Labels: map[string]string{
 						"bar": "baz",
 					},
-				}},
+				},
+			},
 		},
 		{
 			"update recording rule",
@@ -520,3 +524,62 @@ func TestCloseWithEvalInterruption(t *testing.T) {
 	case <-g.finishedCh:
 	}
 }
+
+func TestGroupStartDelay(t *testing.T) {
+	g := &Group{}
+	// interval of 5min and key generate a static delay of 30s
+	g.Interval = time.Minute * 5
+	key := uint64(math.MaxUint64 / 10)
+
+	f := func(atS, expS string) {
+		t.Helper()
+		at, err := time.Parse(time.DateTime, atS)
+		if err != nil {
+			t.Fatal(err)
+		}
+		expTS, err := time.Parse(time.DateTime, expS)
+		if err != nil {
+			t.Fatal(err)
+		}
+		delay := delayBeforeStart(at, key, g.Interval, g.EvalOffset)
+		gotStart := at.Add(delay)
+		if expTS != gotStart {
+			t.Errorf("expected to get %v; got %v instead", expTS, gotStart)
+		}
+	}
+
+	// test group without offset
+	f("2023-01-01 00:00:00", "2023-01-01 00:00:30")
+	f("2023-01-01 00:00:29", "2023-01-01 00:00:30")
+	f("2023-01-01 00:00:31", "2023-01-01 00:05:30")
+
+	// test group with offset smaller than above fixed randSleep,
+	// this way randSleep will always be enough
+	offset := 20 * time.Second
+	g.EvalOffset = &offset
+
+	f("2023-01-01 00:00:00", "2023-01-01 00:00:30")
+	f("2023-01-01 00:00:29", "2023-01-01 00:00:30")
+	f("2023-01-01 00:00:31", "2023-01-01 00:05:30")
+
+	// test group with offset bigger than above fixed randSleep,
+	// this way offset will be added to delay
+	offset = 3 * time.Minute
+	g.EvalOffset = &offset
+
+	f("2023-01-01 00:00:00", "2023-01-01 00:03:30")
+	f("2023-01-01 00:00:29", "2023-01-01 00:03:30")
+	f("2023-01-01 00:01:00", "2023-01-01 00:08:30")
+	f("2023-01-01 00:03:30", "2023-01-01 00:08:30")
+	f("2023-01-01 00:07:30", "2023-01-01 00:13:30")
+
+	offset = 10 * time.Minute
+	g.EvalOffset = &offset
+	// interval of 1h and key generate a static delay of 6m
+	g.Interval = time.Hour
+
+	f("2023-01-01 00:00:00", "2023-01-01 00:16:00")
+	f("2023-01-01 00:05:00", "2023-01-01 00:16:00")
+	f("2023-01-01 00:30:00", "2023-01-01 01:16:00")
+
+}

app/vmalert/main_test.go

@@ -87,10 +87,10 @@ groups:
 	)
 
 	f, err := os.CreateTemp("", "")
-	defer os.Remove(f.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer func() { _ = os.Remove(f.Name()) }()
 	writeToFile(t, f.Name(), rules1)
 
 	*configCheckInterval = 200 * time.Millisecond

app/vmalert/manager_test.go

@@ -168,7 +168,8 @@ func TestManagerUpdate(t *testing.T) {
 					Name:     "TestGroup", Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
-					}},
+					},
+				},
 			},
 		},
 		{
@@ -191,7 +192,8 @@ func TestManagerUpdate(t *testing.T) {
 					Rules: []Rule{
 						Conns,
 						ExampleAlertAlwaysFiring,
-					}},
+					},
+				},
 			},
 		},
 		{
@@ -264,7 +266,8 @@ func TestManagerUpdateNegative(t *testing.T) {
 		{
 			nil,
 			nil,
-			config.Group{Name: "Recording rule only",
+			config.Group{
+				Name: "Recording rule only",
 				Rules: []config.Rule{
 					{Record: "record", Expr: "max(up)"},
 				},
@@ -274,7 +277,8 @@ func TestManagerUpdateNegative(t *testing.T) {
 		{
 			nil,
 			nil,
-			config.Group{Name: "Alerting rule only",
+			config.Group{
+				Name: "Alerting rule only",
 				Rules: []config.Rule{
 					{Alert: "alert", Expr: "up > 0"},
 				},
@@ -284,7 +288,8 @@ func TestManagerUpdateNegative(t *testing.T) {
 		{
 			[]notifier.Notifier{&fakeNotifier{}},
 			nil,
-			config.Group{Name: "Recording and alerting rules",
+			config.Group{
+				Name: "Recording and alerting rules",
 				Rules: []config.Rule{
 					{Alert: "alert1", Expr: "up > 0"},
 					{Alert: "alert2", Expr: "up > 0"},
@@ -296,7 +301,8 @@ func TestManagerUpdateNegative(t *testing.T) {
 		{
 			nil,
 			&remotewrite.Client{},
-			config.Group{Name: "Recording and alerting rules",
+			config.Group{
+				Name: "Recording and alerting rules",
 				Rules: []config.Rule{
 					{Record: "record1", Expr: "max(up)"},
 					{Record: "record2", Expr: "max(up)"},

app/vmalert/notifier/config_watcher_test.go

@@ -14,7 +14,6 @@ import (
 
 func TestConfigWatcherReload(t *testing.T) {
 	f, err := os.CreateTemp("", "")
-	defer os.Remove(f.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -37,7 +36,6 @@ static_configs:
 	}
 
 	f2, err := os.CreateTemp("", "")
-	defer os.Remove(f2.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,7 +63,6 @@ func TestConfigWatcherStart(t *testing.T) {
 	defer consulSDServer.Close()
 
 	consulSDFile, err := os.CreateTemp("", "")
-	defer os.Remove(consulSDFile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +109,6 @@ func TestConfigWatcherReloadConcurrent(t *testing.T) {
 	defer consulSDServer2.Close()
 
 	consulSDFile, err := os.CreateTemp("", "")
-	defer os.Remove(consulSDFile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -129,7 +125,6 @@ consul_sd_configs:
 `, consulSDServer1.URL, consulSDServer2.URL))
 
 	staticAndConsulSDFile, err := os.CreateTemp("", "")
-	defer os.Remove(staticAndConsulSDFile.Name())
 	if err != nil {
 		t.Fatal(err)
 	}

app/vmalert/recording.go

@@ -61,6 +61,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul
 		q: qb.BuildWithParams(datasource.QuerierParams{
 			DataSourceType:     group.Type.String(),
 			EvaluationInterval: group.Interval,
+			EvalOffset:         group.EvalOffset,
 			QueryParams:        group.Params,
 			Headers:            group.Headers,
 		}),

app/vmalert/remotewrite/remotewrite.go

@@ -216,7 +216,9 @@ func (c *Client) flush(ctx context.Context, wr *prompbmarshal.WriteRequest) {
 		retryInterval = maxRetryInterval
 	}
 	timeStart := time.Now()
-	defer sendDuration.Add(time.Since(timeStart).Seconds())
+	defer func() {
+		sendDuration.Add(time.Since(timeStart).Seconds())
+	}()
 L:
 	for attempts := 0; ; attempts++ {
 		err := c.send(ctx, b)

app/vmalert/rule_test.go

@@ -79,7 +79,7 @@ func TestRule_state(t *testing.T) {
 // TestRule_stateConcurrent supposed to test concurrent
 // execution of state updates.
 // Should be executed with -race flag
-func TestRule_stateConcurrent(t *testing.T) {
+func TestRule_stateConcurrent(_ *testing.T) {
 	state := newRuleState(20)
 
 	const workers = 50

app/vmalert/utils/err_group_test.go

@@ -41,7 +41,7 @@ func TestErrGroup(t *testing.T) {
 // TestErrGroupConcurrent supposed to test concurrent
 // use of error group.
 // Should be executed with -race flag
-func TestErrGroupConcurrent(t *testing.T) {
+func TestErrGroupConcurrent(_ *testing.T) {
 	eg := new(ErrGroup)
 
 	const writersN = 4

app/vmauth/README.md

@@ -25,6 +25,7 @@ The auth config can be reloaded via the following ways:
   and apply new changes every 5 seconds.
 
 Docker images for `vmauth` are available [here](https://hub.docker.com/r/victoriametrics/vmauth/tags).
+See how `vmauth` used in [docker-compose env](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/README.md#victoriametrics-cluster).
 
 Pass `-help` to `vmauth` in order to see all the supported command-line flags with their descriptions.
 
@@ -35,9 +36,42 @@ accounting and rate limiting such as [vmgateway](https://docs.victoriametrics.co
 
 Each `url_prefix` in the [-auth.config](#auth-config) may contain either a single url or a list of urls.
 In the latter case `vmauth` balances load among the configured urls in least-loaded round-robin manner.
-`vmauth` retries failing `GET` requests across the configured list of urls.
-This feature is useful for balancing the load among multiple `vmselect` and/or `vminsert` nodes
-in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+
+If the backend at the configured url isn't available, then `vmauth` tries sending the request to the remaining configured urls.
+
+It is possible to configure automatic retry of requests if the backend responds with status code from optional `retry_status_codes` list.
+
+Load balancing feature can be used in the following cases:
+
+- Balancing the load among multiple `vmselect` and/or `vminsert` nodes in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html).
+  The following `-auth.config` file can be used for spreading incoming requests among 3 vmselect nodes and re-trying failed requests
+  or requests with 500 and 502 response status codes:
+
+  ```yml
+  unauthorized_user:
+    url_prefix:
+    - http://vmselect1:8481/
+    - http://vmselect2:8481/
+    - http://vmselect3:8481/
+    retry_status_codes: [500, 502]
+  ```
+
+- Spreading select queries among multiple availability zones (AZs) with identical data. For example, the following config spreads select queries
+  among 3 AZs. Requests are re-tried if some AZs are temporarily unavailable or if some `vmstorage` nodes in some AZs are temporarily unavailable.
+  `vmauth` adds `deny_partial_response=1` query arg to all the queries in order to guarantee to get full response from every AZ.
+  See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-availability) for details.
+
+  ```yml
+  unauthorized_user:
+    url_prefix:
+    - https://vmselect-az1/?deny_partial_response=1
+    - https://vmselect-az2/?deny_partial_response=1
+    - https://vmselect-az3/?deny_partial_response=1
+    retry_status_codes: [500, 502, 503]
+  ```
+
+Load balancig can also be configured independently per each user and per each `url_map` entry.
+See [auth config docs](#auth-config) for more details.
 
 ## Concurrency limiting
 
@@ -117,11 +151,16 @@ users:
 
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # The `X-Server-Hostname` http header is removed from the proxied response.
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
 - bearer_token: "YYY"
   url_prefix: "http://localhost:8428"
+  # extra headers to add to the request or remove from the request (if header value is empty)
   headers:
-  - "X-Scope-OrgID: foobar"
+    - "X-Scope-OrgID: foobar"
+  # extra headers to add to the response or remove from the response (if header value is empty)
+  response_headers:
+    - "X-Server-Hostname:" # empty value means the header will be removed from the response
 
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are proxied to http://localhost:8428 .
@@ -172,9 +211,11 @@ users:
   #     - http://vmselect2:8481/select/42/prometheus
   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+  #   Requests are re-tried at other url_prefix backends if response status codes match 500 or 502.
   #
   # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
   #   The "X-Scope-OrgID: abc" http header is added to these requests.
+  #   The "X-Server-Hostname" http header is removed from the proxied response.
   #
   # Request which do not match `src_paths` from the `url_map` are proxied to the urls from `default_url`
   # in a round-robin manner. The original request path is passed in `request_path` query arg.
@@ -190,10 +231,13 @@ users:
     url_prefix:
     - "http://vmselect1:8481/select/42/prometheus"
     - "http://vmselect2:8481/select/42/prometheus"
+    retry_status_codes: [500, 502]
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
     headers:
     - "X-Scope-OrgID: abc"
+    response_headers:
+    - "X-Server-Hostname:" # empty value means the header will be removed from the response
     ip_filters:
       deny_list: [127.0.0.1]
   default_url:
@@ -201,16 +245,14 @@ users:
   - "http://default2:8888/unsupported_url_handler"
 
 # Requests without Authorization header are routed according to `unauthorized_user` section.
+# Requests are routed in round-robin fashion between `url_prefix` backends.
+# The deny_partial_response query arg is added to all the routed requests.
+# The requests are re-tried if url_prefix backends send 500 or 503 response status codes.
 unauthorized_user:
-  url_map:
-  - src_paths:
-    - /api/v1/query
-    - /api/v1/query_range
-    url_prefix:
-    - http://vmselect1:8481/select/0/prometheus
-    - http://vmselect2:8481/select/0/prometheus
-    ip_filters:
-      allow_list: [8.8.8.8]
+  url_prefix:
+  - http://vmselect-az1/?deny_partial_response=1
+  - http://vmselect-az2/?deny_partial_response=1
+  retry_status_codes: [503, 500]
 
 ip_filters:
   allow_list: ["1.2.3.0/24", "127.0.0.1"]
@@ -221,6 +263,9 @@ ip_filters:
 The config may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values.
 This may be useful for passing secrets to the config.
 
+Please note, vmauth doesn't follow redirects. If destination redirects request to a new location, make sure this 
+location is supported in vmauth `url_map` config.
+
 ## Security
 
 It is expected that all the backend services protected by `vmauth` are located in an isolated private network, so they can be accessed by external users only via `vmauth`.
@@ -240,11 +285,11 @@ Alternatively, [https termination proxy](https://en.wikipedia.org/wiki/TLS_termi
 
 It is recommended protecting the following endpoints with authKeys:
 * `/-/reload` with `-reloadAuthKey` command-line flag, so external users couldn't trigger config reload.
-* `/flags` with `-flagsAuthkey` command-line flag, so unauthorized users couldn't get application command-line flags.
-* `/metrics` with `metricsAuthkey` command-line flag, so unauthorized users couldn't get access to [vmauth metrics](#monitoring).
-* `/debug/pprof` with `pprofAuthKey` command-line flag, so unauthorized users couldn't get access to [profiling information](#profiling).
+* `/flags` with `-flagsAuthKey` command-line flag, so unauthorized users couldn't get application command-line flags.
+* `/metrics` with `-metricsAuthKey` command-line flag, so unauthorized users couldn't get access to [vmauth metrics](#monitoring).
+* `/debug/pprof` with `-pprofAuthKey` command-line flag, so unauthorized users couldn't get access to [profiling information](#profiling).
 
-`vmauth` also supports the ability to restict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
+`vmauth` also supports the ability to restrict access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
 
 ## Monitoring
 
@@ -276,7 +321,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmauth` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmauth` binary and puts it into the `bin` folder.
 
@@ -350,7 +395,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -failTimeout duration
      Sets a delay period for load balancing to skip a malfunctioning backend. (defaults 3s)
   -flagsAuthKey string
@@ -383,6 +428,12 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -logInvalidAuthTokens
      Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page
   -loggerDisableTimestamps
@@ -407,6 +458,9 @@ See the docs at https://docs.victoriametrics.com/vmauth.html .
      The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options (default 1000)
   -maxIdleConnsPerBackend int
      The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100)
+  -maxRequestBodySizeToRetry size
+     The maximum request body size, which can be cached and re-tried at other backends. Bigger values may require more memory
+     Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16384)
   -memory.allowedBytes size
      Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage
      Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)

app/vmauth/auth_config.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/base64"
 	"flag"
 	"fmt"
@@ -37,15 +38,16 @@ type AuthConfig struct {
 
 // UserInfo is user information read from authConfigPath
 type UserInfo struct {
-	Name                  string     `yaml:"name,omitempty"`
-	BearerToken           string     `yaml:"bearer_token,omitempty"`
-	Username              string     `yaml:"username,omitempty"`
-	Password              string     `yaml:"password,omitempty"`
-	URLPrefix             *URLPrefix `yaml:"url_prefix,omitempty"`
-	URLMaps               []URLMap   `yaml:"url_map,omitempty"`
-	Headers               []Header   `yaml:"headers,omitempty"`
-	MaxConcurrentRequests int        `yaml:"max_concurrent_requests,omitempty"`
-	DefaultURL            *URLPrefix `yaml:"default_url,omitempty"`
+	Name                  string      `yaml:"name,omitempty"`
+	BearerToken           string      `yaml:"bearer_token,omitempty"`
+	Username              string      `yaml:"username,omitempty"`
+	Password              string      `yaml:"password,omitempty"`
+	URLPrefix             *URLPrefix  `yaml:"url_prefix,omitempty"`
+	URLMaps               []URLMap    `yaml:"url_map,omitempty"`
+	HeadersConf           HeadersConf `yaml:",inline"`
+	MaxConcurrentRequests int         `yaml:"max_concurrent_requests,omitempty"`
+	DefaultURL            *URLPrefix  `yaml:"default_url,omitempty"`
+	RetryStatusCodes      []int       `yaml:"retry_status_codes,omitempty"`
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
@@ -54,6 +56,12 @@ type UserInfo struct {
 	requestsDuration *metrics.Summary
 }
 
+// HeadersConf represents config for request and response headers.
+type HeadersConf struct {
+	RequestHeaders  []Header `yaml:"headers,omitempty"`
+	ResponseHeaders []Header `yaml:"response_headers,omitempty"`
+}
+
 func (ui *UserInfo) beginConcurrencyLimit() error {
 	select {
 	case ui.concurrencyLimitCh <- struct{}{}:
@@ -105,9 +113,10 @@ func (h *Header) MarshalYAML() (interface{}, error) {
 
 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths  []*SrcPath `yaml:"src_paths,omitempty"`
-	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
-	Headers   []Header   `yaml:"headers,omitempty"`
+	SrcPaths         []*SrcPath  `yaml:"src_paths,omitempty"`
+	URLPrefix        *URLPrefix  `yaml:"url_prefix,omitempty"`
+	HeadersConf      HeadersConf `yaml:",inline"`
+	RetryStatusCodes []int       `yaml:"retry_status_codes,omitempty"`
 }
 
 // SrcPath represents an src path
@@ -282,6 +291,13 @@ func (sp *SrcPath) MarshalYAML() (interface{}, error) {
 	return sp.sOriginal, nil
 }
 
+var (
+	configReloads      = metrics.NewCounter(`vmauth_config_last_reload_total`)
+	configReloadErrors = metrics.NewCounter(`vmauth_config_last_reload_errors_total`)
+	configSuccess      = metrics.NewCounter(`vmauth_config_last_reload_successful`)
+	configTimestamp    = metrics.NewCounter(`vmauth_config_last_reload_success_timestamp_seconds`)
+)
+
 func initAuthConfig() {
 	if len(*authConfigPath) == 0 {
 		logger.Fatalf("missing required `-auth.config` command-line flag")
@@ -292,11 +308,14 @@ func initAuthConfig() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	err := loadAuthConfig()
+	_, err := loadAuthConfig()
 	if err != nil {
 		logger.Fatalf("cannot load auth config: %s", err)
 	}
 
+	configSuccess.Set(1)
+	configTimestamp.Set(fasttime.UnixTimestamp())
+
 	stopCh = make(chan struct{})
 	authConfigWG.Add(1)
 	go func() {
@@ -319,52 +338,75 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 		refreshCh = ticker.C
 	}
 
+	updateFn := func() {
+		configReloads.Inc()
+		updated, err := loadAuthConfig()
+		if err != nil {
+			logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
+			configSuccess.Set(0)
+			configReloadErrors.Inc()
+			return
+		}
+		configSuccess.Set(1)
+		if updated {
+			configTimestamp.Set(fasttime.UnixTimestamp())
+		}
+	}
+
 	for {
 		select {
 		case <-stopCh:
 			return
 		case <-refreshCh:
-			procutil.SelfSIGHUP()
+			updateFn()
 		case <-sighupCh:
 			logger.Infof("SIGHUP received; loading -auth.config=%q", *authConfigPath)
-			err := loadAuthConfig()
-			if err != nil {
-				logger.Errorf("failed to load auth config; using the last successfully loaded config; error: %s", err)
-				continue
-			}
+			updateFn()
 		}
 	}
 }
 
+// authConfigData stores the yaml definition for this config.
+// authConfigData needs to be updated each time authConfig is updated.
+var authConfigData atomic.Pointer[[]byte]
+
 var authConfig atomic.Pointer[AuthConfig]
 var authUsers atomic.Pointer[map[string]*UserInfo]
 var authConfigWG sync.WaitGroup
 var stopCh chan struct{}
 
-func loadAuthConfig() error {
-	ac, err := readAuthConfig(*authConfigPath)
+// loadAuthConfig loads and applies the config from *authConfigPath.
+// It returns bool value to identify if new config was applied.
+// The config can be not applied if there is a parsing error
+// or if there are no changes to the current authConfig.
+func loadAuthConfig() (bool, error) {
+	data, err := fs.ReadFileOrHTTP(*authConfigPath)
 	if err != nil {
-		return fmt.Errorf("failed to load -auth.config=%q: %s", *authConfigPath, err)
+		return false, fmt.Errorf("failed to read -auth.config=%q: %w", *authConfigPath, err)
+	}
+
+	oldData := authConfigData.Load()
+	if oldData != nil && bytes.Equal(data, *oldData) {
+		// there are no updates in the config - skip reloading.
+		return false, nil
+	}
+
+	ac, err := parseAuthConfig(data)
+	if err != nil {
+		return false, fmt.Errorf("failed to parse -auth.config=%q: %w", *authConfigPath, err)
 	}
 
 	m, err := parseAuthConfigUsers(ac)
 	if err != nil {
-		return fmt.Errorf("failed to parse users from -auth.config=%q: %s", *authConfigPath, err)
+		return false, fmt.Errorf("failed to parse users from -auth.config=%q: %w", *authConfigPath, err)
 	}
 	logger.Infof("loaded information about %d users from -auth.config=%q", len(m), *authConfigPath)
 
 	authConfig.Store(ac)
+	authConfigData.Store(&data)
 	authUsers.Store(&m)
 
-	return nil
-}
-
-func readAuthConfig(path string) (*AuthConfig, error) {
-	data, err := fs.ReadFileOrHTTP(path)
-	if err != nil {
-		return nil, err
-	}
-	return parseAuthConfig(data)
+	return true, nil
 }
 
 func parseAuthConfig(data []byte) (*AuthConfig, error) {

app/vmauth/auth_config_test.go

@@ -299,14 +299,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -325,14 +327,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -389,14 +393,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},
@@ -419,14 +425,16 @@ users:
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
 					}),
-					Headers: []Header{
-						{
-							Name:  "foo",
-							Value: "bar",
-						},
-						{
-							Name:  "xxx",
-							Value: "y",
+					HeadersConf: HeadersConf{
+						RequestHeaders: []Header{
+							{
+								Name:  "foo",
+								Value: "bar",
+							},
+							{
+								Name:  "xxx",
+								Value: "y",
+							},
 						},
 					},
 				},

app/vmauth/example_config.yml

@@ -12,11 +12,16 @@ users:
 
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
+  # The `X-Server-Hostname:` http header is removed from the proxied response.
   # For example, http://vmauth:8427/api/v1/query is proxied to http://localhost:8428/api/v1/query
 - bearer_token: "YYY"
   url_prefix: "http://localhost:8428"
+  # extra headers to add to the request or remove from the request (if header value is empty)
   headers:
   - "X-Scope-OrgID: foobar"
+  # extra headers to add to the response or remove from the response (if header value is empty)
+  response_headers:
+  - "X-Server-Hostname:" # empty value means the header will be removed from the response
 
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are proxied to http://localhost:8428 .
@@ -67,6 +72,7 @@ users:
   #     - http://vmselect2:8481/select/42/prometheus
   #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
   #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+  #   Requests are re-tried at other url_prefix backends if response status codes match 500 or 502.
   #
   # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
   #   The "X-Scope-OrgID: abc" http header is added to these requests.
@@ -85,6 +91,7 @@ users:
     url_prefix:
     - "http://vmselect1:8481/select/42/prometheus"
     - "http://vmselect2:8481/select/42/prometheus"
+    retry_status_codes: [500, 502]
   - src_paths: ["/api/v1/write"]
     url_prefix: "http://vminsert:8480/insert/42/prometheus"
     headers:
@@ -94,11 +101,11 @@ users:
   - "http://default2:8888/unsupported_url_handler"
 
 # Requests without Authorization header are routed according to `unauthorized_user` section.
+# Requests are routed in round-robin fashion between `url_prefix` backends.
+# The deny_partial_response query arg is added to all the routed requests.
+# The requests are re-tried if url_prefix backends send 500 or 503 response status codes.
 unauthorized_user:
-  url_map:
-  - src_paths:
-    - /api/v1/query
-    - /api/v1/query_range
-    url_prefix:
-    - http://vmselect1:8481/select/0/prometheus
-    - http://vmselect2:8481/select/0/prometheus
+  url_prefix:
+  - http://vmselect-az1/?deny_partial_response=1
+  - http://vmselect-az2/?deny_partial_response=1
+  retry_status_codes: [503, 500]

app/vmauth/example_config_ent.yml

@@ -39,18 +39,6 @@ users:
   - "http://default1:8888/unsupported_url_handler"
   - "http://default2:8888/unsupported_url_handler"
 
-# Requests without Authorization header are routed according to `unauthorized_user` section.
-unauthorized_user:
-  url_map:
-  - src_paths:
-    - /api/v1/query
-    - /api/v1/query_range
-    url_prefix:
-    - http://vmselect1:8481/select/0/prometheus
-    - http://vmselect2:8481/select/0/prometheus
-    ip_filters:
-      allow_list: [8.8.8.8]
-
 ip_filters:
   allow_list: ["1.2.3.0/24", "127.0.0.1"]
   deny_list:

app/vmauth/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -41,7 +43,9 @@ var (
 	reloadAuthKey        = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...")
 	logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+
 		`Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`)
-	failTimeout = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend.")
+	failTimeout               = flag.Duration("failTimeout", 3*time.Second, "Sets a delay period for load balancing to skip a malfunctioning backend")
+	maxRequestBodySizeToRetry = flagutil.NewBytes("maxRequestBodySizeToRetry", 16*1024, "The maximum request body size, which can be cached and re-tried at other backends. "+
+		"Bigger values may require more memory")
 )
 
 func main() {
@@ -151,7 +155,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
-	up, headers := ui.getURLPrefixAndHeaders(u)
+	up, hc, retryStatusCodes := ui.getURLPrefixAndHeaders(u)
 	isDefault := false
 	if up == nil {
 		missingRouteRequests.Inc()
@@ -159,14 +163,15 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 			httpserver.Errorf(w, r, "missing route for %q", u.String())
 			return
 		}
-		up, headers = ui.DefaultURL, ui.Headers
+		up, hc, retryStatusCodes = ui.DefaultURL, ui.HeadersConf, ui.RetryStatusCodes
 		isDefault = true
 	}
-	r.Body = &readTrackingBody{
-		r: r.Body,
-	}
-
 	maxAttempts := up.getBackendsCount()
+	if maxAttempts > 1 {
+		r.Body = &readTrackingBody{
+			r: r.Body,
+		}
+	}
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getLeastLoadedBackendURL()
 		targetURL := bu.url
@@ -178,7 +183,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, headers)
+		ok := tryProcessingRequest(w, r, targetURL, hc, retryStatusCodes)
 		bu.put()
 		if ok {
 			return
@@ -192,20 +197,24 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, headers []Header) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
-	for _, h := range headers {
-		req.Header.Set(h.Name, h.Value)
-	}
+	updateHeadersByConfig(req.Header, hc.RequestHeaders)
 	transportOnce.Do(transportInit)
 	res, err := transport.RoundTrip(req)
+	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
-		rtb := req.Body.(*readTrackingBody)
-		if rtb.readStarted {
-			// Request body has been already read, so it is impossible to retry the request.
-			// Return the error to the client then.
+		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+			// Do not retry canceled or timed out requests
+			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+			requestURI := httpserver.GetRequestURI(r)
+			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+			return true
+		}
+		if !rtbOK || !rtb.canRetry() {
+			// Request body cannot be re-sent to another backend. Return the error to the client then.
 			err = &httpserver.ErrorWithStatusCode{
 				Err:        fmt.Errorf("cannot proxy the request to %q: %w", targetURL, err),
 				StatusCode: http.StatusServiceUnavailable,
@@ -216,12 +225,23 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
 		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 		// NOTE: do not use httpserver.GetRequestURI
-		// it explicitly reads request body and fails retries.
-		logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying the request to %q: %s", remoteAddr, req.URL, targetURL, err)
+		// it explicitly reads request body, which may fail retries.
+		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because of response error: %s", remoteAddr, req.URL, targetURL, err)
+		return false
+	}
+	if (rtbOK && rtb.canRetry()) && hasInt(retryStatusCodes, res.StatusCode) {
+		// Retry requests at other backends if it matches retryStatusCodes.
+		// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4893
+		remoteAddr := httpserver.GetQuotedRemoteAddr(r)
+		// NOTE: do not use httpserver.GetRequestURI
+		// it explicitly reads request body, which may fail retries.
+		logger.Warnf("remoteAddr: %s; requestURI: %s; retrying the request to %s because response status code=%d belongs to retry_status_codes=%d",
+			remoteAddr, req.URL, targetURL, res.StatusCode, retryStatusCodes)
 		return false
 	}
 	removeHopHeaders(res.Header)
 	copyHeader(w.Header(), res.Header)
+	updateHeadersByConfig(w.Header(), hc.ResponseHeaders)
 	w.WriteHeader(res.StatusCode)
 
 	copyBuf := copyBufPool.Get()
@@ -237,6 +257,15 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 	return true
 }
 
+func hasInt(a []int, n int) bool {
+	for _, x := range a {
+		if x == n {
+			return true
+		}
+	}
+	return false
+}
+
 var copyBufPool bytesutil.ByteBufferPool
 
 func copyHeader(dst, src http.Header) {
@@ -247,6 +276,16 @@ func copyHeader(dst, src http.Header) {
 	}
 }
 
+func updateHeadersByConfig(headers http.Header, config []Header) {
+	for _, h := range config {
+		if h.Value == "" {
+			headers.Del(h.Name)
+		} else {
+			headers.Set(h.Name, h.Value)
+		}
+	}
+}
+
 func sanitizeRequestHeaders(r *http.Request) *http.Request {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := r.Clone(r.Context())
@@ -361,26 +400,86 @@ func handleConcurrencyLimitError(w http.ResponseWriter, r *http.Request, err err
 }
 
 type readTrackingBody struct {
-	r           io.ReadCloser
-	readStarted bool
+	// r contains reader for initial data reading
+	r io.ReadCloser
+
+	// buf is a buffer for data read from r. Buf size is limited by maxRequestBodySizeToRetry.
+	// If more than maxRequestBodySizeToRetry is read from r, then cannotRetry is set to true.
+	buf []byte
+
+	// cannotRetry is set to true when more than maxRequestBodySizeToRetry are read from r.
+	// In this case the read data cannot fit buf, so it cannot be re-read from buf.
+	cannotRetry bool
+
+	// bufComplete is set to true when buf contains complete request body read from r.
+	bufComplete bool
+
+	// needReadBuf is set to true when Read() must be performed from buf instead of r.
+	needReadBuf bool
+
+	// offset is an offset at buf for the next data read if needReadBuf is set to true.
+	offset int
 }
 
 // Read implements io.Reader interface
 // tracks body reading requests
 func (rtb *readTrackingBody) Read(p []byte) (int, error) {
-	if len(p) > 0 {
-		rtb.readStarted = true
+	if rtb.needReadBuf {
+		if rtb.offset >= len(rtb.buf) {
+			return 0, io.EOF
+		}
+		n := copy(p, rtb.buf[rtb.offset:])
+		rtb.offset += n
+		return n, nil
 	}
-	return rtb.r.Read(p)
+
+	if rtb.r == nil {
+		return 0, fmt.Errorf("cannot read data after closing the reader")
+	}
+
+	n, err := rtb.r.Read(p)
+	if rtb.cannotRetry {
+		return n, err
+	}
+	if len(rtb.buf)+n > maxRequestBodySizeToRetry.IntN() {
+		rtb.cannotRetry = true
+		return n, err
+	}
+	rtb.buf = append(rtb.buf, p[:n]...)
+	if err == io.EOF {
+		rtb.bufComplete = true
+	}
+	return n, err
+}
+
+func (rtb *readTrackingBody) canRetry() bool {
+	if rtb.cannotRetry {
+		return false
+	}
+	if len(rtb.buf) > 0 && !rtb.needReadBuf {
+		return false
+	}
+	return true
 }
 
 // Close implements io.Closer interface.
 func (rtb *readTrackingBody) Close() error {
-	// Close rtb.r only if at least a single Read call was performed.
-	// http.Roundtrip performs body.Close call even without any Read calls
-	// so this hack allows us to reuse request body
-	if rtb.readStarted {
-		return rtb.r.Close()
+	rtb.offset = 0
+	if rtb.bufComplete {
+		rtb.needReadBuf = true
 	}
+
+	// Close rtb.r only if the request body is completely read or if it is too big.
+	// http.Roundtrip performs body.Close call even without any Read calls,
+	// so this hack allows us to reuse request body.
+	if rtb.bufComplete || rtb.cannotRetry {
+		if rtb.r == nil {
+			return nil
+		}
+		err := rtb.r.Close()
+		rtb.r = nil
+		return err
+	}
+
 	return nil
 }

app/vmauth/main_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func TestReadTrackingBodyRetrySuccess(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		rtb := &readTrackingBody{
+			r: io.NopCloser(bytes.NewBufferString(s)),
+		}
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true")
+		}
+		for i := 0; i < 5; i++ {
+			data, err := io.ReadAll(rtb)
+			if err != nil {
+				t.Fatalf("unexpected error when reading all the data at iteration %d: %s", i, err)
+			}
+			if string(data) != s {
+				t.Fatalf("unexpected data read at iteration %d\ngot\n%s\nwant\n%s", i, data, s)
+			}
+			if err := rtb.Close(); err != nil {
+				t.Fatalf("unexpected error when closing readTrackingBody at iteration %d: %s", i, err)
+			}
+			if !rtb.canRetry() {
+				t.Fatalf("canRetry() must return true at iteration %d", i)
+			}
+		}
+	}
+
+	f("")
+	f("foo")
+	f("foobar")
+	f(newTestString(maxRequestBodySizeToRetry.IntN()))
+}
+
+func TestReadTrackingBodyRetryFailure(t *testing.T) {
+	f := func(s string) {
+		t.Helper()
+		rtb := &readTrackingBody{
+			r: io.NopCloser(bytes.NewBufferString(s)),
+		}
+		if !rtb.canRetry() {
+			t.Fatalf("canRetry() must return true")
+		}
+		buf := make([]byte, 1)
+		n, err := rtb.Read(buf)
+		if err != nil {
+			t.Fatalf("unexpected error when reading a single byte: %s", err)
+		}
+		if n != 1 {
+			t.Fatalf("unexpected number of bytes read; got %d; want 1", n)
+		}
+		if rtb.canRetry() {
+			t.Fatalf("canRetry() must return false")
+		}
+		data, err := io.ReadAll(rtb)
+		if err != nil {
+			t.Fatalf("unexpected error when reading all the data: %s", err)
+		}
+		if string(buf)+string(data) != s {
+			t.Fatalf("unexpected data read\ngot\n%s\nwant\n%s", string(buf)+string(data), s)
+		}
+		if err := rtb.Close(); err != nil {
+			t.Fatalf("unexpected error when closing readTrackingBody: %s", err)
+		}
+		if rtb.canRetry() {
+			t.Fatalf("canRetry() must return false")
+		}
+
+		data, err = io.ReadAll(rtb)
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+		if len(data) != 0 {
+			t.Fatalf("unexpected non-empty data read: %q", data)
+		}
+	}
+
+	f(newTestString(maxRequestBodySizeToRetry.IntN() + 1))
+	f(newTestString(2 * maxRequestBodySizeToRetry.IntN()))
+}
+
+func newTestString(sLen int) string {
+	return string(make([]byte, sLen))
+}

app/vmauth/target_url.go

@@ -8,6 +8,9 @@ import (
 
 func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	targetURL := *uiURL
+	if strings.HasPrefix(requestURI.Path, "/") {
+		targetURL.Path = strings.TrimSuffix(targetURL.Path, "/")
+	}
 	targetURL.Path += requestURI.Path
 	requestParams := requestURI.Query()
 	// fast path
@@ -29,18 +32,18 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }
 
-func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, []Header) {
+func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf, []int) {
 	for _, e := range ui.URLMaps {
 		for _, sp := range e.SrcPaths {
 			if sp.match(u.Path) {
-				return e.URLPrefix, e.Headers
+				return e.URLPrefix, e.HeadersConf, e.RetryStatusCodes
 			}
 		}
 	}
 	if ui.URLPrefix != nil {
-		return ui.URLPrefix, ui.Headers
+		return ui.URLPrefix, ui.HeadersConf, ui.RetryStatusCodes
 	}
-	return nil, nil
+	return nil, HeadersConf{}, nil
 }
 
 func normalizeURL(uOrig *url.URL) *url.URL {

app/vmauth/target_url_test.go

@@ -3,18 +3,19 @@ package main
 import (
 	"fmt"
 	"net/url"
+	"reflect"
 	"testing"
 )
 
 func TestCreateTargetURLSuccess(t *testing.T) {
-	f := func(ui *UserInfo, requestURI, expectedTarget, expectedHeaders string) {
+	f := func(ui *UserInfo, requestURI, expectedTarget, expectedRequestHeaders, expectedResponseHeaders string, expectedRetryStatusCodes []int) {
 		t.Helper()
 		u, err := url.Parse(requestURI)
 		if err != nil {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
 		u = normalizeURL(u)
-		up, headers := ui.getURLPrefixAndHeaders(u)
+		up, hc, retryStatusCodes := ui.getURLPrefixAndHeaders(u)
 		if up == nil {
 			t.Fatalf("cannot determie backend: %s", err)
 		}
@@ -24,37 +25,43 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		if target.String() != expectedTarget {
 			t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
 		}
-		headersStr := fmt.Sprintf("%q", headers)
-		if headersStr != expectedHeaders {
-			t.Fatalf("unexpected headers; got %s; want %s", headersStr, expectedHeaders)
+		headersStr := fmt.Sprintf("%q", hc.RequestHeaders)
+		if headersStr != expectedRequestHeaders {
+			t.Fatalf("unexpected request headers; got %s; want %s", headersStr, expectedRequestHeaders)
+		}
+		if !reflect.DeepEqual(retryStatusCodes, expectedRetryStatusCodes) {
+			t.Fatalf("unexpected retryStatusCodes; got %d; want %d", retryStatusCodes, expectedRetryStatusCodes)
 		}
 	}
 	// Simple routing with `url_prefix`
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "", "http://foo.bar/.", "[]")
+	}, "", "http://foo.bar/.", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-		Headers: []Header{{
-			Name:  "bb",
-			Value: "aaa",
-		}},
-	}, "/", "http://foo.bar", `[{"bb" "aaa"}]`)
+		HeadersConf: HeadersConf{
+			RequestHeaders: []Header{{
+				Name:  "bb",
+				Value: "aaa",
+			}},
+		},
+		RetryStatusCodes: []int{503, 501},
+	}, "/", "http://foo.bar", `[{"bb" "aaa"}]`, `[]`, []int{503, 501})
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar/federate"),
-	}, "/", "http://foo.bar/federate", "[]")
+	}, "/", "http://foo.bar/federate", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar"),
-	}, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]")
+	}, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/z", "https://sss:3894/x/y/z", "[]")
+	}, "/z", "https://sss:3894/x/y/z", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/../../aaa", "https://sss:3894/x/y/aaa", "[]")
+	}, "/../../aaa", "https://sss:3894/x/y/aaa", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("https://sss:3894/x/y"),
-	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]")
+	}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]", "[]", nil)
 
 	// Complex routing with `url_map`
 	ui := &UserInfo{
@@ -62,16 +69,25 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
-				Headers: []Header{
-					{
-						Name:  "xx",
-						Value: "aa",
+				HeadersConf: HeadersConf{
+					RequestHeaders: []Header{
+						{
+							Name:  "xx",
+							Value: "aa",
+						},
+						{
+							Name:  "yy",
+							Value: "asdf",
+						},
 					},
-					{
-						Name:  "yy",
-						Value: "asdf",
+					ResponseHeaders: []Header{
+						{
+							Name:  "qwe",
+							Value: "rty",
+						},
 					},
 				},
+				RetryStatusCodes: []int{503, 500, 501},
 			},
 			{
 				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
@@ -79,14 +95,21 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
-		Headers: []Header{{
-			Name:  "bb",
-			Value: "aaa",
-		}},
+		HeadersConf: HeadersConf{
+			RequestHeaders: []Header{{
+				Name:  "bb",
+				Value: "aaa",
+			}},
+			ResponseHeaders: []Header{{
+				Name:  "x",
+				Value: "y",
+			}},
+		},
+		RetryStatusCodes: []int{502},
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`)
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
-	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`)
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501})
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil)
+	f(ui, "/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502})
 
 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{
@@ -102,18 +125,17 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 	}
-	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]")
-	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]")
-	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]")
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]")
-	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]")
+	f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]", "[]", nil)
+	f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]", "[]", nil)
+	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil)
+	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil)
+	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
-	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]")
+	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil)
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"),
-	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]")
-
+	}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]", "[]", nil)
 }
 
 func TestCreateTargetURLFailure(t *testing.T) {
@@ -124,12 +146,18 @@ func TestCreateTargetURLFailure(t *testing.T) {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
 		u = normalizeURL(u)
-		up, headers := ui.getURLPrefixAndHeaders(u)
+		up, hc, retryStatusCodes := ui.getURLPrefixAndHeaders(u)
 		if up != nil {
 			t.Fatalf("unexpected non-empty up=%#v", up)
 		}
-		if headers != nil {
-			t.Fatalf("unexpected non-empty headers=%q", headers)
+		if hc.RequestHeaders != nil {
+			t.Fatalf("unexpected non-empty request headers=%q", hc.RequestHeaders)
+		}
+		if hc.ResponseHeaders != nil {
+			t.Fatalf("unexpected non-empty response headers=%q", hc.ResponseHeaders)
+		}
+		if retryStatusCodes != nil {
+			t.Fatalf("unexpected non-empty retryStatusCodes=%d", retryStatusCodes)
 		}
 	}
 	f(&UserInfo{}, "/foo/bar")

app/vmbackup/README.md

@@ -89,6 +89,23 @@ Do not forget to remove old backups when they are no longer needed in order to s
 
 See also [vmbackupmanager tool](https://docs.victoriametrics.com/vmbackupmanager.html) for automating smart backups.
 
+### Server-side copy of the existing backup
+
+Sometimes it is needed to make server-side copy of the existing backup. This can be done by specifying the source backup path via `-origin` command-line flag,
+while the destination path for backup copy must be specified via `-dst` command-line flag. For example, the following command copies backup
+from `gs://bucket/foo` to `gs://bucket/bar`:
+
+```console
+./vmbackup -origin=gs://bucket/foo -dst=gs://bucket/bar
+```
+
+The `-origin` and `-dst` must point to the same object storage bucket or to the same filesystem.
+
+The server-side backup copy is usually performed at much faster speed comparing to the usual backup, since backup data isn't transferred
+between the remote storage and locally running `vmbackup` tool.
+
+If the `-dst` already contains some data, then its' contents is synced with the `-origin` data. This allows making incremental server-side copies of backups.
+
 ## How does it work?
 
 The backup algorithm is the following:
@@ -126,20 +143,23 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 
 ## Advanced usage
 
-* Obtaining credentials from a file.
 
-  Add flag `-credsFilePath=/etc/credentials` with the following content:
+### Providing credentials as a file
 
-    for s3 (aws, minio or other s3 compatible storages):
+Obtaining credentials from a file.
 
+Add flag `-credsFilePath=/etc/credentials` with the following content:
+
+- for S3 (AWS, MinIO or other S3 compatible storages):
+    
      ```console
      [default]
      aws_access_key_id=theaccesskey
      aws_secret_access_key=thesecretaccesskeyvalue
     ```
 
-    for gce cloud storage:
-
+- for GCP cloud storage:
+    
     ```json
     {
            "type": "service_account",
@@ -154,24 +174,99 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
            "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
     }
     ```
-* Obtaining credentials from env variables.
-    - For AWS S3 compatible storages set env variable `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
-      Also you can set env variable `AWS_SHARED_CREDENTIALS_FILE` with path to credentials file.
-    - For GCE cloud storage set env variable `GOOGLE_APPLICATION_CREDENTIALS` with path to credentials file.
-    - For Azure storage either set env variables `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY`, or `AZURE_STORAGE_ACCOUNT_CONNECTION_STRING`.
 
-* Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
-  You have to add a custom url endpoint via flag:
+### Providing credentials via env variables 
 
-```console
-  # for minio
-  -customS3Endpoint=http://localhost:9000
+Obtaining credentials from env variables.
+- For AWS S3 compatible storages set env variable `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. 
+  Also you can set env variable `AWS_SHARED_CREDENTIALS_FILE` with path to credentials file.
+- For GCE cloud storage set env variable `GOOGLE_APPLICATION_CREDENTIALS` with path to credentials file.
+- For Azure storage either set env variables `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY`, or `AZURE_STORAGE_ACCOUNT_CONNECTION_STRING`.
 
-  # for aws gov region
-  -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+Please, note that `vmbackup` will use credentials provided by cloud providers metadata service [when applicable](https://docs.victoriametrics.com/vmbackup.html#using-cloud-providers-metadata-service).
+
+### Using cloud providers metadata service
+
+`vmbackup` and `vmbackupmanager` will automatically use cloud providers metadata service in order to obtain credentials if they are running in cloud environment
+and credentials are not explicitly provided via flags or env variables.
+
+### Providing credentials in Kubernetes
+
+The simplest way to provide credentials in Kubernetes is to use [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/)
+and inject them into the pod as environment variables. For example, the following secret can be used for AWS S3 credentials:
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vmbackup-credentials
+data:
+  access_key: key
+  secret_key: secret
+```
+And then it can be injected into the pod as environment variables:
+```yaml
+...
+env:
+- name: AWS_ACCESS_KEY_ID
+  valueFrom:
+    secretKeyRef:
+      key: access_key
+      name: vmbackup-credentials
+- name: AWS_SECRET_ACCESS_KEY
+  valueFrom:
+    secretKeyRef:
+      key: secret_key
+      name: vmbackup-credentials
+...
 ```
 
-* Run `vmbackup -help` in order to see all the available options:
+A more secure way is to use IAM roles to provide tokens for pods instead of managing credentials manually. 
+
+For AWS deployments it will be required to configure [IAM roles for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html).
+In order to use IAM roles for service accounts with `vmbackup` or `vmbackupmanager` it is required to create ServiceAccount with IAM role mapping:
+```yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: monitoring-backups
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::{ACCOUNT_ID}:role/{ROLE_NAME}
+```
+And [configure pod to use service account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/).
+After this `vmbackup` and `vmbackupmanager` will automatically use IAM role for service account in order to obtain credentials.
+
+For GCP deployments it will be required to configure [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity).
+In order to use Workload Identity with `vmbackup` or `vmbackupmanager` it is required to create ServiceAccount with Workload Identity annotation:
+```yaml
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: monitoring-backups
+  annotations:
+    iam.gke.io/gcp-service-account: {sa_name}@{project_name}.iam.gserviceaccount.com
+```
+And [configure pod to use service account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/).
+After this `vmbackup` and `vmbackupmanager` will automatically use Workload Identity for servicpe account in order to obtain credentials.
+
+### Using custom S3 endpoint
+
+Usage with s3 custom url endpoint. It is possible to use `vmbackup` with s3 compatible storages like minio, cloudian, etc.
+You have to add a custom url endpoint via flag:
+
+- for MinIO
+    ```console
+      -customS3Endpoint=http://localhost:9000
+    ```
+
+- for aws gov region
+    ```console
+      -customS3Endpoint=https://s3-fips.us-gov-west-1.amazonaws.com
+    ```
+
+### Command-line flags
+
+Run `vmbackup -help` in order to see all the available options:
 
 ```console
   -concurrency int
@@ -196,7 +291,7 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -flagsAuthKey string
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
@@ -225,6 +320,12 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -301,7 +402,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmbackup` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmbackup` binary and puts it into the `bin` folder.
 

app/vmbackup/main.go

@@ -92,8 +92,6 @@ func main() {
 				logger.Fatalf("cannot delete snapshot: %s", err)
 			}
 		}
-	} else if len(*snapshotName) == 0 {
-		logger.Fatalf("`-snapshotName` or `-snapshot.createURL` must be provided")
 	}
 
 	go httpserver.Serve(*httpListenAddr, false, nil)
@@ -113,34 +111,48 @@ func main() {
 }
 
 func makeBackup() error {
-	if err := snapshot.Validate(*snapshotName); err != nil {
-		return fmt.Errorf("invalid -snapshotName=%q: %s", *snapshotName, err)
-	}
-
-	srcFS, err := newSrcFS()
-	if err != nil {
-		return err
-	}
 	dstFS, err := newDstFS()
 	if err != nil {
 		return err
 	}
-	originFS, err := newOriginFS()
-	if err != nil {
-		return err
+	if *snapshotName == "" {
+		// Make server-side copy from -origin to -dst
+		originFS, err := newRemoteOriginFS()
+		if err != nil {
+			return err
+		}
+		a := &actions.RemoteBackupCopy{
+			Concurrency: *concurrency,
+			Src:         originFS,
+			Dst:         dstFS,
+		}
+		if err := a.Run(); err != nil {
+			return err
+		}
+		originFS.MustStop()
+	} else {
+		// Make backup from srcFS to -dst
+		srcFS, err := newSrcFS()
+		if err != nil {
+			return err
+		}
+		originFS, err := newOriginFS()
+		if err != nil {
+			return err
+		}
+		a := &actions.Backup{
+			Concurrency: *concurrency,
+			Src:         srcFS,
+			Dst:         dstFS,
+			Origin:      originFS,
+		}
+		if err := a.Run(); err != nil {
+			return err
+		}
+		srcFS.MustStop()
+		originFS.MustStop()
 	}
-	a := &actions.Backup{
-		Concurrency: *concurrency,
-		Src:         srcFS,
-		Dst:         dstFS,
-		Origin:      originFS,
-	}
-	if err := a.Run(); err != nil {
-		return err
-	}
-	srcFS.MustStop()
 	dstFS.MustStop()
-	originFS.MustStop()
 	return nil
 }
 
@@ -155,6 +167,9 @@ See the docs at https://docs.victoriametrics.com/vmbackup.html .
 }
 
 func newSrcFS() (*fslocal.FS, error) {
+	if err := snapshot.Validate(*snapshotName); err != nil {
+		return nil, fmt.Errorf("invalid -snapshotName=%q: %s", *snapshotName, err)
+	}
 	snapshotPath := filepath.Join(*storageDataPath, "snapshots", *snapshotName)
 
 	// Verify the snapshot exists.
@@ -205,7 +220,20 @@ func hasFilepathPrefix(path, prefix string) bool {
 	if err != nil {
 		return false
 	}
-	return strings.HasPrefix(pathAbs, prefixAbs)
+	if prefixAbs == pathAbs {
+		return true
+	}
+	rel, err := filepath.Rel(prefixAbs, pathAbs)
+	if err != nil {
+		// if paths can't be related - they don't match
+		return false
+	}
+	if i := strings.Index(rel, "."); i == 0 {
+		// if path can be related only with . as first char - they still don't match
+		return false
+	}
+	// if paths are related - it is a match
+	return true
 }
 
 func newOriginFS() (common.OriginFS, error) {
@@ -218,3 +246,14 @@ func newOriginFS() (common.OriginFS, error) {
 	}
 	return fs, nil
 }
+
+func newRemoteOriginFS() (common.RemoteFS, error) {
+	if len(*origin) == 0 {
+		return nil, fmt.Errorf("-origin cannot be empty when -snapshotName and -snapshot.createURL aren't set")
+	}
+	fs, err := actions.NewRemoteFS(*origin)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse `-origin`=%q: %w", *origin, err)
+	}
+	return fs, nil
+}

app/vmbackup/main_test.go

@@ -26,4 +26,9 @@ func TestHasFilepathPrefix(t *testing.T) {
 	f("fs://"+pwd+"/foo", pwd+"/foo/bar", false)
 	f("fs://"+pwd+"/foo/bar", pwd+"/foo", true)
 	f("fs://"+pwd+"/foo", pwd+"/bar", false)
+	f("fs:///data1", "/data", false)
+	f("fs:///data", "/data1", false)
+	f("fs:///data", "/data/foo", false)
+	f("fs:///data/foo", "/data", true)
+	f("fs:///data/foo/", "/data/", true)
 }

app/vmbackupmanager/README.md

@@ -110,6 +110,9 @@ The result on the GCS bucket
 
   <img alt="latest folder" src="vmbackupmanager_latest_folder.png">
 
+Please, see [vmbackup docs](https://docs.victoriametrics.com/vmbackup.html#advanced-usage) for more examples of authentication with different
+storage types.
+
 ## Backup Retention Policy
 
 Backup retention policy is controlled by:
@@ -429,7 +432,7 @@ command-line flags:
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -flagsAuthKey string
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
@@ -466,6 +469,12 @@ command-line flags:
      Keep last N monthly backups. If 0 is specified next retention cycle removes all backups for given time period. (default -1)
   -keepLastWeekly int
      Keep last N weekly backups. If 0 is specified next retention cycle removes all backups for given time period. (default -1)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int

app/vmctl/README.md

@@ -11,6 +11,7 @@ Features:
 - migrate data from [Mimir](#migrating-data-from-mimir) to VictoriaMetrics
 - migrate data from [InfluxDB](#migrating-data-from-influxdb-1x) to VictoriaMetrics
 - migrate data from [OpenTSDB](#migrating-data-from-opentsdb) to VictoriaMetrics
+- migrate data from [Promscale](#migrating-data-from-promscale)
 - migrate data between [VictoriaMetrics](#migrating-data-from-victoriametrics) single or cluster version.
 - migrate data by [Prometheus remote read protocol](#migrating-data-by-remote-read-protocol) to VictoriaMetrics
 - [verify](#verifying-exported-blocks-from-victoriametrics) exported blocks from VictoriaMetrics single or cluster version.
@@ -306,6 +307,46 @@ Please see more about time filtering [here](https://docs.influxdata.com/influxdb
 Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
 You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
 
+## Migrating data from Promscale
+
+[Promscale](https://github.com/timescale/promscale) supports [Prometheus Remote Read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+To migrate historical data from Promscale to VictoriaMetrics we recommend using `vmctl`
+in [remote-read](https://docs.victoriametrics.com/vmctl.html#migrating-data-by-remote-read-protocol) mode.
+
+See the example of migration command below:
+```console
+./vmctl remote-read --remote-read-src-addr=http://<promscale>:9201/read \
+                    --remote-read-step-interval=day \
+                    --remote-read-use-stream=false \ # promscale doesn't support streaming
+                    --vm-addr=http://<victoriametrics>:8428 \
+                    --remote-read-filter-time-start=2023-08-21T00:00:00Z \
+                    --remote-read-disable-path-append=true # promscale has custom remote read API HTTP path
+Selected time range "2023-08-21 00:00:00 +0000 UTC" - "2023-08-21 14:11:41.561979 +0000 UTC" will be split into 1 ranges according to "day" step. Continue? [Y/n] y
+VM worker 0:↙ 82831 samples/s                                                                                                                                                                        
+VM worker 1:↙ 54378 samples/s                                                                                                                                                                        
+VM worker 2:↙ 121616 samples/s                                                                                                                                                                       
+VM worker 3:↙ 59164 samples/s                                                                                                                                                                        
+VM worker 4:↙ 59220 samples/s                                                                                                                                                                        
+VM worker 5:↙ 102072 samples/s                                                                                                                                                                       
+Processing ranges: 1 / 1 [██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/08/21 16:11:55 Import finished!
+2023/08/21 16:11:55 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 14.047045459s;
+  total samples: 262111;
+  samples/s: 18659.51;
+  total bytes: 5.3 MB;
+  bytes/s: 376.4 kB;
+  import requests: 6;
+  import requests retries: 0;
+2023/08/21 16:11:55 Total time: 14.063458792s
+```
+
+Here we specify the full path to Promscale's Remote Read API via `--remote-read-src-addr`, and disable auto-path 
+appending via `--remote-read-disable-path-append` cmd-line flags. This is necessary, as Promscale has a different to 
+Prometheus API path. Promscale doesn't support stream mode for Remote Read API,
+so we disable it via `--remote-read-use-stream=false`. 
+
 ## Migrating data from Prometheus
 
 `vmctl` supports the `prometheus` mode for migrating data from Prometheus to VictoriaMetrics time-series database.
@@ -438,28 +479,34 @@ Found 2 blocks to import. Continue? [Y/n] y
 
 ## Migrating data by remote read protocol
 
-`vmctl` supports the `remote-read` mode for migrating data from databases which support 
-[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/)
+`vmctl` provides the `remote-read` mode for migrating data from remote databases supporting 
+[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+Remote read API has two implementations of remote read API: default (`SAMPLES`) and 
+[streamed](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/) (`STREAMED_XOR_CHUNKS`).
+Streamed version is more efficient but has lower adoption (e.g. [Promscale](#migrating-data-from-promscale)
+doesn't support it).
 
 See `./vmctl remote-read --help` for details and full list of flags.
 
 To start the migration process configure the following flags:
+
 1. `--remote-read-src-addr` - data source address to read from;
 1. `--vm-addr` - VictoriaMetrics address to write to. For single-node VM is usually equal to `--httpListenAddr`, 
    and for cluster version is equal to `--httpListenAddr` flag of vminsert component (for example `http://<vminsert>:8480/insert/<accountID>/prometheus`);
 1. `--remote-read-filter-time-start` - the time filter in RFC3339 format to select time series with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z';
 1. `--remote-read-filter-time-end` - the time filter in RFC3339 format to select time series with timestamp equal or smaller than provided value. E.g. '2020-01-01T20:07:00Z'. Current time is used when omitted.;
 1. `--remote-read-step-interval` - split export data into chunks. Valid values are `month, day, hour, minute`;
+1. `--remote-read-use-stream` - defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. By default, is uses `SAMPLES` mode.
 
 The importing process example for local installation of Prometheus
 and single-node VictoriaMetrics(`http://localhost:8428`):
 
 ```
 ./vmctl remote-read \
---remote-read-src-addr=http://127.0.0.1:9091 \
+--remote-read-src-addr=http://<prometheus>:9091 \
 --remote-read-filter-time-start=2021-10-18T00:00:00Z \
 --remote-read-step-interval=hour \
---vm-addr=http://127.0.0.1:8428 \
+--vm-addr=http://<victoria-metrics>:8428 \
 --vm-concurrency=6
 
 Split defined times into 8798 ranges to import. Continue? [Y/n]
@@ -536,6 +583,7 @@ then import it into VM using `vmctl` in `prometheus` mode.
     1. Run the `minio/mc` Docker container.
     1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
     1. `mc cp -r minio/prometheus thanos-data`
+
 1. Import using `vmctl`.
     1. Follow the [instructions](#how-to-build) to compile `vmctl` on your machine.
     1. Use [prometheus](#migrating-data-from-prometheus) mode to import data:
@@ -553,7 +601,7 @@ service (or anything that exposes gRPC StoreAPI e.g. Querier) via Prometheus rem
 If you want to migrate data, you should run [thanos-remote-read](https://github.com/G-Research/thanos-remote-read) proxy
 and define the Thanos store address `./thanos-remote-read -store 127.0.0.1:19194`. 
 It is important to know that `store` flag is Thanos Store API gRPC endpoint. 
-Also, it is important to know that thanos-remote-read proxy  doesn't support `STREAMED_XOR_CHUNKS` mode.
+Also, it is important to know that thanos-remote-read proxy doesn't support stream mode.
 When you run thanos-remote-read proxy, it exposes port to serve HTTP on `10080 by default`.
 
 The importing process example for local installation of Thanos
@@ -616,7 +664,7 @@ api:
 If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
 By default, Cortex uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
 
-It is important to know that Cortex doesn't support the `STREAMED_XOR_CHUNKS` mode.
+It is important to know that Cortex doesn't support the stream mode.
 When you run Cortex, it exposes a port to serve HTTP on `9009 by default`.
 
 The importing process example for the local installation of Cortex
@@ -658,26 +706,24 @@ requires an Authentication header like `X-Scope-OrgID`. You can define it via th
 
 ## Migrating data from Mimir
 
-Mimir has similar implementation as Cortex and also support of the Prometheus remote read protocol. That means
-`vmctl` in mode `remote-read` may also be used for Mimir historical data migration.
-These instructions may vary based on the details of your Mimir configuration.
+Mimir has similar implementation as Cortex and supports Prometheus remote read API. That means historical data
+from Mimir can be migrated via `vmctl` in mode `remote-read` mode.
+The instructions for data migration via vmctl vary based on the details of your Mimir configuration.
 Please read carefully and verify as you go.
 
 ### Remote read protocol
 
-If you want to migrate data, you should check your Mimir configuration in the section
-```yaml
-api:
-  prometheus_http_prefix:
-```
+By default, Mimir uses the `prometheus` path prefix so specifying the source
+should be as simple as `--remote-read-src-addr=http://<mimir>:9009/prometheus`.
+But if prefix was overriden via `prometheus_http_prefix`, then source address should be updated
+to `--remote-read-src-addr=http://<mimir>:9009/{prometheus_http_prefix}`.
 
-If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
-By default, Mimir uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
+Mimir supports [streamed remote read API](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/),
+so it is recommended setting `--remote-read-use-stream=true` flag for better performance and resource usage.
 
-Mimir supports both remote read mode, so you can use `STREAMED_XOR_CHUNKS` mode and `SAMPLES` mode.
 When you run Mimir, it exposes a port to serve HTTP on `8080 by default`.
 
-Next example of the local installation was in multi-tenant mode (3 instances of mimir) with nginx as load balancer.
+Next example of the local installation was in multi-tenant mode (3 instances of Mimir) with nginx as load balancer.
 Load balancer expose single port `:9090`.
 
 As you can see in the example we call `:9009` instead of `:8080` because of proxy.
@@ -687,13 +733,12 @@ and single-node VictoriaMetrics(`http://localhost:8428`):
 
 ```
 ./vmctl remote-read 
---remote-read-src-addr=http://127.0.0.1:9009/prometheus \
+--remote-read-src-addr=http://<mimir>:9009/prometheus \
 --remote-read-filter-time-start=2021-10-18T00:00:00Z \
 --remote-read-step-interval=hour \
 --remote-read-headers=X-Scope-OrgID:demo \
 --remote-read-use-stream=true \
---vm-addr=http://127.0.0.1:8428 \
---vm-concurrency=6
+--vm-addr=http://<victoria-metrics>:8428 \
 ```
 
 And when the process finishes, you will see the following:
@@ -826,7 +871,7 @@ It is possible split migration process into set of smaller batches based on time
 migrating large volumes of data as this adds indication of progress and ability to restore process from certain point 
 in case of failure.
 
-To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `day`, `hour`, `minute`.
+To use this you need to specify `--vm-native-step-interval` flag. Supported values are: `month`, `week`, `day`, `hour`, `minute`.
 Note that in order to use this it is required `--vm-native-filter-time-start` to be set to calculate time ranges for 
 export process.
 
@@ -836,7 +881,7 @@ Every range is being processed independently, which means that:
 so it is possible to restart process starting from failed range.
 
 It is recommended using the `month` step when migrating the data over multiple months, 
-since the migration with `day` and `hour` steps may take longer time to complete because of additional overhead.
+since the migration with `week`, `day` and `hour` steps may take longer time to complete because of additional overhead.
 
 Usage example:
 ```console
@@ -1023,7 +1068,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmctl` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmctl` binary and puts it into the `bin` folder.
 
@@ -1052,7 +1097,7 @@ ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://b
 
 #### Development ARM build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmctl-linux-arm` or `make vmctl-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmctl-linux-arm` or `vmctl-linux-arm64` binary respectively and puts it into the `bin` folder.
 

app/vmctl/flags.go

@@ -361,8 +361,9 @@ var (
 			Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats",
 		},
 		&cli.StringFlag{
-			Name:  vmNativeStepInterval,
-			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s'.", vmNativeFilterTimeStart, stepper.StepMonth, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
+			Name: vmNativeStepInterval,
+			Usage: fmt.Sprintf("Split export data into chunks. Requires setting --%s. Valid values are '%s','%s','%s','%s','%s'.", vmNativeFilterTimeStart,
+				stepper.StepMonth, stepper.StepWeek, stepper.StepDay, stepper.StepHour, stepper.StepMinute),
 			Value: stepper.StepMonth,
 		},
 		&cli.BoolFlag{
@@ -556,8 +557,8 @@ var (
 		},
 		&cli.BoolFlag{
 			Name:  remoteReadDisablePathAppend,
-			Usage: "Whether to disable automatic appending of the path to the remote storage.",
-			Value: true,
+			Usage: "Whether to disable automatic appending of the /api/v1/read suffix to --remote-read-src-addr",
+			Value: false,
 		},
 	}
 )

app/vmctl/opentsdb.go

@@ -170,8 +170,5 @@ func (op *otsdbProcessor) do(s queryObj) error {
 		Timestamps: data.Timestamps,
 		Values:     data.Values,
 	}
-	if err := op.im.Input(&ts); err != nil {
-		return err
-	}
-	return nil
+	return op.im.Input(&ts)
 }

app/vmctl/opentsdb/parser.go

@@ -180,7 +180,7 @@ func modifyData(msg Metric, normalize bool) (Metric, error) {
 	/*
 		replace bad characters in metric name with _ per the data model
 	*/
-	finalMsg.Metric = promrelabel.SanitizeName(name)
+	finalMsg.Metric = promrelabel.SanitizeMetricName(name)
 	// replace bad characters in tag keys with _ per the data model
 	for key, value := range msg.Tags {
 		// if normalization requested, lowercase the key and value
@@ -191,7 +191,7 @@ func modifyData(msg Metric, normalize bool) (Metric, error) {
 		/*
 			replace all explicitly bad characters with _
 		*/
-		key = promrelabel.SanitizeName(key)
+		key = promrelabel.SanitizeLabelName(key)
 		// tags that start with __ are considered custom stats for internal prometheus stuff, we should drop them
 		if !strings.HasPrefix(key, "__") {
 			finalMsg.Tags[key] = value

app/vmctl/stepper/split.go

@@ -8,10 +8,10 @@ import (
 const (
 	// StepMonth represents a one month interval
 	StepMonth string = "month"
-	// StepDay represents a one day interval
-	StepDay string = "day"
 	// StepWeek represents a one week interval
 	StepWeek string = "week"
+	// StepDay represents a one day interval
+	StepDay string = "day"
 	// StepHour represents a one hour interval
 	StepHour string = "hour"
 	// StepMinute represents a one minute interval

app/vmctl/vm_native.go

@@ -201,7 +201,12 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		}
 
 		if len(metrics) == 0 {
-			return fmt.Errorf("no metrics found")
+			errMsg := "no metrics found"
+			if tenantID != "" {
+				errMsg = fmt.Sprintf("%s for tenant id: %s", errMsg, tenantID)
+			}
+			log.Println(errMsg)
+			return nil
 		}
 		foundSeriesMsg = fmt.Sprintf("Found %d metrics to import", len(metrics))
 	}

app/vmgateway/README.md

@@ -338,7 +338,7 @@ The shortlist of configuration flags include the following:
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -flagsAuthKey string
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
@@ -369,6 +369,12 @@ The shortlist of configuration flags include the following:
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int

app/vminsert/common/insert_ctx.go

@@ -171,7 +171,7 @@ func (ctx *InsertCtx) dropAggregatedRows(matchIdxs []byte) {
 	src := ctx.mrs
 	if !*streamAggrDropInput {
 		for idx, match := range matchIdxs {
-			if match != 0 {
+			if match == 1 {
 				continue
 			}
 			dst = append(dst, src[idx])

app/vminsert/opentelemetry/request_handler.go

@@ -1,6 +1,7 @@
 package opentelemetry
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/common"
@@ -23,6 +24,9 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
+	if req.Header.Get("Content-Type") == "application/json" {
+		return fmt.Errorf("json encoding isn't supported for opentelemetry format. Use protobuf encoding")
+	}
 	return stream.ParseStream(req.Body, isGzipped, func(tss []prompbmarshal.TimeSeries) error {
 		return insertRows(tss, extraLabels)
 	})

app/vminsert/prometheusimport/request_handler.go

@@ -29,7 +29,7 @@ func InsertHandler(req *http.Request) error {
 		return err
 	}
 	isGzipped := req.Header.Get("Content-Encoding") == "gzip"
-	return stream.Parse(req.Body, defaultTimestamp, isGzipped, func(rows []parser.Row) error {
+	return stream.Parse(req.Body, defaultTimestamp, isGzipped, true, func(rows []parser.Row) error {
 		return insertRows(rows, extraLabels)
 	}, func(s string) {
 		httpserver.LogError(req, s)

app/vminsert/relabel/relabel.go

@@ -134,9 +134,9 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label {
 		for i := range tmpLabels {
 			label := &tmpLabels[i]
 			if label.Name == "__name__" {
-				label.Value = promrelabel.SanitizeName(label.Value)
+				label.Value = promrelabel.SanitizeMetricName(label.Value)
 			} else {
-				label.Name = promrelabel.SanitizeName(label.Name)
+				label.Name = promrelabel.SanitizeLabelName(label.Name)
 			}
 		}
 	}

app/vmrestore/README.md

@@ -100,7 +100,7 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
   -envflag.prefix string
      Prefix for environment variables if -envflag.enable is set
   -eula
-     By specifying this flag, you confirm that you have an enterprise license and accept the EULA https://victoriametrics.com/assets/VM_EULA.pdf . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+     Deprecated, please use -license or -licenseFile flags instead. By specifying this flag, you confirm that you have an enterprise license and accept the ESA https://victoriametrics.com/legal/esa/ . This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -flagsAuthKey string
      Auth key for /flags endpoint. It must be passed via authKey query arg. It overrides httpAuth.* settings
   -fs.disableMmap
@@ -129,6 +129,12 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
      Whether to disable caches for interned strings. This may reduce memory usage at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringCacheExpireDuration and -internStringMaxLen
   -internStringMaxLen int
      The maximum length for strings to intern. A lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning . See also -internStringDisableCache and -internStringCacheExpireDuration (default 500)
+  -license string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -license.forceOffline
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
+  -licenseFile string
+     See https://victoriametrics.com/products/enterprise/ for trial license. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html
   -loggerDisableTimestamps
      Whether to disable writing timestamps in logs
   -loggerErrorsPerSecondLimit int
@@ -201,7 +207,7 @@ It is recommended using [binary releases](https://github.com/VictoriaMetrics/Vic
 
 ### Development build
 
-1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.19.
+1. [Install Go](https://golang.org/doc/install). The minimum supported version is Go 1.20.
 1. Run `make vmrestore` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
    It builds `vmrestore` binary and puts it into the `bin` folder.
 

app/vmselect/graphite/aggr_state.go

@@ -114,7 +114,7 @@ func (as *aggrStateAvgZero) Update(values []float64) {
 	as.seriesTotal++
 }
 
-func (as *aggrStateAvgZero) Finalize(xFilesFactor float64) []float64 {
+func (as *aggrStateAvgZero) Finalize(_ float64) []float64 {
 	sums := as.sums
 	values := make([]float64, as.pointsLen)
 	count := float64(as.seriesTotal)

app/vmselect/graphite/functions_api.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputils"
 )
@@ -14,7 +13,7 @@ import (
 // FunctionsHandler implements /functions handler.
 //
 // See https://graphite.readthedocs.io/en/latest/functions.html#function-api
-func FunctionsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
+func FunctionsHandler(w http.ResponseWriter, r *http.Request) error {
 	grouped := httputils.GetBool(r, "grouped")
 	group := r.FormValue("group")
 	result := make(map[string]interface{})
@@ -40,7 +39,7 @@ func FunctionsHandler(startTime time.Time, w http.ResponseWriter, r *http.Reques
 // FunctionDetailsHandler implements /functions/<func_name> handler.
 //
 // See https://graphite.readthedocs.io/en/latest/functions.html#function-api
-func FunctionDetailsHandler(startTime time.Time, funcName string, w http.ResponseWriter, r *http.Request) error {
+func FunctionDetailsHandler(funcName string, w http.ResponseWriter, r *http.Request) error {
 	result := funcs[funcName]
 	if result == nil {
 		return fmt.Errorf("cannot find function %q", funcName)

app/vmselect/graphite/metrics_api.go

@@ -85,7 +85,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	if leavesOnly {
 		paths = filterLeaves(paths, delimiter)
 	}
-	paths = deduplicatePaths(paths, delimiter)
+	paths = deduplicatePaths(paths)
 	sortPaths(paths, delimiter)
 	contentType := getContentType(jsonp)
 	w.Header().Set("Content-Type", contentType)
@@ -99,7 +99,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ
 	return nil
 }
 
-func deduplicatePaths(paths []string, delimiter string) []string {
+func deduplicatePaths(paths []string) []string {
 	if len(paths) == 0 {
 		return nil
 	}

app/vmselect/graphite/render_response.qtpl

@@ -47,7 +47,7 @@ See https://graphite.readthedocs.io/en/stable/render_api.html#json
 			{% code timestamps := s.Timestamps %}
 			{% for i, v := range s.Values %}
 				[
-					{% if math.IsNaN(v) %}null{% else %}{%f= v %}{% endif %},
+					{% if math.IsNaN(v) || math.IsInf(v, 0) %}null{% else %}{%f= v %}{% endif %},
 					{%dl= timestamps[i]/1e3 %}
 				]
 				{% if i+1 < len(timestamps) %},{% endif %}

app/vmselect/graphite/render_response.qtpl.go

@@ -148,7 +148,7 @@ func streamrenderSeriesJSON(qw422016 *qt422016.Writer, s *series) {
 //line app/vmselect/graphite/render_response.qtpl:48
 		qw422016.N().S(`[`)
 //line app/vmselect/graphite/render_response.qtpl:50
-		if math.IsNaN(v) {
+		if math.IsNaN(v) || math.IsInf(v, 0) {
 //line app/vmselect/graphite/render_response.qtpl:50
 			qw422016.N().S(`null`)
 //line app/vmselect/graphite/render_response.qtpl:50

app/vmselect/graphite/transform.go

@@ -189,7 +189,7 @@ func init() {
 	}
 }
 
-func transformTODO(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, error) {
+func transformTODO(_ *evalConfig, _ *graphiteql.FuncExpr) (nextSeriesFunc, error) {
 	return nil, fmt.Errorf("TODO: implement this function")
 }
 
@@ -1062,7 +1062,7 @@ func transformCumulative(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFun
 	if err != nil {
 		return nil, err
 	}
-	return consolidateBy(ec, fe, nextSeries, "sum")
+	return consolidateBy(fe, nextSeries, "sum")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.consolidateBy
@@ -1079,10 +1079,10 @@ func transformConsolidateBy(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeries
 	if err != nil {
 		return nil, err
 	}
-	return consolidateBy(ec, fe, nextSeries, funcName)
+	return consolidateBy(fe, nextSeries, funcName)
 }
 
-func consolidateBy(ec *evalConfig, expr graphiteql.Expr, nextSeries nextSeriesFunc, funcName string) (nextSeriesFunc, error) {
+func consolidateBy(expr graphiteql.Expr, nextSeries nextSeriesFunc, funcName string) (nextSeriesFunc, error) {
 	consolidateFunc, err := getAggrFunc(funcName)
 	if err != nil {
 		return nil, err
@@ -1843,10 +1843,10 @@ func transformHighest(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc,
 	if err != nil {
 		return nil, err
 	}
-	return highestGeneric(ec, fe, nextSeries, n, funcName)
+	return highestGeneric(fe, nextSeries, n, funcName)
 }
 
-func highestGeneric(ec *evalConfig, expr graphiteql.Expr, nextSeries nextSeriesFunc, n float64, funcName string) (nextSeriesFunc, error) {
+func highestGeneric(expr graphiteql.Expr, nextSeries nextSeriesFunc, n float64, funcName string) (nextSeriesFunc, error) {
 	aggrFunc, err := getAggrFunc(funcName)
 	if err != nil {
 		_, _ = drainAllSeries(nextSeries)
@@ -1928,7 +1928,7 @@ func transformHighestAverage(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSerie
 	if err != nil {
 		return nil, err
 	}
-	return highestGeneric(ec, fe, nextSeries, n, "average")
+	return highestGeneric(fe, nextSeries, n, "average")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.highestCurrent
@@ -1945,7 +1945,7 @@ func transformHighestCurrent(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSerie
 	if err != nil {
 		return nil, err
 	}
-	return highestGeneric(ec, fe, nextSeries, n, "current")
+	return highestGeneric(fe, nextSeries, n, "current")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.highestMax
@@ -1962,7 +1962,7 @@ func transformHighestMax(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFun
 	if err != nil {
 		return nil, err
 	}
-	return highestGeneric(ec, fe, nextSeries, n, "max")
+	return highestGeneric(fe, nextSeries, n, "max")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.hitcount
@@ -2379,10 +2379,10 @@ func transformLowest(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, e
 	if err != nil {
 		return nil, err
 	}
-	return lowestGeneric(ec, fe, nextSeries, n, funcName)
+	return lowestGeneric(fe, nextSeries, n, funcName)
 }
 
-func lowestGeneric(ec *evalConfig, expr graphiteql.Expr, nextSeries nextSeriesFunc, n float64, funcName string) (nextSeriesFunc, error) {
+func lowestGeneric(expr graphiteql.Expr, nextSeries nextSeriesFunc, n float64, funcName string) (nextSeriesFunc, error) {
 	aggrFunc, err := getAggrFunc(funcName)
 	if err != nil {
 		_, _ = drainAllSeries(nextSeries)
@@ -2459,7 +2459,7 @@ func transformLowestAverage(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeries
 	if err != nil {
 		return nil, err
 	}
-	return lowestGeneric(ec, fe, nextSeries, n, "average")
+	return lowestGeneric(fe, nextSeries, n, "average")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.lowestCurrent
@@ -2476,7 +2476,7 @@ func transformLowestCurrent(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeries
 	if err != nil {
 		return nil, err
 	}
-	return lowestGeneric(ec, fe, nextSeries, n, "current")
+	return lowestGeneric(fe, nextSeries, n, "current")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.maxSeries
@@ -2607,7 +2607,7 @@ func transformMostDeviant(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFu
 	if err != nil {
 		return nil, err
 	}
-	return highestGeneric(ec, fe, nextSeries, n, "stddev")
+	return highestGeneric(fe, nextSeries, n, "stddev")
 }
 
 // See https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.movingAverage
@@ -3862,7 +3862,11 @@ func nextSeriesConcurrentWrapper(nextSeries nextSeriesFunc, f func(s *series) (*
 		}
 		if r.err != nil {
 			// Drain the rest of series before returning the error.
-			for range resultCh {
+			for {
+				_, ok := <-resultCh
+				if !ok {
+					break
+				}
 			}
 			<-errCh
 			return nil, r.err
@@ -4733,7 +4737,7 @@ func transformSortByTotal(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFu
 	if err != nil {
 		return nil, err
 	}
-	return sortByGeneric(ec, fe, nextSeries, "sum", true)
+	return sortByGeneric(fe, nextSeries, "sum", true)
 }
 
 // https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.sortBy
@@ -4754,10 +4758,10 @@ func transformSortBy(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesFunc, e
 	if err != nil {
 		return nil, err
 	}
-	return sortByGeneric(ec, fe, nextSeries, funcName, reverse)
+	return sortByGeneric(fe, nextSeries, funcName, reverse)
 }
 
-func sortByGeneric(ec *evalConfig, fe *graphiteql.FuncExpr, nextSeries nextSeriesFunc, funcName string, reverse bool) (nextSeriesFunc, error) {
+func sortByGeneric(fe *graphiteql.FuncExpr, nextSeries nextSeriesFunc, funcName string, reverse bool) (nextSeriesFunc, error) {
 	aggrFunc, err := getAggrFunc(funcName)
 	if err != nil {
 		_, _ = drainAllSeries(nextSeries)
@@ -4868,7 +4872,7 @@ func transformSortByMinima(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesF
 		}
 		return s, nil
 	})
-	return sortByGeneric(ec, fe, f, "min", false)
+	return sortByGeneric(fe, f, "min", false)
 }
 
 // https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.sortByMaxima
@@ -4881,7 +4885,7 @@ func transformSortByMaxima(ec *evalConfig, fe *graphiteql.FuncExpr) (nextSeriesF
 	if err != nil {
 		return nil, err
 	}
-	return sortByGeneric(ec, fe, nextSeries, "max", true)
+	return sortByGeneric(fe, nextSeries, "max", true)
 }
 
 // https://graphite.readthedocs.io/en/stable/functions.html#graphite.render.functions.smartSummarize
@@ -5286,7 +5290,7 @@ func holtWinterConfidenceBands(ec *evalConfig, fe *graphiteql.FuncExpr, args []*
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		s.consolidate(&ecCopy, step)
 		timeStamps := s.Timestamps[trimWindowPoints:]
-		analysis := holtWintersAnalysis(&ecCopy, s, seasonalityMs)
+		analysis := holtWintersAnalysis(s, seasonalityMs)
 		forecastValues := analysis.predictions.Values[trimWindowPoints:]
 		deviationValues := analysis.deviations.Values[trimWindowPoints:]
 		valuesLen := len(forecastValues)
@@ -5450,7 +5454,7 @@ func transformHoltWintersForecast(ec *evalConfig, fe *graphiteql.FuncExpr) (next
 	trimWindowPoints := ecCopy.pointsLen(step) - ec.pointsLen(step)
 	f := nextSeriesConcurrentWrapper(nextSeries, func(s *series) (*series, error) {
 		s.consolidate(&ecCopy, step)
-		analysis := holtWintersAnalysis(&ecCopy, s, seasonalityMs)
+		analysis := holtWintersAnalysis(s, seasonalityMs)
 		predictions := analysis.predictions
 
 		s.Tags["holtWintersForecast"] = "1"
@@ -5468,7 +5472,7 @@ func transformHoltWintersForecast(ec *evalConfig, fe *graphiteql.FuncExpr) (next
 
 }
 
-func holtWintersAnalysis(ec *evalConfig, s *series, seasonality int64) holtWintersAnalysisResult {
+func holtWintersAnalysis(s *series, seasonality int64) holtWintersAnalysisResult {
 	alpha := 0.1
 	gamma := alpha
 	beta := 0.0035

app/vmselect/main.go

@@ -236,7 +236,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		funcName = strings.TrimPrefix(funcName, "/")
 		if funcName == "" {
 			graphiteFunctionsRequests.Inc()
-			if err := graphite.FunctionsHandler(startTime, w, r); err != nil {
+			if err := graphite.FunctionsHandler(w, r); err != nil {
 				graphiteFunctionsErrors.Inc()
 				httpserver.Errorf(w, r, "%s", err)
 				return true
@@ -244,7 +244,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 			return true
 		}
 		graphiteFunctionDetailsRequests.Inc()
-		if err := graphite.FunctionDetailsHandler(startTime, funcName, w, r); err != nil {
+		if err := graphite.FunctionDetailsHandler(funcName, w, r); err != nil {
 			graphiteFunctionDetailsErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -480,6 +480,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		expandWithExprsRequests.Inc()
 		prometheus.ExpandWithExprs(w, r)
 		return true
+	case "/prettify-query":
+		prettifyQueryRequests.Inc()
+		prometheus.PrettifyQuery(w, r)
+		return true
 	case "/api/v1/rules", "/rules":
 		rulesRequests.Inc()
 		if len(*vmalertProxyURL) > 0 {
@@ -655,6 +659,7 @@ var (
 	graphiteFunctionDetailsErrors   = metrics.NewCounter(`vm_http_request_errors_total{path="/functions/<func_name>"}`)
 
 	expandWithExprsRequests = metrics.NewCounter(`vm_http_requests_total{path="/expand-with-exprs"}`)
+	prettifyQueryRequests   = metrics.NewCounter(`vm_http_requests_total{path="/prettify-query"}`)
 
 	vmalertRequests = metrics.NewCounter(`vm_http_requests_total{path="/vmalert"}`)
 	rulesRequests   = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`)

app/vmselect/netstorage/netstorage.go

@@ -1243,7 +1243,8 @@ func setupTfss(qt *querytracer.Tracer, tr storage.TimeRange, tagFilterss [][]sto
 				}
 				if len(paths) >= maxMetrics {
 					return nil, fmt.Errorf("more than %d time series match Graphite query %q; "+
-						"either narrow down the query or increase the corresponding -search.max* command-line flag value", maxMetrics, query)
+						"either narrow down the query or increase the corresponding -search.max* command-line flag value; "+
+						"see https://docs.victoriametrics.com/#resource-usage-limits", maxMetrics, query)
 				}
 				tfs.AddGraphiteQuery(query, paths, tf.IsNegative)
 				continue

app/vmselect/prometheus/prometheus.go

@@ -3,6 +3,7 @@ package prometheus
 import (
 	"flag"
 	"fmt"
+	"github.com/VictoriaMetrics/metricsql"
 	"math"
 	"net/http"
 	"runtime"
@@ -75,6 +76,23 @@ func ExpandWithExprs(w http.ResponseWriter, r *http.Request) {
 	_ = bw.Flush()
 }
 
+// PrettifyQuery handles the request /prettify-query
+func PrettifyQuery(w http.ResponseWriter, r *http.Request) {
+	query := r.FormValue("query")
+	bw := bufferedwriter.Get(w)
+	defer bufferedwriter.Put(bw)
+	w.Header().Set("Content-Type", "application/json")
+	httpserver.EnableCORS(w, r)
+
+	prettyQuery, err := metricsql.Prettify(query)
+	if err != nil {
+		fmt.Fprintf(bw, `{"status": "error", "msg": %q}`, err)
+	} else {
+		fmt.Fprintf(bw, `{"status": "success", "query": %q}`, prettyQuery)
+	}
+	_ = bw.Flush()
+}
+
 // FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
 func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
 	defer federateDuration.UpdateDuration(startTime)
@@ -649,10 +667,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
 		qt.Donef("start=%d, end=%d", cp.start, cp.end)
 	}
 	WriteSeriesResponse(bw, metricNames, qt, qtDone)
-	if err := bw.Flush(); err != nil {
-		return err
-	}
-	return nil
+	return bw.Flush()
 }
 
 var seriesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/series"}`)
@@ -694,7 +709,10 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		return err
 	}
 	if childQuery, windowExpr, offsetExpr := promql.IsMetricSelectorWithRollup(query); childQuery != "" {
-		window := windowExpr.Duration(step)
+		window, err := windowExpr.NonNegativeDuration(step)
+		if err != nil {
+			return fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err)
+		}
 		offset := offsetExpr.Duration(step)
 		start -= offset
 		end := start
@@ -723,11 +741,17 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		return nil
 	}
 	if childQuery, windowExpr, stepExpr, offsetExpr := promql.IsRollup(query); childQuery != "" {
-		newStep := stepExpr.Duration(step)
+		newStep, err := stepExpr.NonNegativeDuration(step)
+		if err != nil {
+			return fmt.Errorf("cannot parse step in square brackets at %s: %w", query, err)
+		}
 		if newStep > 0 {
 			step = newStep
 		}
-		window := windowExpr.Duration(step)
+		window, err := windowExpr.NonNegativeDuration(step)
+		if err != nil {
+			return fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", query, err)
+		}
 		offset := offsetExpr.Duration(step)
 		start -= offset
 		end := start

app/vmselect/promql/active_queries.go

@@ -12,7 +12,7 @@ import (
 // ActiveQueriesHandler returns response to /api/v1/status/active_queries
 //
 // It writes a JSON with active queries to w.
-func ActiveQueriesHandler(w http.ResponseWriter, r *http.Request) {
+func ActiveQueriesHandler(w http.ResponseWriter, _ *http.Request) {
 	aqes := activeQueriesV.GetAll()
 
 	w.Header().Set("Content-Type", "application/json")

app/vmselect/promql/eval.go

@@ -896,11 +896,17 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName
 	// TODO: determine whether to use rollupResultCacheV here.
 	qt = qt.NewChild("subquery")
 	defer qt.Done()
-	step := re.Step.Duration(ec.Step)
+	step, err := re.Step.NonNegativeDuration(ec.Step)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse step in square brackets at %s: %w", expr.AppendString(nil), err)
+	}
 	if step == 0 {
 		step = ec.Step
 	}
-	window := re.Window.Duration(ec.Step)
+	window, err := re.Window.NonNegativeDuration(ec.Step)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
+	}
 
 	ecSQ := copyEvalConfig(ec)
 	ecSQ.Start -= window + maxSilenceInterval + step
@@ -1042,7 +1048,10 @@ var (
 func evalRollupFuncWithMetricExpr(qt *querytracer.Tracer, ec *EvalConfig, funcName string, rf rollupFunc,
 	expr metricsql.Expr, me *metricsql.MetricExpr, iafc *incrementalAggrFuncContext, windowExpr *metricsql.DurationExpr) ([]*timeseries, error) {
 	var rollupMemorySize int64
-	window := windowExpr.Duration(ec.Step)
+	window, err := windowExpr.NonNegativeDuration(ec.Step)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse lookbehind window in square brackets at %s: %w", expr.AppendString(nil), err)
+	}
 	if qt.Enabled() {
 		qt = qt.NewChild("rollup %s(): timeRange=%s, step=%d, window=%d", funcName, ec.timeRangeString(), ec.Step, window)
 		defer func() {

app/vmselect/promql/exec.go

@@ -73,7 +73,7 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 		}
 		qt.Printf("leave only the first point in every series")
 	}
-	maySort := maySortResults(e, rv)
+	maySort := maySortResults(e)
 	result, err := timeseriesToResult(rv, maySort)
 	if err != nil {
 		return nil, err
@@ -95,7 +95,7 @@ func Exec(qt *querytracer.Tracer, ec *EvalConfig, q string, isFirstPointOnly boo
 	return result, nil
 }
 
-func maySortResults(e metricsql.Expr, tss []*timeseries) bool {
+func maySortResults(e metricsql.Expr) bool {
 	switch v := e.(type) {
 	case *metricsql.FuncExpr:
 		switch strings.ToLower(v.Name) {
@@ -111,6 +111,12 @@ func maySortResults(e metricsql.Expr, tss []*timeseries) bool {
 			"bottomk_max", "bottomk_min", "bottomk_avg", "bottomk_median", "bottomk_last":
 			return false
 		}
+	case *metricsql.BinaryOpExpr:
+		if strings.ToLower(v.Op) == "or" {
+			// Do not sort results for `a or b` in the same way as Prometheus does.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4763
+			return false
+		}
 	}
 	return true
 }

app/vmselect/promql/exec_test.go

@@ -5,10 +5,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
 func TestEscapeDots(t *testing.T) {
@@ -45,7 +46,7 @@ func TestEscapeDotsInRegexpLabelFilters(t *testing.T) {
 	f("2", "2")
 	f(`foo.bar + 123`, `foo.bar + 123`)
 	f(`foo{bar=~"baz.xx.yyy"}`, `foo{bar=~"baz\\.xx\\.yyy"}`)
-	f(`foo(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + x.y(1,sum({x=~"aa.bb"}))`, `foo(a.b{c="d.e",x=~"a\\.b.+[\\.a]",y!~"aaa\\.bb|cc\\.dd"}) + x.y(1, sum({x=~"aa\\.bb"}))`)
+	f(`sum(a.b{c="d.e",x=~"a.b.+[.a]",y!~"aaa.bb|cc.dd"}) + avg_over_time(1,sum({x=~"aa.bb"}))`, `sum(a.b{c="d.e",x=~"a\\.b.+[\\.a]",y!~"aaa\\.bb|cc\\.dd"}) + avg_over_time(1, sum({x=~"aa\\.bb"}))`)
 }
 
 func TestExecSuccess(t *testing.T) {
@@ -86,6 +87,28 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("int_with_underscores", func(t *testing.T) {
+		t.Parallel()
+		q := `123_456_789`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{123456789, 123456789, 123456789, 123456789, 123456789, 123456789},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
+	t.Run("float_with_underscores", func(t *testing.T) {
+		t.Parallel()
+		q := `1_2.3_456_789`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{12.3456789, 12.3456789, 12.3456789, 12.3456789, 12.3456789, 12.3456789},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("duration-constant", func(t *testing.T) {
 		t.Parallel()
 		q := `1h23m5S`
@@ -141,6 +164,17 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("num-with-suffix-5", func(t *testing.T) {
+		t.Parallel()
+		q := `1_234M`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1234e6, 1234e6, 1234e6, 1234e6, 1234e6, 1234e6},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("simple-arithmetic", func(t *testing.T) {
 		t.Parallel()
 		q := `-1+2 *3 ^ 4+5%6`
@@ -212,6 +246,27 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("bitmap_and(NaN, 1)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(NaN, 1)`
+		f(q, nil)
+	})
+	t.Run("bitmap_and(1, NaN)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(1, NaN)`
+		f(q, nil)
+	})
+	t.Run("bitmap_and(round(rand(1) > 0.5, 1), 1)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_and(round(rand(1) > 0.5, 1), 1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, nan, nan, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("bitmap_or(0xA2, 0x11)", func(t *testing.T) {
 		t.Parallel()
 		q := `bitmap_or(0xA2, 0x11)`
@@ -234,6 +289,22 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("bitmap_or(NaN, 1)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_or(NaN, 1)`
+		f(q, nil)
+	})
+	t.Run("bitmap_or(round(rand(1) > 0.5, 1), 1)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_or(round(rand(1) > 0.5, 1), 1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, nan, nan, 1},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("bitmap_xor(0xB3, 0x11)", func(t *testing.T) {
 		t.Parallel()
 		q := `bitmap_xor(0xB3, 0x11)`
@@ -256,6 +327,22 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r}
 		f(q, resultExpected)
 	})
+	t.Run("bitmap_xor(NaN, 1)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_xor(NaN, 1)`
+		f(q, nil)
+	})
+	t.Run("bitmap_xor(round(rand(1) > 0.5, 1), 1)", func(t *testing.T) {
+		t.Parallel()
+		q := `bitmap_xor(round(rand(1) > 0.5, 1), 1)`
+		r := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{0, 0, 0, nan, nan, 0},
+			Timestamps: timestampsExpected,
+		}
+		resultExpected := []netstorage.Result{r}
+		f(q, resultExpected)
+	})
 	t.Run("timezone_offset(UTC)", func(t *testing.T) {
 		t.Parallel()
 		q := `timezone_offset("UTC")`
@@ -7636,7 +7723,7 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`aggr_over_time(multi-func)`, func(t *testing.T) {
 		t.Parallel()
-		q := `sort(aggr_over_time(("min_over_time", "count_over_time", "max_over_time"), round(rand(0),0.1)[:10s]))`
+		q := `sort(aggr_over_time(("min_over_time", "median_over_time", "max_over_time"), round(rand(0),0.1)[:10s]))`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{0, 0, 0, 0, 0, 0},
@@ -7648,21 +7735,21 @@ func TestExecSuccess(t *testing.T) {
 		}}
 		r2 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{0.8, 0.9, 1, 0.9, 1, 0.9},
+			Values:     []float64{0.4, 0.5, 0.5, 0.75, 0.6, 0.45},
 			Timestamps: timestampsExpected,
 		}
 		r2.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
-			Value: []byte("max_over_time"),
+			Value: []byte("median_over_time"),
 		}}
 		r3 := netstorage.Result{
 			MetricName: metricNameExpected,
-			Values:     []float64{20, 20, 20, 20, 20, 20},
+			Values:     []float64{0.8, 0.9, 1, 0.9, 1, 0.9},
 			Timestamps: timestampsExpected,
 		}
 		r3.MetricName.Tags = []storage.Tag{{
 			Key:   []byte("rollup"),
-			Value: []byte("count_over_time"),
+			Value: []byte("max_over_time"),
 		}}
 		resultExpected := []netstorage.Result{r1, r2, r3}
 		f(q, resultExpected)
@@ -8446,11 +8533,11 @@ func TestExecSuccess(t *testing.T) {
 	})
 	t.Run(`result sorting`, func(t *testing.T) {
 		t.Parallel()
-		q := `label_set(1, "instance", "localhost:1001", "type", "free")
-			or label_set(1, "instance", "localhost:1001", "type", "buffers")
-			or label_set(1, "instance", "localhost:1000", "type", "buffers")
-			or label_set(1, "instance", "localhost:1000", "type", "free")
-`
+		q := `(label_set(1, "instance", "localhost:1001", "type", "free"),
+			label_set(1, "instance", "localhost:1001", "type", "buffers"),
+			label_set(1, "instance", "localhost:1000", "type", "buffers"),
+			label_set(1, "instance", "localhost:1000", "type", "free"),
+		)`
 		r1 := netstorage.Result{
 			MetricName: metricNameExpected,
 			Values:     []float64{1, 1, 1, 1, 1, 1},
@@ -8482,6 +8569,34 @@ func TestExecSuccess(t *testing.T) {
 		resultExpected := []netstorage.Result{r1, r2, r3, r4}
 		f(q, resultExpected)
 	})
+	t.Run(`no_sorting_for_or`, func(t *testing.T) {
+		t.Parallel()
+		q := `label_set(2, "foo", "bar") or label_set(1, "foo", "baz")`
+		r1 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{2, 2, 2, 2, 2, 2},
+			Timestamps: timestampsExpected,
+		}
+		r1.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("bar"),
+			},
+		}
+		r2 := netstorage.Result{
+			MetricName: metricNameExpected,
+			Values:     []float64{1, 1, 1, 1, 1, 1},
+			Timestamps: timestampsExpected,
+		}
+		r2.MetricName.Tags = []storage.Tag{
+			{
+				Key:   []byte("foo"),
+				Value: []byte("baz"),
+			},
+		}
+		resultExpected := []netstorage.Result{r1, r2}
+		f(q, resultExpected)
+	})
 	t.Run(`sort_by_label_numeric(multiple_labels_only_string)`, func(t *testing.T) {
 		t.Parallel()
 		q := `sort_by_label_numeric((

app/vmselect/promql/rollup.go

@@ -7,11 +7,12 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/VictoriaMetrics/metrics"
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metrics"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
 var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The minimum interval for staleness calculations. "+
@@ -58,6 +59,7 @@ var rollupFuncs = map[string]newRollupFunc{
 	"lifetime":                newRollupFuncOneArg(rollupLifetime),
 	"mad_over_time":           newRollupFuncOneArg(rollupMAD),
 	"max_over_time":           newRollupFuncOneArg(rollupMax),
+	"median_over_time":        newRollupFuncOneArg(rollupMedian),
 	"min_over_time":           newRollupFuncOneArg(rollupMin),
 	"mode_over_time":          newRollupFuncOneArg(rollupModeOverTime),
 	"predict_linear":          newRollupPredictLinear,
@@ -125,6 +127,7 @@ var rollupAggrFuncs = map[string]rollupFunc{
 	"lifetime":                rollupLifetime,
 	"mad_over_time":           rollupMAD,
 	"max_over_time":           rollupMax,
+	"median_over_time":        rollupMedian,
 	"min_over_time":           rollupMin,
 	"mode_over_time":          rollupModeOverTime,
 	"present_over_time":       rollupPresent,
@@ -224,6 +227,7 @@ var rollupFuncsKeepMetricName = map[string]bool{
 	"holt_winters":          true,
 	"last_over_time":        true,
 	"max_over_time":         true,
+	"median_over_time":      true,
 	"min_over_time":         true,
 	"mode_over_time":        true,
 	"predict_linear":        true,
@@ -1396,6 +1400,10 @@ func rollupMax(rfa *rollupFuncArg) float64 {
 	return maxValue
 }
 
+func rollupMedian(rfa *rollupFuncArg) float64 {
+	return quantile(0.5, rfa.values)
+}
+
 func rollupTmin(rfa *rollupFuncArg) float64 {
 	// There is no need in handling NaNs here, since they must be cleaned up
 	// before calling rollup funcs.
@@ -2222,7 +2230,7 @@ func rollupIntegrate(rfa *rollupFuncArg) float64 {
 	return sum
 }
 
-func rollupFake(rfa *rollupFuncArg) float64 {
+func rollupFake(_ *rollupFuncArg) float64 {
 	logger.Panicf("BUG: rollupFake shouldn't be called")
 	return 0
 }

app/vmselect/promql/rollup_test.go

@@ -160,7 +160,7 @@ func TestDerivValues(t *testing.T) {
 	testRowsEqual(t, values, timestamps, valuesExpected, timestamps)
 }
 
-func testRollupFunc(t *testing.T, funcName string, args []interface{}, meExpected *metricsql.MetricExpr, vExpected float64) {
+func testRollupFunc(t *testing.T, funcName string, args []interface{}, vExpected float64) {
 	t.Helper()
 	nrf := getRollupFunc(funcName)
 	if nrf == nil {
@@ -203,7 +203,7 @@ func TestRollupDurationOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, maxIntervals}
-		testRollupFunc(t, "duration_over_time", args, &me, dExpected)
+		testRollupFunc(t, "duration_over_time", args, dExpected)
 	}
 	f(-123, 0)
 	f(0, 0)
@@ -224,7 +224,7 @@ func TestRollupShareLEOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
-		testRollupFunc(t, "share_le_over_time", args, &me, vExpected)
+		testRollupFunc(t, "share_le_over_time", args, vExpected)
 	}
 
 	f(-123, 0)
@@ -247,7 +247,7 @@ func TestRollupShareGTOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, gts}
-		testRollupFunc(t, "share_gt_over_time", args, &me, vExpected)
+		testRollupFunc(t, "share_gt_over_time", args, vExpected)
 	}
 
 	f(-123, 1)
@@ -270,7 +270,7 @@ func TestRollupShareEQOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, eqs}
-		testRollupFunc(t, "share_eq_over_time", args, &me, vExpected)
+		testRollupFunc(t, "share_eq_over_time", args, vExpected)
 	}
 
 	f(-123, 0)
@@ -289,7 +289,7 @@ func TestRollupCountLEOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, les}
-		testRollupFunc(t, "count_le_over_time", args, &me, vExpected)
+		testRollupFunc(t, "count_le_over_time", args, vExpected)
 	}
 
 	f(-123, 0)
@@ -312,7 +312,7 @@ func TestRollupCountGTOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, gts}
-		testRollupFunc(t, "count_gt_over_time", args, &me, vExpected)
+		testRollupFunc(t, "count_gt_over_time", args, vExpected)
 	}
 
 	f(-123, 12)
@@ -335,7 +335,7 @@ func TestRollupCountEQOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, eqs}
-		testRollupFunc(t, "count_eq_over_time", args, &me, vExpected)
+		testRollupFunc(t, "count_eq_over_time", args, vExpected)
 	}
 
 	f(-123, 0)
@@ -354,7 +354,7 @@ func TestRollupCountNEOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, nes}
-		testRollupFunc(t, "count_ne_over_time", args, &me, vExpected)
+		testRollupFunc(t, "count_ne_over_time", args, vExpected)
 	}
 
 	f(-123, 12)
@@ -373,7 +373,7 @@ func TestRollupQuantileOverTime(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}}
-		testRollupFunc(t, "quantile_over_time", args, &me, vExpected)
+		testRollupFunc(t, "quantile_over_time", args, vExpected)
 	}
 
 	f(-123, math.Inf(-1))
@@ -395,7 +395,7 @@ func TestRollupPredictLinear(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, secs}
-		testRollupFunc(t, "predict_linear", args, &me, vExpected)
+		testRollupFunc(t, "predict_linear", args, vExpected)
 	}
 
 	f(0e-3, 65.07405077267295)
@@ -434,7 +434,7 @@ func TestRollupHoltWinters(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}, sfs, tfs}
-		testRollupFunc(t, "holt_winters", args, &me, vExpected)
+		testRollupFunc(t, "holt_winters", args, vExpected)
 	}
 
 	f(-1, 0.5, nan)
@@ -462,7 +462,7 @@ func TestRollupHoeffdingBoundLower(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}}
-		testRollupFunc(t, "hoeffding_bound_lower", args, &me, vExpected)
+		testRollupFunc(t, "hoeffding_bound_lower", args, vExpected)
 	}
 
 	f(0.5, 28.21949401521037)
@@ -483,7 +483,7 @@ func TestRollupHoeffdingBoundUpper(t *testing.T) {
 		}}
 		var me metricsql.MetricExpr
 		args := []interface{}{phis, &metricsql.RollupExpr{Expr: &me}}
-		testRollupFunc(t, "hoeffding_bound_upper", args, &me, vExpected)
+		testRollupFunc(t, "hoeffding_bound_upper", args, vExpected)
 	}
 
 	f(0.5, 65.9471726514563)
@@ -500,7 +500,7 @@ func TestRollupNewRollupFuncSuccess(t *testing.T) {
 		t.Helper()
 		var me metricsql.MetricExpr
 		args := []interface{}{&metricsql.RollupExpr{Expr: &me}}
-		testRollupFunc(t, funcName, args, &me, vExpected)
+		testRollupFunc(t, funcName, args, vExpected)
 	}
 
 	f("default_rollup", 34)

app/vmselect/promql/transform.go

@@ -11,12 +11,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/metricsql"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/metricsql"
 )
 
 var transformFuncs = map[string]transformFunc{
@@ -1090,18 +1091,18 @@ func transformHour(t time.Time) int {
 	return t.Hour()
 }
 
-func runningSum(a, b float64, idx int) float64 {
+func runningSum(a, b float64, _ int) float64 {
 	return a + b
 }
 
-func runningMax(a, b float64, idx int) float64 {
+func runningMax(a, b float64, _ int) float64 {
 	if a > b {
 		return a
 	}
 	return b
 }
 
-func runningMin(a, b float64, idx int) float64 {
+func runningMin(a, b float64, _ int) float64 {
 	if a < b {
 		return a
 	}
@@ -2589,7 +2590,12 @@ func newTransformBitmap(bitmapFunc func(a, b uint64) uint64) func(tfa *transform
 		}
 		tf := func(values []float64) {
 			for i, v := range values {
-				values[i] = float64(bitmapFunc(uint64(v), uint64(ns[i])))
+				w := ns[i]
+				result := nan
+				if !math.IsNaN(v) && !math.IsNaN(w) {
+					result = float64(bitmapFunc(uint64(v), uint64(w)))
+				}
+				values[i] = result
 			}
 		}
 		return doTransformValues(args[0], tf, tfa.fe)

app/vmselect/vmui/asset-manifest.json

@@ -1,14 +1,14 @@
 {
   "files": {
-    "main.css": "./static/css/main.c93e8dee.css",
-    "main.js": "./static/js/main.05892dd1.js",
+    "main.css": "./static/css/main.4ab6595e.css",
+    "main.js": "./static/js/main.dbf8fb4f.js",
     "static/js/522.b5ae4365.chunk.js": "./static/js/522.b5ae4365.chunk.js",
     "static/media/Lato-Regular.ttf": "./static/media/Lato-Regular.d714fec1633b69a9c2e9.ttf",
     "static/media/Lato-Bold.ttf": "./static/media/Lato-Bold.32360ba4b57802daa4d6.ttf",
     "index.html": "./index.html"
   },
   "entrypoints": [
-    "static/css/main.c93e8dee.css",
-    "static/js/main.05892dd1.js"
+    "static/css/main.4ab6595e.css",
+    "static/js/main.dbf8fb4f.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.05892dd1.js"></script><link href="./static/css/main.c93e8dee.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="UI for VictoriaMetrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="./preview.jpg"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:site" content="@VictoriaMetrics"><meta property="og:title" content="Metric explorer for VictoriaMetrics"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta property="og:image" content="./preview.jpg"><meta property="og:type" content="website"><script defer="defer" src="./static/js/main.dbf8fb4f.js"></script><link href="./static/css/main.4ab6595e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>