Improvements for backup description and configuration for single node, cluster , quick start

Adding additional reference for snapshot troubleshooting for better accessibility

.github/pull_request_template.md

@@ -6,4 +6,4 @@ Please provide a brief description of the changes you made. Be as specific as po
 
 The following checks are **mandatory**:
 
-- [ ] My change adheres [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).
+- [ ] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/contributing/).

app/vmagent/remotewrite/client.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -384,7 +385,7 @@ func (c *client) newRequest(url string, body []byte) (*http.Request, error) {
 	h := req.Header
 	h.Set("User-Agent", "vmagent")
 	h.Set("Content-Type", "application/x-protobuf")
-	if c.useVMProto {
+	if encoding.IsZstd(body) {
 		h.Set("Content-Encoding", "zstd")
 		h.Set("X-VictoriaMetrics-Remote-Write-Version", "1")
 	} else {

app/vmalert-tool/deployment/Dockerfile

@@ -0,0 +1,8 @@
+ARG base_image=non-existing
+FROM $base_image
+
+EXPOSE 8880
+
+ENTRYPOINT ["/vmalert-tool-prod"]
+ARG src_binary=non-existing
+COPY $src_binary ./vmalert-tool-prod

app/vmbackup/main.go

@@ -33,7 +33,8 @@ var (
 		"All created snapshots will be automatically deleted. Example: http://victoriametrics:8428/snapshot/delete")
 	dst = flag.String("dst", "", "Where to put the backup on the remote storage. "+
 		"Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup/dir\n"+
-		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded")
+		"-dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded\n"+
+		"Note: If custom S3 endpoint is used, URL should contain only name of the bucket, while hostname of S3 server must be specified via the -customS3Endpoint command-line flag.")
 	origin            = flag.String("origin", "", "Optional origin directory on the remote storage with old backup for server-side copying when performing full backup. This speeds up full backups")
 	concurrency       = flag.Int("concurrency", 10, "The number of concurrent workers. Higher concurrency may reduce backup duration")
 	maxBytesPerSecond = flagutil.NewBytes("maxBytesPerSecond", 0, "The maximum upload speed. There is no limit if it is set to 0")

app/vmrestore/main.go

@@ -20,7 +20,8 @@ import (
 var (
 	httpListenAddr = flag.String("httpListenAddr", ":8421", "TCP address for exporting metrics at /metrics page")
 	src            = flag.String("src", "", "Source path with backup on the remote storage. "+
-		"Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup")
+		"Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup\n"+
+		"Note: If custom S3 endpoint is used, URL should contain only name of the bucket, while hostname of S3 server must be specified via the -customS3Endpoint command-line flag.")
 	storageDataPath = flag.String("storageDataPath", "victoria-metrics-data", "Destination path where backup must be restored. "+
 		"VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir "+
 		"is synchronized with -src contents, i.e. it works like 'rsync --delete'")

apptest/testcase.go

@@ -124,6 +124,23 @@ func (tc *TestCase) MustStartVminsert(instance string, flags []string) *Vminsert
 	return app
 }
 
+// MustStartVmagent is a test helper function that starts an instance of
+// vmagent and fails the test if the app fails to start.
+func (tc *TestCase) MustStartVmagent(instance string, flags []string, promScrapeConfigFileYAML string) *Vmagent {
+	tc.t.Helper()
+
+	promScrapeConfigFilePath := path.Join(tc.t.TempDir(), "prometheus.yml")
+	if err := os.WriteFile(promScrapeConfigFilePath, []byte(promScrapeConfigFileYAML), os.ModePerm); err != nil {
+		tc.t.Fatalf("cannot init vmagent: prom config file write failed: %s", err)
+	}
+	app, err := StartVmagent(instance, flags, tc.cli, promScrapeConfigFilePath)
+	if err != nil {
+		tc.t.Fatalf("Could not start %s: %v", instance, err)
+	}
+	tc.addApp(instance, app)
+	return app
+}
+
 // Vmcluster represents a typical cluster setup: several vmstorage replicas, one
 // vminsert, and one vmselect.
 //

apptest/tests/vmagent_remotewrite_test.go

@@ -0,0 +1,54 @@
+package tests
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+// TestSingleVMAgentZstdRemoteWrite verifies that vmagent can successfully perform
+// a remote write to vmsingle using VM protocol (zstd).
+func TestSingleVMAgentZstdRemoteWrite(t *testing.T) {
+	testSingleVMAgentRemoteWrite(t, false)
+}
+
+// TestSingleVMAgentSnappyRemoteWrite verifies that vmagent can successfully perform
+// a remote write to vmsingle using Prometheus protocol (snappy).
+func TestSingleVMAgentSnappyRemoteWrite(t *testing.T) {
+	testSingleVMAgentRemoteWrite(t, true)
+}
+
+func testSingleVMAgentRemoteWrite(t *testing.T, forcePromProto bool) {
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+
+	vmsingle := tc.MustStartDefaultVmsingle()
+
+	vmagent := tc.MustStartVmagent("vmagent", []string{
+		`-remoteWrite.flushInterval=50ms`,
+		fmt.Sprintf(`-remoteWrite.forcePromProto=%v`, forcePromProto),
+		fmt.Sprintf(`-remoteWrite.url=http://%s/api/v1/write`, vmsingle.HTTPAddr()),
+	}, ``)
+
+	vmagent.APIV1ImportPrometheus(t, []string{
+		"foo_bar 1 1652169600000", // 2022-05-10T08:00:00Z
+	}, apptest.QueryOpts{})
+
+	vmsingle.ForceFlush(t)
+
+	tc.Assert(&at.AssertOptions{
+		Msg: `unexpected metrics stored on vmagent remote write`,
+		Got: func() any {
+			return vmsingle.PrometheusAPIV1Series(t, `{__name__="foo_bar"}`, at.QueryOpts{
+				Start: "2022-05-10T00:00:00Z",
+				End:   "2022-05-10T23:59:59Z",
+			}).Sort()
+		},
+		Want: &at.PrometheusAPIV1SeriesResponse{
+			Status: "success",
+			Data:   []map[string]string{{"__name__": "foo_bar"}},
+		},
+	})
+}

apptest/vmagent.go

@@ -0,0 +1,107 @@
+package apptest
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Vmagent holds the state of a vmagent app and provides vmagent-specific functions
+type Vmagent struct {
+	*app
+	*ServesMetrics
+
+	httpListenAddr           string
+	apiV1ImportPrometheusURL string
+}
+
+// StartVmagent starts an instance of vmagent with the given flags. It also
+// sets the default flags and populates the app instance state with runtime
+// values extracted from the application log (such as httpListenAddr)
+func StartVmagent(instance string, flags []string, cli *Client, promScrapeConfigFilePath string) (*Vmagent, error) {
+	extractREs := []*regexp.Regexp{
+		httpListenAddrRE,
+	}
+
+	app, stderrExtracts, err := startApp(instance, "../../bin/vmagent", flags, &appOptions{
+		defaultFlags: map[string]string{
+			"-httpListenAddr":    "127.0.0.1:0",
+			"-promscrape.config": promScrapeConfigFilePath,
+		},
+		extractREs: extractREs,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &Vmagent{
+		app: app,
+		ServesMetrics: &ServesMetrics{
+			metricsURL: fmt.Sprintf("http://%s/metrics", stderrExtracts[0]),
+			cli:        cli,
+		},
+		httpListenAddr:           stderrExtracts[0],
+		apiV1ImportPrometheusURL: fmt.Sprintf("http://%s/api/v1/import/prometheus", stderrExtracts[0]),
+	}, nil
+}
+
+// APIV1ImportPrometheus is a test helper function that inserts a
+// collection of records in Prometheus text exposition format for the given
+// tenant by sending a HTTP POST request to /api/v1/import/prometheus vmagent endpoint.
+//
+// The call is blocked until the data is flushed to vmstorage or the timeout is reached.
+//
+// See https://docs.victoriametrics.com/url-examples/#apiv1importprometheus
+func (app *Vmagent) APIV1ImportPrometheus(t *testing.T, records []string, _ QueryOpts) {
+	t.Helper()
+
+	data := []byte(strings.Join(records, "\n"))
+	app.sendBlocking(t, len(records), func() {
+		_, statusCode := app.cli.Post(t, app.apiV1ImportPrometheusURL, "text/plain", data)
+		if statusCode != http.StatusNoContent {
+			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
+		}
+	})
+}
+
+// sendBlocking sends the data to vmstorage by executing `send` function and
+// waits until the data is actually sent.
+//
+// vmagent does not send the data immediately. It first puts the data into a
+// buffer. Then a background goroutine takes the data from the buffer sends it
+// to the vmstorage. This happens every 1s by default.
+//
+// Waiting is implemented a retrieving the value of `vmagent_remotewrite_requests_total`
+// metric and checking whether it is equal or greater than the wanted value.
+// If it is, then the data has been sent to vmstorage.
+//
+// Unreliable if the records are inserted concurrently.
+func (app *Vmagent) sendBlocking(t *testing.T, numRecordsToSend int, send func()) {
+	t.Helper()
+
+	send()
+
+	const (
+		retries = 20
+		period  = 100 * time.Millisecond
+	)
+	wantRowsSentCount := app.remoteWriteRequestsTotal(t) + numRecordsToSend
+	for range retries {
+		if app.remoteWriteRequestsTotal(t) >= wantRowsSentCount {
+			return
+		}
+		time.Sleep(period)
+	}
+	t.Fatalf("timed out while waiting for inserted rows to be sent to vmstorage")
+}
+
+func (app *Vmagent) remoteWriteRequestsTotal(t *testing.T) int {
+	total := 0.0
+	for _, v := range app.GetMetricsByPrefix(t, "vmagent_remotewrite_requests_total") {
+		total += v
+	}
+	return int(total)
+}

apptest/vmsingle.go

@@ -350,6 +350,12 @@ func (app *Vmsingle) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResponse
 	return &res
 }
 
+// HTTPAddr returns the address at which the vmstorage process is listening
+// for http connections.
+func (app *Vmsingle) HTTPAddr() string {
+	return app.httpListenAddr
+}
+
 // String returns the string representation of the vmsingle app state.
 func (app *Vmsingle) String() string {
 	return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q}", []any{

deployment/docker/docker-compose-cluster.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.114.0
+    image: victoriametrics/vmagent:v1.115.0
     depends_on:
       - "vminsert"
     ports:
@@ -39,7 +39,7 @@ services:
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
     container_name: vmstorage-1
-    image: victoriametrics/vmstorage:v1.114.0-cluster
+    image: victoriametrics/vmstorage:v1.115.0-cluster
     ports:
       - 8482
       - 8400
@@ -51,7 +51,7 @@ services:
     restart: always
   vmstorage-2:
     container_name: vmstorage-2
-    image: victoriametrics/vmstorage:v1.114.0-cluster
+    image: victoriametrics/vmstorage:v1.115.0-cluster
     ports:
       - 8482
       - 8400
@@ -66,7 +66,7 @@ services:
   # pre-process them and distributes across configured vmstorage shards.
   vminsert:
     container_name: vminsert
-    image: victoriametrics/vminsert:v1.114.0-cluster
+    image: victoriametrics/vminsert:v1.115.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -81,7 +81,7 @@ services:
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
     container_name: vmselect-1
-    image: victoriametrics/vmselect:v1.114.0-cluster
+    image: victoriametrics/vmselect:v1.115.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -94,7 +94,7 @@ services:
     restart: always
   vmselect-2:
     container_name: vmselect-2
-    image: victoriametrics/vmselect:v1.114.0-cluster
+    image: victoriametrics/vmselect:v1.115.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -112,7 +112,7 @@ services:
   # It can be used as an authentication proxy.
   vmauth:
     container_name: vmauth
-    image: victoriametrics/vmauth:v1.114.0
+    image: victoriametrics/vmauth:v1.115.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -127,7 +127,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.114.0
+    image: victoriametrics/vmalert:v1.115.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/docker-compose.yml

@@ -4,7 +4,7 @@ services:
   #  And forward them to --remoteWrite.url
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.114.0
+    image: victoriametrics/vmagent:v1.115.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -20,7 +20,7 @@ services:
   # storing metrics and serve read requests.
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.114.0
+    image: victoriametrics/victoria-metrics:v1.115.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -59,7 +59,7 @@ services:
   # vmalert executes alerting and recording rules
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.114.0
+    image: victoriametrics/vmalert:v1.115.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/victorialogs/datadog-serverless/aws/Dockerfile

@@ -8,7 +8,7 @@ RUN \
   cd /tmp/aws-lambda-rie && \
   CGO_ENABLED=0 go build -buildvcs=false -ldflags "-s -w" -o /aws-lambda-rie ./cmd/aws-lambda-rie
 
-FROM python:3.12-bullseye
+FROM python:3.13-bullseye
 
 RUN \
   apt update && \
@@ -26,7 +26,7 @@ RUN \
 WORKDIR /var/task
 COPY --from=aws-lambda-rie /aws-lambda-rie /var/task/aws-lambda-rie
 COPY main.py /var/task/
-COPY --from=public.ecr.aws/datadog/lambda-extension:68 /opt/. /opt/
+COPY --from=public.ecr.aws/datadog/lambda-extension:73 /opt/. /opt/
 
 ENTRYPOINT ["/var/task/aws-lambda-rie"]
 CMD ["/usr/local/bin/python", "-m", "awslambdaric", "main.lambda_handler"]

deployment/docker/victorialogs/datadog-serverless/aws/compose.yml

@@ -14,7 +14,7 @@ services:
       DD_API_KEY: test
       DD_DD_URL: http://dd-proxy:8427
       DD_PROFILING_ENABLED: false
-      DD_ENHANCED_METRICS: false
+      DD_ENHANCED_METRICS: true
       DD_LOGS_CONFIG_LOGS_DD_URL: http://dd-proxy:8427
       DD_SERVERLESS_FLUSH_STRATEGY: periodically,100
     depends_on:

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -2,9 +2,9 @@
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics) (v1.114.0)
-  -  [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.114.0)
-  -  [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.114.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/single-server-victoriametrics) (v1.115.0)
+  -  [vmalert](https://docs.victoriametrics.com/vmalert/) (v1.115.0)
+  -  [vmagent](https://docs.victoriametrics.com/vmagent/) (v1.115.0)
 - [Grafana](https://grafana.com/) (v.10.2.1)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@@ -315,7 +315,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.114.0
+    image: victoriametrics/vmagent:v1.115.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -332,7 +332,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.114.0
+    image: victoriametrics/victoria-metrics:v1.115.0
     ports:
       - 8428:8428
     volumes:
@@ -365,7 +365,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.114.0
+    image: victoriametrics/vmalert:v1.115.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmgateway-openid-configuration/README.md

@@ -49,7 +49,7 @@ See details about all supported options in the [vmgateway documentation](https:/
    ![Client secret](client-secret.webp)
    Copy the value of `Client secret`. It will be used later in Grafana configuration.<br>
 1. Go to `Clients` -> `grafana` -> `Client scopes`.<br>
-   Click at `grafana-dedicated` -> `Add mapper` -> `By configuration` -> `User attribute`.<br>
+   Click at `grafana-dedicated` -> `Configure a new mapper` -> `User attribute`.<br>
    ![Create mapper 1](create-mapper-1.webp)
    ![Create mapper 2](create-mapper-2.webp)
    Configure the mapper as follows<br>
@@ -61,8 +61,13 @@ See details about all supported options in the [vmgateway documentation](https:/
    
    ![Create mapper 3](create-mapper-3.webp)
    Click `Save`.<br>
-1. Go to `Users` -> select user to configure claims -> `Attributes`.<br>
-   Specify `vm_access` as `Key`.<br>
+1. Go to `Realm settings` -> `User profile`.<br>
+   Click `Create attribute`.<br>
+   Specify `vm_access` as `Attribute [Name]`.<br>
+   ![User attributes](create-attribute.webp)
+   Click `Save`.<br>
+1. Go to `Users` -> select user to configure.<br>
+   Modify value of `vm_access` attribute.<br>
    For the purpose of this example, we will use 2 users:<br>
    - for the first user we will specify `{"tenant_id" : {"account_id": 0, "project_id": 0 },"extra_labels":{ "team": "admin" }}` as `Value`.
    - for the second user we will specify `{"tenant_id" : {"account_id": 0, "project_id": 1 },"extra_labels":{ "team": "dev" }}` as `Value`.
@@ -219,44 +224,44 @@ version: '3'
 
 services:
   keycloak:
-    image: quay.io/keycloak/keycloak:21.0
+    image: quay.io/keycloak/keycloak:26.1
     command:
       - start-dev
     ports:
       - 3001:8080
     environment:
-      KEYCLOAK_ADMIN: admin
-      KEYCLOAK_ADMIN_PASSWORD: change_me
+      KC_BOOTSTRAP_ADMIN_USERNAME: admin
+      KC_BOOTSTRAP_ADMIN_PASSWORD: change_me
 
   grafana:
-    image: grafana/grafana:10.4.2
+    image: grafana/grafana:11.5.2
     network_mode: host
     volumes:
       - ./grafana.ini:/etc/grafana/grafana.ini
       - grafana_data:/var/lib/grafana/
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.114.0
+    image: victoriametrics/victoria-metrics:v1.115.0
     command:
       - -httpListenAddr=0.0.0.0:8429
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.114.0-cluster
+    image: victoriametrics/vmstorage:v1.115.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.114.0-cluster
+    image: victoriametrics/vminsert:v1.115.0-cluster
     command:
       - -storageNode=vmstorage:8400
       - -httpListenAddr=0.0.0.0:8480
 
   vmselect:
-    image: victoriametrics/vmselect:v1.114.0-cluster
+    image: victoriametrics/vmselect:v1.115.0-cluster
     command:
       - -storageNode=vmstorage:8401
       - -httpListenAddr=0.0.0.0:8481
 
   vmagent:
-    image: victoriametrics/vmagent:v1.114.0
+    image: victoriametrics/vmagent:v1.115.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -265,7 +270,7 @@ services:
       - -remoteWrite.url=http://vmsingle:8429/api/v1/write
 
   vmgateway-cluster:
-    image: victoriametrics/vmgateway:v1.114.0-enterprise
+    image: victoriametrics/vmgateway:v1.115.0-enterprise
     ports:
       - 8431:8431
     volumes:
@@ -281,7 +286,7 @@ services:
       - -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
 
   vmgateway-single:
-    image: victoriametrics/vmgateway:v1.114.0-enterprise
+    image: victoriametrics/vmgateway:v1.115.0-enterprise
     ports:
       - 8432:8431
     volumes:
@@ -369,9 +374,9 @@ In order to create a client for vmagent to use, follow the steps below:
    ![Client secret](vmagent-client-secret.webp)
    Copy the value of `Client secret`. It will be used later in vmagent configuration.<br>
 1. Go to `Clients` -> `vmagent` -> `Client scopes`.<br>
-   Click at `vmagent-dedicated` -> `Add mapper` -> `By configuration` -> `User attribute`.<br>
-   ![Create mapper 1](create-mapper-1.webp)
-   ![Create mapper 2](create-mapper-2.webp)
+   Click at `vmagent-dedicated` -> `Configure a new mapper` -> `User attribute`.<br>
+   ![Create mapper 1](vmagent-create-mapper-1.webp)
+   ![Create mapper 2](vmagent-create-mapper-2.webp)
    Configure the mapper as follows<br>
    - `Name` as `vm_access`.
    - `Token Claim Name` as `vm_access`.
@@ -379,13 +384,12 @@ In order to create a client for vmagent to use, follow the steps below:
    - `Claim JSON Type` as `JSON`.
      Enable `Add to ID token` and `Add to access token`.<br>
 
-   ![Create mapper 3](create-mapper-3.webp)
+   ![Create mapper 3](vmagent-create-mapper-3.webp)
    Click `Save`.<br>
 1. Go to `Service account roles` -> click on `service-account-vmagent`.<br>
    ![vmagent service account](vmagent-sa.webp)
 1. Go to `Attributes` tab and add an attribute.
-   Specify `vm_access` as `Key`.<br>
-   Specify `{"tenant_id" : {"account_id": 0, "project_id": 0 }}` as a value.<br>
+   Change `vm_access` attribute value to `{"tenant_id" : {"account_id": 0, "project_id": 0 }}`. <br>
    ![User attributes](vmagent-sa-attributes.webp)
    Click `Save`.
 
@@ -393,7 +397,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
 
 ```yaml
   vmagent:
-    image: victoriametrics/vmagent:v1.114.0
+    image: victoriametrics/vmagent:v1.115.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
       - ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -954,23 +954,7 @@ to ensure query results consistency, even if storage layer didn't complete dedup
 
 ## Backups
 
-It is recommended performing periodical backups from [instant snapshots](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282)
-for protecting from user errors such as accidental data deletion.
-
-The following steps must be performed for each `vmstorage` node for creating a backup:
-
-1. Create an instant snapshot by navigating to `/snapshot/create` HTTP handler. It will create snapshot and return its name.
-1. Archive the created snapshot from `<-storageDataPath>/snapshots/<snapshot_name>` folder using [vmbackup](https://docs.victoriametrics.com/vmbackup/).
-   The archival process doesn't interfere with `vmstorage` work, so it may be performed at any suitable time.
-1. Delete unused snapshots via `/snapshot/delete?snapshot=<snapshot_name>` or `/snapshot/delete_all` in order to free up occupied storage space.
-
-There is no need in synchronizing backups among all the `vmstorage` nodes.
-
-Restoring from backup:
-
-1. Stop `vmstorage` node with `kill -INT`.
-1. Restore data from backup using [vmrestore](https://docs.victoriametrics.com/vmrestore/) into `-storageDataPath` directory.
-1. Start `vmstorage` node.
+For backup configuration, please refer for [vmbackup documentation](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
 
 ## Retention filters
 

docs/victoriametrics/LTS-releases.md

@@ -22,5 +22,5 @@ to [the latest available releases](https://docs.victoriametrics.com/changelog/).
 
 ## Currently supported LTS release lines
 
-- v1.110.x - the latest one is [v1.110.3 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.3)
-- v1.102.x - the latest one is [v1.102.16 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.16)
+- v1.110.x - the latest one is [v1.110.5 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.5)
+- v1.102.x - the latest one is [v1.102.18 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.18)

docs/victoriametrics/Quick-Start.md

@@ -54,9 +54,9 @@ Download the latest available [Docker image of VictoriaMetrics](https://hub.dock
 and start it at port 8428, while storing the ingested data at `victoria-metrics-data` subdirectory
 under the current directory:
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.114.0
+docker pull victoriametrics/victoria-metrics:v1.115.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.114.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.115.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 
 You should see:
@@ -429,11 +429,12 @@ Understanding resource usage and performance of VictoriaMetrics also requires kn
 [slow inserts](https://docs.victoriametrics.com/faq/#what-is-a-slow-insert).
 All of them are present in [Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards).
 
-### Data safety
+### Data safety and backups
 
 It is recommended to read [Replication and data safety](https://docs.victoriametrics.com/cluster-victoriametrics/#replication-and-data-safety),
 [Why replication doesn’t save from disaster?](https://valyala.medium.com/speeding-up-backups-for-big-time-series-databases-533c1a927883)
-and [backups](https://docs.victoriametrics.com/single-server-victoriametrics/#backups).
+
+For backup configuration, please refer to [vmbackup documentation](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
 
 ### Configuring limits
 

docs/victoriametrics/README.md

@@ -1208,6 +1208,7 @@ Note that `production` builds are not supported via Podman because Podman does n
 
 ## How to work with snapshots
 
+### Create snapshot
 Send a request to `http://<victoriametrics-addr>:8428/snapshot/create` endpoint in order to create
 an [instant snapshot](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282).
 The page returns the following JSON response on successful creation of snapshot:
@@ -1233,6 +1234,8 @@ See also [snapshot troubleshooting](#snapshot-troubleshooting).
 
 The `http://<victoriametrics-addr>:8428/snapshot/list` endpoint returns the list of available snapshots.
 
+### Delete snapshots
+
 Send a query to `http://<victoriametrics-addr>:8428/snapshot/delete?snapshot=<snapshot-name>` in order
 to delete the snapshot with `<snapshot-name>` name.
 
@@ -2753,11 +2756,7 @@ See also [high availability docs](#high-availability) and [backup docs](#backups
 
 ## Backups
 
-VictoriaMetrics supports backups via [vmbackup](https://docs.victoriametrics.com/vmbackup/)
-and [vmrestore](https://docs.victoriametrics.com/vmrestore/) tools.
-We also provide [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/) tool for enterprise subscribers.
-Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
-See how to request a free trial license [here](https://victoriametrics.com/products/enterprise/trial/).
+For backup configuration and setup, please refer to [vmbackup documentation](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
 
 ## vmalert
 

docs/victoriametrics/Troubleshooting.md

@@ -446,7 +446,7 @@ If too much disk space is used by a [single-node VictoriaMetrics](https://docs.v
 at [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/), then please check the following:
 
 - Make sure that there are no old snapsots, since they can occupy disk space. See [how to work with snapshots](https://docs.victoriametrics.com/#how-to-work-with-snapshots)
-  and [snapshot troubleshooting](https://docs.victoriametrics.com/#snapshot-troubleshooting).
+  , [snapshot troubleshooting](https://docs.victoriametrics.com/#snapshot-troubleshooting) and [vmbackup troubleshooting](https://docs.victoriametrics.com/victoriametrics/vmbackup/#troubleshooting).
 
 - Under normal conditions the size of `<-storageDataPath>/indexdb` folder must be smaller than the size of `<-storageDataPath>/data` folder, where `-storageDataPath`
   is the corresponding command-line flag value. This can be checked by the following query if [VictoriaMetrics monitoring](#monitoring) is properly set up:

docs/victoriametrics/changelog/CHANGELOG.md

@@ -18,6 +18,13 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 ## tip
 
+* FEATURE: all the VictoriaMetrics components: mask `authKey` value from log messages. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973) for details.
+
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): remote write client sets correct content encoding header based on actual body content, rather than relying on configuration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8650).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): properly configure s3 client with if `configFilePath` is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8668) for details.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): support `identity` value in `Content-Encoding` HTTP header. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8649).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): fix timeseries with empty timestamp and value in Datadog sketches API. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8649).
+
 ## [v1.115.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.115.0)
 
 Released at 2025-04-04
@@ -58,7 +65,7 @@ Released at 2025-03-21
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation): fix panic on `rate` output. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8469).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation): prevent unexpected samples drop when [deduplication](https://docs.victoriametrics.com/stream-aggregation/#deduplication) and [aggregation windows](https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows) are enabled.
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): fix metric that shows number of active time series when per-day index is disabled. Previously, once per-day index was disabled, the active time series metric would stop being populated and the `Active time series` chart would show 0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8411) for details.
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric that shows number of active time series when per-day index is disabled. Previously, once per-day index was disabled, the active time series metric would stop being populated and the `Active time series` chart would show 0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8411) for details.
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix graceful shutdown delay when stopping the backup manager. Previously, sending a `SIGINT` would lead to a long delay before the backup manager would stop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8554) for details.
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): prevent a backup being scheduled 1 second after the previous one. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8499).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): do not set `vm_backup_last_run_failed` metric to `1` when running vmbackupmanager with an empty storage. Previously, the metrics was set to `1` even though backup was not created yet. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8490).
@@ -101,10 +108,10 @@ Released at 2025-03-07
 * FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229) and [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): show number of parts in the last partition on `LSM parts max by type` panel. Before, the resulting graph could be skewed by the max number of parts across all partitions. Displaying parts for the latest partition is the correct way to show if storage is currently impacted by merge delays.
 * FEATURE: [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): add panel `Partial query results` that shows the number of served [partial responses](https://docs.victoriametrics.com/cluster-victoriametrics/#cluster-availability) by vmselects.
 
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
-* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): properly apply global relabeling configuration, defined with  `-relabelConfig` flag, for metrics scrapped with `-promscrape.config`. Bug was introduces in [v1.108.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.108.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8389).
-* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): properly propagate an error message when applying retention policy fails. Previously, an actual error messages was discarded.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): properly apply global relabeling configuration, defined with  `-relabelConfig` flag, for metrics scraped with `-promscrape.config`. Bug was introduces in [v1.108.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.108.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8389).
+* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): properly propagate an error message when applying retention policy fails. Previously, an actual error message was discarded.
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): fix data query in [rate limiter](https://docs.victoriametrics.com/vmgateway/#rate-limiter). The bug was introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/68bad22fd26d1436ad0236b1f3ced8604c5d851c) starting from [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0).
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): properly apply the [rate limiter](https://docs.victoriametrics.com/vmgateway/#rate-limiter) for the `rows_inserted` limit type. Previously, the rate limit for this type was ignored.
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): properly handle HTTP requests with path ending with a trailing `/` when using the [rate limiter](https://docs.victoriametrics.com/vmgateway/#rate-limiter). Previously, the trailing slash was removed and caused an incorrect redirect path when visiting VMUI. Thanks to @jindov for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8439).
@@ -113,7 +120,7 @@ Released at 2025-03-07
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix an infinite loader on the [Downsampling filters debug page](https://docs.victoriametrics.com/#vmui) when provided configuration matches no series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8339).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): fix filters pushdown logic for expression like `foo{a="a"} ifnot bar{a="b"}`. Previously, filters from right operand were incorrectly propagated to the left operand and could result in empty query results even if `foo{a="a"}` matches time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8435).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent possible panic for `foo @ bar` expression when first sample in `bar` starts with `NaN` or starts long after first sample in `foo`. Now, VM will try to find first non-NaN value in `bar` and could yield an error `@ modifier must return a non-NaN value` if it won't find it. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8444).
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): prevent panic when using with rules that have zero interval: `-downsampling.period=5m:5m,0s:0s`. Such rule configuration shouldn't be rejected and cause an error when used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8454).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent panic when using with rules that have zero interval: `-downsampling.period=5m:5m,0s:0s`. Such rule configuration shouldn't be rejected and cause an error when used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8454).
 
 ## [v1.112.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.112.0)
 
@@ -124,9 +131,9 @@ Released at 2025-02-21
 * SECURITY: upgrade Go builder from Go1.23.5 to Go1.23.6. See the list of issues addressed in [Go1.23.6](https://github.com/golang/go/issues?q=milestone%3AGo1.23.6+label%3ACherryPickApproved).
 * SECURITY: upgrade base docker image (Alpine) from 3.21.2 to 3.21.3. See [Alpine 3.21.3 release notes](https://alpinelinux.org/posts/Alpine-3.18.12-3.19.7-3.20.6-3.21.3-released.html).
 
-* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): allow [disabling per-day indexes](https://docs.victoriametrics.com/#index-tuning-for-low-churn-rate) for workloads with low or zero churn rates via `-disablePerDayIndex` cmd-line flag. This option works the best for IoT cases and significantly loads resource and disk usage. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6976).
+* FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): allow [disabling per-day indexes](https://docs.victoriametrics.com/#index-tuning-for-low-churn-rate) for workloads with low or zero churn rates via `-disablePerDayIndex` cmd-line flag. This option works the best for IoT cases and significantly loads resource and disk usage. See [this PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6976).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): add command-line flag `-notifier.sendTimeout(default 10s)` to allow configuring request timeout when sending alerts to the corresponding `-notifier.url`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8287) for details. Thanks to @gjkim42 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8297).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): add support of [aggregation windows](https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows) to improve accuracy of stream aggregation. This feature is especially important for aggregation of [histrograms](https://docs.victoriametrics.com/keyconcepts/#histogram), but enabling this feature requires additional memory. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): add support of [aggregation windows](https://docs.victoriametrics.com/stream-aggregation/#aggregation-windows) to improve accuracy of stream aggregation. This feature is especially important for aggregation of [histograms](https://docs.victoriametrics.com/keyconcepts/#histogram), but enabling this feature requires additional memory. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): add `-influx.forceStreamMode` cmd-line flag to force stream processing for data ingested via [InfluxDB protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8269).
 * FEATURE: [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): support overriding vmstorage `-maxUniqueTimeseries` with specific resource limits: `-search.maxLabelsAPISeries`, `-search. maxSeries`, `-search.maxTSDBStatusSeries` and `-search.maxDeleteSeries`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7857).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add legend customization options - table view, hiding common values, grouping by label, and custom label format. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8031) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8263).
@@ -149,7 +156,7 @@ Released at 2025-02-21
 
 Released at 2025-02-10
 
-**Update note 1: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/) stop exposing `vm_index_search_duration_seconds` histogram metric. This metric records time spent on search operations in the index. It was introduced in [v1.56.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.56.0). However, this metric was used neither in dashboards nor in alerting rules. It also has high cardinality because index search operations latency can differ by 3 orders of magnitude. Hence, dropping it as unused.**
+**Update note 1: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/) stop exposing `vm_index_search_duration_seconds` histogram metric. This metric records time spent on search operations in the index. It was introduced in [v1.56.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.56.0). However, this metric was used neither in dashboards nor in alerting rules. It also has high cardinality because index search operations latency can differ by 3 orders of magnitude. Hence, dropping it as unused.**
 
 * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): improve startup times when opening a storage with the [retention](https://docs.victoriametrics.com/#retention) exceeding a few months.
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the ability to switch the heatmap to a line chart. Now, vmui would suggest to switch to line graph display if heatmap can't be properly rendered. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8057).
@@ -163,6 +170,20 @@ Released at 2025-02-10
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/cluster-victoriametrics/): fix discrepancies when using `or` binary operator. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640) issues for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly update number of unique series for [cardinality limiter](https://docs.victoriametrics.com/#cardinality-limiter) on ingestion. Previously, limit could undercount the real number of the ingested unique series. 
 
+## [v1.110.5](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.5)
+
+Released at 2025-04-04
+
+**v1.110.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/changelog/#v11100) release**
+
+* SECURITY: upgrade Go builder from Go1.23.6 to Go1.23.8. See the list of issues addressed in [Go1.23.8](https://github.com/golang/go/issues?q=milestone%3AGo1.23.8+label%3ACherryPickApproved).
+
+* BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): properly set the `Host` header when routing requests to `-write.url` and `-read.url`, which is needed for reverse proxies like Traefik.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): return `502 Bad Gateway` status code in case request failed due to a network timeout. Previously, vmauth would incorrectly return `200 OK`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8621).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmrestore](https://docs.victoriametrics.com/vmrestore/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix compatibility with S3-compatible storages which do not support data integrity checks. See [this issue](https://github.com/victoriaMetrics/victoriaMetrics/issues/8622).
+
 ## [v1.110.4](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.4)
 
 Released at 2025-03-21
@@ -193,7 +214,7 @@ The v1.110.x line will be supported for at least 12 months since [v1.110.0](http
 
 **Update note 1: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/) include a fix which enforces IPv6 addresses escaping for containers discovered with [Kubernetes service-discovery](https://docs.victoriametrics.com/sd_configs/#kubernetes_sd_configs) and `role: pod` which do not have exposed ports defined. This means that `address` for these containers will always be wrapped in square brackets, this might affect some relabeling rules which were relying on previous behaviour.**
 
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/): properly apply global relabeling configuration, defined with  `-relabelConfig` flag, for metrics scrapped with `-promscrape.config`. Bug was introduces in [v1.108.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.108.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8389).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): properly propagate an error message when applying retention policy fails. Previously, an actual error messages was discarded.
@@ -204,7 +225,7 @@ The v1.110.x line will be supported for at least 12 months since [v1.110.0](http
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix an infinite loader on the [Downsampling filters debug page](https://docs.victoriametrics.com/#vmui) when provided configuration matches no series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8339).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): fix filters pushdown logic for expression like `foo{a="a"} ifnot bar{a="b"}`. Previously, filters from right operand were incorrectly propagated to the left operand and could result in empty query results even if `foo{a="a"}` matches time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8435).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent possible panic for `foo @ bar` expression when first sample in `bar` starts with `NaN` or starts long after first sample in `foo`. Now, VM will try to find first non-NaN value in `bar` and could yield an error `@ modifier must return a non-NaN value` if it won't find it. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8444).
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): prevent panic when using with rules that have zero interval: `-downsampling.period=5m:5m,0s:0s`. Such rule configuration shouldn't be rejected and cause an error when used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8454).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent panic when using with rules that have zero interval: `-downsampling.period=5m:5m,0s:0s`. Such rule configuration shouldn't be rejected and cause an error when used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8454).
 
 ## [v1.110.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.2)
 
@@ -331,6 +352,19 @@ See changes [here](https://docs.victoriametrics.com/changelog_2024/#v11040)
 
 See changes [here](https://docs.victoriametrics.com/changelog_2024/#v11030)
 
+## [v1.102.18](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.18)
+
+Released at 2025-04-04
+
+**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
+
+* SECURITY: upgrade Go builder from Go1.23.6 to Go1.23.8. See the list of issues addressed in [Go1.23.8](https://github.com/golang/go/issues?q=milestone%3AGo1.23.8+label%3ACherryPickApproved).
+
+* BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): properly set the `Host` header when routing requests to `-write.url` and `-read.url`, which is needed for reverse proxies like Traefik.
+* BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth/): return `502 Bad Gateway` status code in case request failed due to a network timeout. Previously, vmauth would incorrectly return `200 OK`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8621).
+
 ## [v1.102.17](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.17)
 
 Released at 2025-03-21
@@ -361,7 +395,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): properly apply the [rate limiter](https://docs.victoriametrics.com/vmgateway/#rate-limiter) for the `rows_inserted` limit type. Previously, the rate limit for this type was ignored.
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/vmgateway): properly handle HTTP requests with path ending with a trailing `/` when using the [rate limiter](https://docs.victoriametrics.com/vmgateway/#rate-limiter). Previously, the trailing slash was removed and caused an incorrect redirect path when visiting VMUI. Thanks to @jindov for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8439).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent possible panic for `foo @ bar` expression when first sample in `bar` starts with `NaN` or starts long after first sample in `foo`. Now, VM will try to find first non-NaN value in `bar` and could yield an error `@ modifier must return a non-NaN value` if it won't find it. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8444).
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): prevent panic when using with rules that have zero interval: `-downsampling.period=5m:5m,0s:0s`. Such rule configuration shouldn't be rejected and cause an error when used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8454).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): prevent panic when using with rules that have zero interval: `-downsampling.period=5m:5m,0s:0s`. Such rule configuration shouldn't be rejected and cause an error when used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8454).
 
 ## [v1.102.15](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.15)
 
@@ -375,7 +409,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 
 * SECURITY: upgrade golang.org/x/net from v0.31.0 to v0.33.0 to address [GHSA-w32m-9786-jp63](https://github.com/advisories/GHSA-w32m-9786-jp63).
 
-* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
+* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/cluster-victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): properly escape IPv6 address in [Kubernetes service-discovery](https://docs.victoriametrics.com/sd_configs/#kubernetes_sd_configs) with `role: pod` for containers without exposed ports. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8374).
 

docs/victoriametrics/enterprise.md

@@ -82,7 +82,7 @@ VictoriaMetrics Enterprise components are available in the following forms:
 It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Binary releases of VictoriaMetrics Enterprise are available [at the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.114.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.115.0-enterprise.tar.gz`.
 
 In order to run binary release of VictoriaMetrics Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and unpack it. Then run the unpacked binary.
@@ -100,8 +100,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.114.0/victoria-metrics-linux-amd64-v1.114.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.114.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.115.0/victoria-metrics-linux-amd64-v1.115.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.115.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -116,7 +116,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for VictoriaMetrics Enterprise are available at VictoriaMetrics [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.114.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.115.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via command-line
 flag as described [here](#binary-releases).
@@ -126,13 +126,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.114.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.115.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.114.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.115.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -141,7 +141,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.114.0
+    image: victoriametrics/victoria-metrics:v1.115.0
     ports:
       - 8428:8428
     volumes:
@@ -173,7 +173,7 @@ is used to provide key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.114.0-enterprise
+    tag: v1.115.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -184,7 +184,7 @@ In order to provide key via existing secret, the following values file is used:
 ```yaml
 server:
   image:
-    tag: v1.114.0-enterprise
+    tag: v1.115.0-enterprise
 
 license:
   secret:
@@ -233,7 +233,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.114.0-enterprise
+    tag: v1.115.0-enterprise
 ```
 
 In order to provide key via existing secret, the following custom resource is used:
@@ -250,7 +250,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.114.0-enterprise
+    tag: v1.115.0-enterprise
 ```
 
 Example secret with license key:

docs/victoriametrics/scrape_config_examples.md

@@ -30,8 +30,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to the same directory:
 
 ```
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.114.0/victoria-metrics-linux-amd64-v1.114.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.114.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.115.0/victoria-metrics-linux-amd64-v1.115.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.115.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -146,8 +146,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/) acco
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.114.0/victoria-metrics-linux-amd64-v1.114.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.114.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.115.0/victoria-metrics-linux-amd64-v1.115.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.115.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/vmbackup.md

@@ -8,20 +8,41 @@ title: vmbackup
 aliases:
   - /vmbackup.html
 ---
-`vmbackup` creates VictoriaMetrics data backups from [instant snapshots](https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-work-with-snapshots).
-
-`vmbackup` supports incremental and full backups. Incremental backups are created automatically if the destination path already contains data from the previous backup.
-Full backups can be accelerated with `-origin` pointing to an already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
-data between the existing backup and new backup. It saves time and costs on data transfer.
+## Quick start  
 
+`vmbackup` creates VictoriaMetrics data backups to protect from user errors such as accidental data deletion.
+Whether you are using a single-node  or a cluster version, it is recommended to use `vmbackup` to perform periodical data backup from instant snapshots.
+More information how to work with them could be found in [instant snapshots](https://docs.victoriametrics.com/single-server-victoriametrics/#how-to-work-with-snapshots) documentation.
 Backup process can be interrupted at any time. It is automatically resumed from the interruption point when restarting `vmbackup` with the same args.
-
 Backed up data can be restored with [vmrestore](https://docs.victoriametrics.com/vmrestore/).
 
 See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-series-databases-533c1a927883) for more details.
 
-See also [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/) tool built on top of `vmbackup`. This tool simplifies
+If you are running enterprise version, you can also use [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/) tool built on top of `vmbackup`. This tool simplifies
 creation of hourly, daily, weekly and monthly backups.
+Enterprise binaries can be downloaded and evaluated for free from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+See how to request a free trial license [here](https://victoriametrics.com/products/enterprise/trial/).
+
+### Single node backup
+In order to perform a complete backup for VictoriaMetrics single node, refer to [full backup creation](https://docs.victoriametrics.com/victoriametrics/vmbackup/#full-backup).
+We recommend using smart backup strategy as a best practice.
+
+### Cluster backup
+
+In order to perform a complete backup for [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/), `vmbackup` must be run on each `vmstorage` node in cluster. Backups must
+be placed into different directories on the remote storage in order to avoid conflicts between backups from different nodes.
+
+For example, when creating a backup with 3 `vmstorage` nodes, the following commands must be run:
+
+```sh
+vmstorage-1$ /vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage1:8482/snapshot/create -dst=gs://<bucket>/vmstorage-1
+vmstorage-2$ /vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage2:8482/snapshot/create -dst=gs://<bucket>/vmstorage-2
+vmstorage-3$ /vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage3:8482/snapshot/create -dst=gs://<bucket>/vmstorage-3
+````
+
+Note that `vmbackup` needs access to data folder of every `vmstorage` node. It is recommended to run `vmbackup` on the same machine where `vmstorage` is running.
+For Kubernetes deployments it is recommended to use [sidecar containers](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) for running `vmbackup` on the same pod with `vmstorage`.
+
 
 ## Supported storage types
 
@@ -33,11 +54,20 @@ creation of hourly, daily, weekly and monthly backups.
 * Any S3-compatible storage such as [MinIO](https://github.com/minio/minio), [Ceph](https://docs.ceph.com/en/pacific/radosgw/s3/) or [Swift](https://platform.swiftstack.com/docs/admin/middleware/s3_middleware.html). See [these docs](#advanced-usage) for details.
 * Local filesystem. Example: `fs://</absolute/path/to/backup>`. Note that `vmbackup` prevents from storing the backup into the directory pointed by `-storageDataPath` command-line flag, since this directory should be managed solely by VictoriaMetrics or `vmstorage`.
 
-## Use cases
+## Backup types
+`vmbackup` supports incremental and full backups. Incremental backups are created automatically if the destination path already contains data from the previous backup.
+Full backups can be accelerated with `-origin` pointing to an already existing backup on the same remote storage. In this case `vmbackup` makes server-side copy for the shared
+data between the existing backup and new backup. It saves time and costs on data transfer.
 
-### Regular backups
+All commands below are provided for a single node version. 
+For a cluster version `vmbackup` should be executed on each vmstorage node and `-snapshot.createUrl` should point to vmstorage:
+```sh
+./vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage1:8482/snapshot/create -dst=gs://<bucket>/vmstorage-1
+```
 
-Regular backup can be performed with the following command:
+### Full backups
+
+Full backup can be performed on a single node with the following command:
 
 ```sh
 ./vmbackup -storageDataPath=</path/to/victoria-metrics-data> -snapshot.createURL=http://localhost:8428/snapshot/create -dst=gs://<bucket>/<path/to/new/backup>
@@ -49,7 +79,7 @@ Regular backup can be performed with the following command:
 * `<bucket>` is an already existing name for [GCS bucket](https://cloud.google.com/storage/docs/creating-buckets).
 * `<path/to/new/backup>` is the destination path where new backup will be placed.
 
-### Regular backups with server-side copy from existing backup
+### Full backups with server-side copy from existing backup
 
 If the destination GCS bucket already contains the previous backup at `-origin` path, then new backup can be accelerated
 with the following command:
@@ -74,8 +104,8 @@ It saves time and network bandwidth costs when working with big backups:
 ```
 
 ### Smart backups
-
-Smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
+Smart backup is a backup strategy that combines full and incremental backups with cleanup operations to efficiently manage data backups based on defined settings and available storage space.
+In case of VictoriaMetrics smart backups mean storing full daily backups into `YYYYMMDD` folders and creating incremental hourly backup into `latest` folder:
 
 * Run the following command every hour:
 
@@ -127,23 +157,6 @@ which perform full object copy during server-side copying. This may be slow and
 
 If the `-dst` already contains some data, then its' contents is synced with the `-origin` data. This allows making incremental server-side copies of backups.
 
-### Backups for VictoriaMetrics cluster
-
-`vmbackup` can be used for creating backups for [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/).
-In order to perform a complete backup for the cluster, `vmbackup` must be run on each `vmstorage` node in cluster. Backups must
-be placed into different directories on the remote storage in order to avoid conflicts between backups from different nodes.
-
-For example, when creating a backup with 3 `vmstorage` nodes, the following commands must be run:
-
-```sh
-vmstorage-1$ /vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage1:8482/snapshot/create -dst=gs://<bucket>/vmstorage-1
-vmstorage-2$ /vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage2:8482/snapshot/create -dst=gs://<bucket>/vmstorage-2
-vmstorage-3$ /vmbackup -storageDataPath=</path/to/vmstorage-data> -snapshot.createURL=http://vmstorage3:8482/snapshot/create -dst=gs://<bucket>/vmstorage-3
-````
-
-Note that `vmbackup` needs access to data folder of every `vmstorage` node. It is recommended to run `vmbackup` on the same machine where `vmstorage` is running.
-For Kubernetes deployments it is recommended to use [sidecar containers](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) for running `vmbackup` on the same pod with `vmstorage`.
-
 ## How does it work?
 
 The backup algorithm is the following:
@@ -176,10 +189,9 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
 * If the backup is slow, then try setting higher value for `-concurrency` flag. This will increase the number of concurrent workers that upload data to backup storage.
 * If `vmbackup` eats all the network bandwidth or CPU, then either decrease the `-concurrency` command-line flag value or set `-maxBytesPerSecond` command-line flag value to lower value.
 * If `vmbackup` consumes all the CPU on systems with big number of CPU cores, then try running it with `-filestream.disableFadvise` command-line flag.
-* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process.
+* If `vmbackup` has been interrupted due to temporary error, then just restart it with the same args. It will resume the backup process. After backup process has finished successfully, please remove old snapshot that was created during failed attempt.
 * Backups created from [single-node VictoriaMetrics](https://docs.victoriametrics.com/single-server-victoriametrics/) cannot be restored
   at [cluster VictoriaMetrics](https://docs.victoriametrics.com/cluster-victoriametrics/) and vice versa.
-
 ## Advanced usage
 
 
@@ -343,6 +355,7 @@ Run `vmbackup -help` in order to see all the available options:
   -dst string
      Where to put the backup on the remote storage. Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup/dir
      -dst can point to the previous backup. In this case incremental backup is performed, i.e. only changed data is uploaded
+     Note: If custom S3 endpoint is used, URL should contain only name of the bucket, while hostname of S3 server must be specified via the -customS3Endpoint command-line flag.
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable

docs/victoriametrics/vmbackupmanager.md

@@ -460,6 +460,7 @@ command-line flags:
      Disable weekly run. Default false
   -dst string
      The root folder of Victoria Metrics backups. Example: gs://bucket/path/to/backup/dir, s3://bucket/path/to/backup/dir or fs:///path/to/local/backup/dir
+     Note: If custom S3 endpoint is used, URL should contain only name of the bucket, while hostname of S3 server must be specified via the -customS3Endpoint command-line flag.
   -enableTCP6
      Whether to enable IPv6 for listening and dialing. By default, only IPv4 TCP and UDP are used
   -envflag.enable

docs/victoriametrics/vmrestore.md

@@ -188,6 +188,7 @@ Run `vmrestore -help` in order to see all the available options:
      Whether to skip checking for 'backup complete' file in -src. This may be useful for restoring from old backups, which were created without 'backup complete' file
   -src string
      Source path with backup on the remote storage. Example: gs://bucket/path/to/backup, s3://bucket/path/to/backup, azblob://container/path/to/backup or fs:///path/to/local/backup
+     Note: If custom S3 endpoint is used, URL should contain only name of the bucket, while hostname of S3 server must be specified via the -customS3Endpoint command-line flag.
   -storageDataPath string
      Destination path where backup must be restored. VictoriaMetrics must be stopped when restoring from backup. -storageDataPath dir can be non-empty. In this case the contents of -storageDataPath dir is synchronized with -src contents, i.e. it works like 'rsync --delete' (default "victoria-metrics-data")
   -tls array

lib/backup/s3remote/s3.go

@@ -98,7 +98,6 @@ func (fs *FS) Init() error {
 		fs.Dir += "/"
 	}
 	configOpts := []func(*config.LoadOptions) error{
-		config.WithSharedConfigProfile(fs.ProfileName),
 		config.WithDefaultRegion("us-east-1"),
 		config.WithRetryer(func() aws.Retryer {
 			return retry.NewStandard(func(o *retry.StandardOptions) {
@@ -113,6 +112,9 @@ func (fs *FS) Init() error {
 		}),
 	}
 
+	if len(fs.ProfileName) > 0 {
+		configOpts = append(configOpts, config.WithSharedConfigProfile(fs.ProfileName))
+	}
 	if len(fs.ConfigFilePath) > 0 {
 		configOpts = append(configOpts, config.WithSharedConfigFiles([]string{
 			fs.ConfigFilePath,

lib/encoding/util.go

@@ -0,0 +1,12 @@
+package encoding
+
+import "encoding/binary"
+
+// IsZstd checks if the given data is compressed using the zstd format.
+// It does this by verifying the presence of the zstd magic number (0xFD2FB528)
+// at the beginning of the byte slice.
+//
+// See: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+func IsZstd(data []byte) bool {
+	return len(data) >= 4 && binary.LittleEndian.Uint32(data) == 0xFD2FB528
+}

lib/encoding/util_test.go

@@ -0,0 +1,45 @@
+package encoding_test
+
+import (
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
+	"github.com/golang/snappy"
+)
+
+func TestIsZstd(t *testing.T) {
+	// nil
+	if encoding.IsZstd(nil) {
+		t.Fatalf("unexpected IsZstd result; got true; expecting false")
+	}
+
+	// empty
+	if encoding.IsZstd([]byte{}) {
+		t.Fatalf("unexpected IsZstd result; got true; expecting false")
+	}
+
+	// less than 4 bytes
+	if encoding.IsZstd([]byte(`foo`)) {
+		t.Fatalf("unexpected IsZstd result; got true; expecting false")
+	}
+
+	// plain text
+	if encoding.IsZstd([]byte(`foobar`)) {
+		t.Fatalf("unexpected IsZstd result; got true; expecting false")
+	}
+
+	// snappy compressed
+	if encoding.IsZstd(snappy.Encode(nil, []byte(`foobar`))) {
+		t.Fatalf("unexpected IsZstd result; got true; expecting false")
+	}
+
+	// zstd minimum compressed level
+	if !encoding.IsZstd(encoding.CompressZSTDLevel(nil, []byte(`foobar`), -22)) {
+		t.Fatalf("unexpected IsZstd result; got false; expecting true")
+	}
+
+	// zstd maximum compressed level
+	if !encoding.IsZstd(encoding.CompressZSTDLevel(nil, []byte(`foobar`), 22)) {
+		t.Fatalf("unexpected IsZstd result; got false; expecting true")
+	}
+}

lib/httpserver/httpserver.go

@@ -12,6 +12,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/pprof"
+	"net/url"
 	"os"
 	"runtime"
 	"strconv"
@@ -752,15 +753,32 @@ func GetRequestURI(r *http.Request) string {
 		return requestURI
 	}
 	_ = r.ParseForm()
-	queryArgs := r.PostForm.Encode()
-	if len(queryArgs) == 0 {
+	if len(r.PostForm) == 0 {
 		return requestURI
 	}
+	// code copied from url.Query.Encode
+	var queryArgs strings.Builder
+	for k := range r.PostForm {
+		vs := r.PostForm[k]
+		// mask authKey as well-known secret
+		if k == "authKey" {
+			vs = []string{"secret"}
+		}
+		keyEscaped := url.QueryEscape(k)
+		for _, v := range vs {
+			if queryArgs.Len() > 0 {
+				queryArgs.WriteByte('&')
+			}
+			queryArgs.WriteString(keyEscaped)
+			queryArgs.WriteByte('=')
+			queryArgs.WriteString(url.QueryEscape(v))
+		}
+	}
 	delimiter := "?"
 	if strings.Contains(requestURI, delimiter) {
 		delimiter = "&"
 	}
-	return requestURI + delimiter + queryArgs
+	return requestURI + delimiter + queryArgs.String()
 }
 
 // Redirect redirects to the given url.

lib/protoparser/datadogsketches/parser.go

@@ -164,7 +164,7 @@ func (s *Sketch) ToSummary() []*Metric {
 	maxPoints := make([]Point, 0, len(dogsketches))
 
 	for i, q := range quantiles {
-		points := make([]Point, len(dogsketches))
+		points := make([]Point, 0, len(dogsketches))
 		for _, d := range dogsketches {
 			timestamp := d.Ts * 1000
 			points = append(points, Point{

lib/protoparser/datadogsketches/stream/streamparser.go

@@ -16,6 +16,11 @@ import (
 // callback shouldn't hold series after returning.
 func Parse(r io.Reader, encoding string, callback func(series []*datadogsketches.Sketch) error) error {
 	readCalls.Inc()
+	// Datadog extensions sends Content-Encoding: identity, which is not supported by RFC 2616
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8649
+	if encoding == "identity" {
+		encoding = ""
+	}
 	err := protoparserutil.ReadUncompressedData(r, encoding, datadogutil.MaxInsertRequestSize, func(data []byte) error {
 		return parseData(data, callback)
 	})

lib/protoparser/datadogv1/stream/streamparser.go

@@ -16,6 +16,12 @@ import (
 // callback shouldn't hold series after returning.
 func Parse(r io.Reader, encoding string, callback func(series []datadogv1.Series) error) error {
 	readCalls.Inc()
+
+	// Datadog extensions sends Content-Encoding: identity, which is not supported by RFC 2616
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8649
+	if encoding == "identity" {
+		encoding = ""
+	}
 	err := protoparserutil.ReadUncompressedData(r, encoding, datadogutil.MaxInsertRequestSize, func(data []byte) error {
 		return parseData(data, callback)
 	})

lib/protoparser/datadogv2/stream/streamparser.go

@@ -16,6 +16,11 @@ import (
 // callback shouldn't hold series after returning.
 func Parse(r io.Reader, encoding, contentType string, callback func(series []datadogv2.Series) error) error {
 	readCalls.Inc()
+	// Datadog extensions sends Content-Encoding: identity, which is not supported by RFC 2616
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8649
+	if encoding == "identity" {
+		encoding = ""
+	}
 	err := protoparserutil.ReadUncompressedData(r, encoding, datadogutil.MaxInsertRequestSize, func(data []byte) error {
 		return parseData(data, contentType, callback)
 	})

lib/protoparser/promremotewrite/stream/streamparser.go

@@ -44,6 +44,10 @@ func Parse(r io.Reader, isVMRemoteWrite bool, callback func(tss []prompb.TimeSer
 			// Fall back to Snappy decompression, since vmagent may send snappy-encoded messages
 			// with 'Content-Encoding: zstd' header if they were put into persistent queue before vmagent restart.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5301
+			//
+			// Newer vmagent sends proper 'Content-Encoding' header.
+			// The logic is preserved for backwards compatibility.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8650
 			zstdErr := err
 			bb.B, err = snappy.Decode(bb.B[:cap(bb.B)], ctx.reqBuf.B)
 			if err != nil {
@@ -56,6 +60,10 @@ func Parse(r io.Reader, isVMRemoteWrite bool, callback func(tss []prompb.TimeSer
 			// Fall back to zstd decompression, since vmagent may send zstd-encoded messages
 			// without 'Content-Encoding: zstd' header if they were put into persistent queue before vmagent restart.
 			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5301#issuecomment-1815871992
+			//
+			// Newer vmagent sends proper 'Content-Encoding' header.
+			// The logic is preserved for backwards compatibility.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8650
 			snappyErr := err
 			bb.B, err = zstd.Decompress(bb.B[:0], ctx.reqBuf.B)
 			if err != nil {