apptest: Add basic backup/restore integration test (#9133 )

This commit adds a basic backup/restore test for vmsingle and vmcluster. A more sophisticated was originally added to the partition index PR (#8134) and was aimed to test backup/restore when switching back and forth between legacy and partition index. During the code review it was decided that it would be good to have a separate test as well since legacy code will be removed in future and so will the test. Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
docs/vmanomaly: release v1.23.2 (#9135 )
2026-06-18 00:03:47 +03:00 · 2025-06-09 12:17:54 +02:00 · 2025-06-09 11:28:56 +02:00 · 2025-06-08 23:07:02 +02:00 · 2025-06-08 13:47:13 +02:00 · 2025-06-08 09:12:54 +02:00
34 changed files with 692 additions and 280 deletions
--- a/2
+++ b/2
@@ -545,7 +545,7 @@ test-full:
 test-full-386:
 	GOEXPERIMENT=synctest GOARCH=386 go test -coverprofile=coverage.txt -covermode=atomic ./lib/... ./app/...

-integration-test: victoria-metrics vmagent vmalert vmauth vmctl
+integration-test: victoria-metrics vmagent vmalert vmauth vmctl vmbackup vmrestore
 	go test ./apptest/... -skip="^TestCluster.*"

 benchmark:
--- a/app/vlselect/vmui/assets/index-D5re9hC6.css
+++ b/app/vlselect/vmui/assets/index-D5re9hC6.css
--- a/app/vlselect/vmui/assets/index-DhqzKCNf.js
+++ b/app/vlselect/vmui/assets/index-DhqzKCNf.js
--- a/app/vlselect/vmui/index.html
+++ b/app/vlselect/vmui/index.html
@@ -35,10 +35,10 @@
  <meta property="og:title" content="UI for VictoriaLogs">
  <meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
  <meta property="og:description" content="Explore your log data with VictoriaLogs UI">
-  <script type="module" crossorigin src="./assets/index-BaRvaPfA.js"></script>
+  <script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
  <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -818,6 +818,7 @@ func QueryHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseWr
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
 		EnforcedTagFilterss: etfs,
+		CacheTagFilters:     etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
@@ -927,6 +928,7 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
 		LookbackDelta:       lookbackDelta,
 		RoundDigits:         getRoundDigits(r),
 		EnforcedTagFilterss: etfs,
+		CacheTagFilters:     etfs,
 		GetRequestURI: func() string {
 			return httpserver.GetRequestURI(r)
 		},
--- a/app/vmselect/promql/eval.go
+++ b/app/vmselect/promql/eval.go
@@ -140,6 +140,13 @@ type EvalConfig struct {
 	// EnforcedTagFilterss may contain additional label filters to use in the query.
 	EnforcedTagFilterss [][]storage.TagFilter

+	// CacheTagFilters stores the original tag-filter sets and extra_label from the request.
+	// The slice is never modified after creation and is used only to build
+	// the query-cache key.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9001
+	CacheTagFilters [][]storage.TagFilter
+
 	// The callback, which returns the request URI during logging.
 	// The request URI isn't stored here because its' construction may take non-trivial amounts of CPU.
 	GetRequestURI func() string
@@ -166,6 +173,7 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
 	ec.LookbackDelta = src.LookbackDelta
 	ec.RoundDigits = src.RoundDigits
 	ec.EnforcedTagFilterss = src.EnforcedTagFilterss
+	ec.CacheTagFilters = src.CacheTagFilters
 	ec.GetRequestURI = src.GetRequestURI
 	ec.QueryStats = src.QueryStats

--- a/app/vmselect/promql/rollup_result_cache.go
+++ b/app/vmselect/promql/rollup_result_cache.go
@@ -291,7 +291,7 @@ func (rrc *rollupResultCache) GetSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	bb := bbPool.Get()
 	defer bbPool.Put(bb)

-	bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+	bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
 	metainfoBuf := rrc.c.Get(nil, bb.B)
 	if len(metainfoBuf) == 0 {
 		qt.Printf("nothing found")
@@ -313,7 +313,7 @@ func (rrc *rollupResultCache) GetSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	if !ok {
 		mi.RemoveKey(key)
 		metainfoBuf = mi.Marshal(metainfoBuf[:0])
-		bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+		bb.B = marshalRollupResultCacheKeyForSeries(bb.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
 		rrc.c.Set(bb.B, metainfoBuf)
 		return nil, ec.Start
 	}
@@ -419,7 +419,7 @@ func (rrc *rollupResultCache) PutSeries(qt *querytracer.Tracer, ec *EvalConfig,
 	metainfoBuf := bbPool.Get()
 	defer bbPool.Put(metainfoBuf)

-	metainfoKey.B = marshalRollupResultCacheKeyForSeries(metainfoKey.B[:0], expr, window, ec.Step, ec.EnforcedTagFilterss)
+	metainfoKey.B = marshalRollupResultCacheKeyForSeries(metainfoKey.B[:0], expr, window, ec.Step, ec.CacheTagFilters)
 	metainfoBuf.B = rrc.c.Get(metainfoBuf.B[:0], metainfoKey.B)
 	var mi rollupResultCacheMetainfo
 	if len(metainfoBuf.B) > 0 {
--- a/app/vmselect/vmui/assets/MetricsQL--yJ2CeO1.md
+++ b/app/vmselect/vmui/assets/MetricsQL--yJ2CeO1.md
--- a/app/vmselect/vmui/assets/index-D-ssBbZq.js
+++ b/app/vmselect/vmui/assets/index-D-ssBbZq.js
--- a/app/vmselect/vmui/assets/index-D5re9hC6.css
+++ b/app/vmselect/vmui/assets/index-D5re9hC6.css
--- a/app/vmselect/vmui/assets/index-xmjGcv4-.js
+++ b/app/vmselect/vmui/assets/index-xmjGcv4-.js
--- a/app/vmselect/vmui/index.html
+++ b/app/vmselect/vmui/index.html
@@ -36,10 +36,10 @@
  <meta property="og:title" content="UI for VictoriaMetrics">
  <meta property="og:url" content="https://victoriametrics.com/">
  <meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
-  <script type="module" crossorigin src="./assets/index-xmjGcv4-.js"></script>
+  <script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
  <link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
  <link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
-  <link rel="stylesheet" crossorigin href="./assets/index-C85_NB5q.css">
+  <link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
 </head>
 <body>
 <noscript>You need to enable JavaScript to run this app.</noscript>
--- a/app/vmui/Dockerfile-web
+++ b/app/vmui/Dockerfile-web
@@ -1,4 +1,4 @@
-FROM golang:1.24.3 AS build-web-stage
+FROM golang:1.24.4 AS build-web-stage
 COPY build /build

 WORKDIR /build
@@ -6,7 +6,7 @@ COPY web/ /build/
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
    GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/

-FROM alpine:3.21.3
+FROM alpine:3.22.0
 USER root

 COPY --from=build-web-stage /build/web-amd64 /app/web
--- a/apptest/testcase.go
+++ b/apptest/testcase.go
@@ -189,6 +189,36 @@ func (tc *TestCase) MustStartVmauth(instance string, flags []string, configFileY
 	return app
 }

+// MustStartVmbackup is a test helper that starts an instance of vmbackup
+// and waits until the app exits. It fails the test if the app fails to start or
+// exits with non zero code.
+func (tc *TestCase) MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string) {
+	tc.t.Helper()
+
+	if err := StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst); err != nil {
+		tc.t.Fatalf("vmbackup %q failed to start or exited with non-zero code: %v", instance, err)
+	}
+
+	// Do not add the process to the list of running apps using
+	// tc.addApp(instance, app), because the method blocks until the process
+	// exits.
+}
+
+// MustStartVmrestore is a test helper that starts an instance of vmrestore
+// and waits until the app exits. It fails the test if the app fails to start or
+// exits with non zero code.
+func (tc *TestCase) MustStartVmrestore(instance, src, storageDataPath string) {
+	tc.t.Helper()
+
+	if err := StartVmrestore(instance, src, storageDataPath); err != nil {
+		tc.t.Fatalf("vmrestore %q failed to start or exited with non-zero code: %v", instance, err)
+	}
+
+	// Do not add the process to the list of running apps using
+	// tc.addApp(instance, app), because the method blocks until the process
+	// exits.
+}
+
 // MustStartDefaultCluster starts a typical cluster configuration with default
 // flags.
 func (tc *TestCase) MustStartDefaultCluster() *Vmcluster {
--- a/apptest/tests/backup_restore_test.go
+++ b/apptest/tests/backup_restore_test.go
@@ -0,0 +1,239 @@
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+)
+
+type testBackupRestoreOpts struct {
+	startSUT           func() at.PrometheusWriteQuerier
+	stopSUT            func()
+	storageDataPaths   []string
+	snapshotCreateURLs func(at.PrometheusWriteQuerier) []string
+}
+
+func TestSingleBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storageDataPath := filepath.Join(tc.Dir(), "vmsingle")
+
+	opts := testBackupRestoreOpts{
+		startSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartVmsingle("vmsingle", []string{
+				"-storageDataPath=" + storageDataPath,
+				"-retentionPeriod=100y",
+				"-search.maxStalenessInterval=1m",
+			})
+		},
+		stopSUT: func() {
+			tc.StopApp("vmsingle")
+		},
+		storageDataPaths: []string{
+			storageDataPath,
+		},
+		snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
+			return []string{
+				sut.(*at.Vmsingle).SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testBackupRestore(tc, opts)
+}
+
+func TestClusterBackupRestore(t *testing.T) {
+	tc := at.NewTestCase(t)
+	defer tc.Stop()
+
+	storage1DataPath := filepath.Join(tc.Dir(), "vmstorage1")
+	storage2DataPath := filepath.Join(tc.Dir(), "vmstorage2")
+
+	opts := testBackupRestoreOpts{
+		startSUT: func() at.PrometheusWriteQuerier {
+			return tc.MustStartCluster(&at.ClusterOptions{
+				Vmstorage1Instance: "vmstorage1",
+				Vmstorage1Flags: []string{
+					"-storageDataPath=" + storage1DataPath,
+					"-retentionPeriod=100y",
+				},
+				Vmstorage2Instance: "vmstorage2",
+				Vmstorage2Flags: []string{
+					"-storageDataPath=" + storage2DataPath,
+					"-retentionPeriod=100y",
+				},
+				VminsertInstance: "vminsert",
+				VminsertFlags:    []string{},
+				VmselectInstance: "vmselect",
+				VmselectFlags: []string{
+					"-search.maxStalenessInterval=1m",
+				},
+			})
+		},
+		stopSUT: func() {
+			tc.StopApp("vminsert")
+			tc.StopApp("vmselect")
+			tc.StopApp("vmstorage1")
+			tc.StopApp("vmstorage2")
+		},
+		storageDataPaths: []string{
+			storage1DataPath,
+			storage2DataPath,
+		},
+		snapshotCreateURLs: func(sut at.PrometheusWriteQuerier) []string {
+			c := sut.(*at.Vmcluster)
+			return []string{
+				c.Vmstorages[0].SnapshotCreateURL(),
+				c.Vmstorages[1].SnapshotCreateURL(),
+			}
+		},
+	}
+
+	testBackupRestore(tc, opts)
+}
+
+func testBackupRestore(tc *at.TestCase, opts testBackupRestoreOpts) {
+	t := tc.T()
+
+	const msecPerMinute = 60 * 1000
+	genData := func(count int, prefix string, start int64) (recs []string, wantSeries []map[string]string, wantQueryResults []*at.QueryResult) {
+		recs = make([]string, count)
+		wantSeries = make([]map[string]string, count)
+		wantQueryResults = make([]*at.QueryResult, count)
+		for i := range count {
+			name := fmt.Sprintf("%s_%03d", prefix, i)
+			value := float64(i)
+			timestamp := start + int64(i)*msecPerMinute
+
+			recs[i] = fmt.Sprintf("%s %f %d", name, value, timestamp)
+			wantSeries[i] = map[string]string{"__name__": name}
+			wantQueryResults[i] = &at.QueryResult{
+				Metric:  map[string]string{"__name__": name},
+				Samples: []*at.Sample{{Timestamp: timestamp, Value: value}},
+			}
+		}
+		return recs, wantSeries, wantQueryResults
+	}
+
+	backupBaseDir, err := filepath.Abs(filepath.Join(tc.Dir(), "backups"))
+	if err != nil {
+		t.Fatalf("could not get absolute path for the backup base dir")
+	}
+
+	// assertSeries retrieves set of all metric names from the storage and
+	// compares it with the expected set.
+	assertSeries := func(app at.PrometheusQuerier, query string, start, end int64, want []map[string]string) {
+		t.Helper()
+
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/series response",
+			Got: func() any {
+				return app.PrometheusAPIV1Series(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+				}).Sort()
+			},
+			Want: &at.PrometheusAPIV1SeriesResponse{
+				Status: "success",
+				Data:   want,
+			},
+			FailNow: true,
+		})
+	}
+
+	// assertSeries retrieves all data from the storage and compares it with the
+	// expected result.
+	assertQueryResults := func(app at.PrometheusQuerier, query string, start, end int64, want []*at.QueryResult) {
+		t.Helper()
+		tc.Assert(&at.AssertOptions{
+			Msg: "unexpected /api/v1/query_range response",
+			Got: func() any {
+				return app.PrometheusAPIV1QueryRange(t, query, at.QueryOpts{
+					Start: fmt.Sprintf("%d", start),
+					End:   fmt.Sprintf("%d", end),
+					Step:  "60s",
+				})
+			},
+			Want: &at.PrometheusAPIV1QueryResponse{
+				Status: "success",
+				Data: &at.QueryData{
+					ResultType: "matrix",
+					Result:     want,
+				},
+			},
+			FailNow: true,
+			Retries: 300,
+		})
+	}
+
+	createBackup := func(sut at.PrometheusWriteQuerier, name string) {
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmbackup-%s-%s", name, replica)
+			snapshotCreateURL := opts.snapshotCreateURLs(sut)[i]
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmbackup(instance, storageDataPath, snapshotCreateURL, backupPath)
+		}
+	}
+
+	restoreFromBackup := func(name string) {
+		for i, storageDataPath := range opts.storageDataPaths {
+			replica := fmt.Sprintf("replica-%d", i)
+			instance := fmt.Sprintf("vmrestore-%s-%s", name, replica)
+			backupPath := "fs://" + filepath.Join(backupBaseDir, name, replica)
+			tc.MustStartVmrestore(instance, backupPath, storageDataPath)
+		}
+	}
+
+	// Use the same number of metrics and time range for all the data ingestions
+	// below.
+	const numMetrics = 1000
+	// With 1000 metrics (one per minute), the time range spans 2 months.
+	end := time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC).UnixMilli()
+	start := end - numMetrics*msecPerMinute
+
+	// Verify backup/restore:
+	//
+	// - Start vmsingle with empty storage data dir.
+	// - Ingest first batch or records (batch1) and ensure they can be queried.
+	// - Create batch1 backup
+	// - Ingest second batch of records (batch2) and ensure the queries return
+	//   (batch1 + batch2) data.
+	// - Stop vmsingle
+	// - Restore batch1 from backup
+	// - Start vmsingle
+	// - Ensure that the queries return batch1 data only.
+
+	batch1Data, wantBatch1Series, wantBatch1QueryResults := genData(numMetrics, "batch1", start)
+	batch2Data, wantBatch2Series, wantBatch2QueryResults := genData(numMetrics, "batch2", start)
+	wantBatch12Series := slices.Concat(wantBatch1Series, wantBatch2Series)
+	wantBatch12QueryResults := slices.Concat(wantBatch1QueryResults, wantBatch2QueryResults)
+
+	sut := opts.startSUT()
+
+	sut.PrometheusAPIV1ImportPrometheus(t, batch1Data, at.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+	createBackup(sut, "batch1")
+
+	sut.PrometheusAPIV1ImportPrometheus(t, batch2Data, at.QueryOpts{})
+	sut.ForceFlush(t)
+	assertSeries(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12Series)
+	assertQueryResults(sut, `{__name__=~"batch(1|2).*"}`, start, end, wantBatch12QueryResults)
+	createBackup(sut, "batch12")
+
+	opts.stopSUT()
+
+	restoreFromBackup("batch1")
+
+	sut = opts.startSUT()
+
+	assertSeries(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1Series)
+	assertQueryResults(sut, `{__name__=~"batch1.*"}`, start, end, wantBatch1QueryResults)
+}
--- a/apptest/tests/rollup_result_cache_test.go
+++ b/apptest/tests/rollup_result_cache_test.go
@@ -0,0 +1,78 @@
+package tests
+
+import (
+	"os"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func TestClusterRollupResultCache(t *testing.T) {
+	os.RemoveAll(t.Name())
+
+	cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
+
+	tc := apptest.NewTestCase(t)
+	defer tc.Stop()
+	vmstorage := tc.MustStartVmstorage("vmstorage", []string{
+		"-storageDataPath=" + tc.Dir() + "/vmstorage",
+		"-retentionPeriod=100y",
+	})
+	vminsert := tc.MustStartVminsert("vminsert", []string{
+		"-storageNode=" + vmstorage.VminsertAddr(),
+	})
+	vmselect := tc.MustStartVmselect("vmselect", []string{
+		"-storageNode=" + vmstorage.VmselectAddr(),
+		"-search.tenantCacheExpireDuration=0",
+	})
+
+	var tenantLabelsSamples = []string{
+		`foo_bar{vm_account_id="5"} 1.00 1652169720000`,                    // 2022-05-10T08:00:00Z'
+		`foo_bar{vm_account_id="5",vm_project_id="15"} 3.00 1652169720000`, // 2022-05-10T08:02:00Z
+	}
+
+	vminsert.PrometheusAPIV1ImportPrometheus(t, tenantLabelsSamples, apptest.QueryOpts{Tenant: "multitenant"})
+	vmstorage.ForceFlush(t)
+
+	want := apptest.NewPrometheusAPIV1QueryResponse(t,
+		`{"data":
+	   {"result":[
+	        {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id": "0"},"values":[[1652169720,"1"],[1652169780,"1"]]},
+	        {"metric":{"__name__":"foo_bar","vm_account_id":"5","vm_project_id":"15"},"values":[[1652169720,"3"],[1652169780,"3"]]}
+	               ]
+	   }
+	}`,
+	)
+
+	got := vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
+		Tenant:       "multitenant",
+		Start:        "2022-05-10T07:59:00.000Z",
+		End:          "2022-05-10T08:05:00.000Z",
+		Step:         "1m",
+		ExtraFilters: []string{`{vm_account_id="5",vm_project_id="15"}`, `{vm_account_id="5",vm_project_id="0"}`},
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+	want = apptest.NewPrometheusAPIV1QueryResponse(t,
+		`{"data":
+	   {"result":[]}
+	}`,
+	)
+
+	got = vmselect.PrometheusAPIV1QueryRange(t, `foo_bar{}`, apptest.QueryOpts{
+		Tenant:       "multitenant",
+		Start:        "2022-05-10T07:59:00.000Z",
+		End:          "2022-05-10T08:05:00.000Z",
+		Step:         "1m",
+		ExtraFilters: []string{`{vm_account_id="99",vm_project_id="99"}`},
+	})
+	if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
+		t.Errorf("unexpected response (-want, +got):\n%s", diff)
+	}
+
+}
--- a/apptest/vmbackup.go
+++ b/apptest/vmbackup.go
@@ -0,0 +1,13 @@
+package apptest
+
+// StartVmbackup starts an instance of vmbackup with the given flags and waits
+// until it exits.
+func StartVmbackup(instance, storageDataPath, snapshotCreateURL, dst string) error {
+	flags := []string{
+		"-storageDataPath=" + storageDataPath,
+		"-snapshot.createURL=" + snapshotCreateURL,
+		"-dst=" + dst,
+	}
+	_, _, err := startApp(instance, "../../bin/vmbackup", flags, &appOptions{wait: true})
+	return err
+}
--- a/apptest/vmrestore.go
+++ b/apptest/vmrestore.go
@@ -0,0 +1,12 @@
+package apptest
+
+// StartVmrestore starts an instance of vmrestore with the given flags and waits
+// until it exits.
+func StartVmrestore(instance, src, storageDataPath string) error {
+	flags := []string{
+		"-src=" + src,
+		"-storageDataPath=" + storageDataPath,
+	}
+	_, _, err := startApp(instance, "../../bin/vmrestore", flags, &appOptions{wait: true})
+	return err
+}
--- a/apptest/vmsingle.go
+++ b/apptest/vmsingle.go
@@ -363,8 +363,7 @@ func (app *Vmsingle) APIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts Qu
 func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	t.Helper()

-	queryURL := fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, "", nil)
+	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), "", nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -377,6 +376,11 @@ func (app *Vmsingle) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	return &res
 }

+// SnapshotCreateURL returns the URL for creating snapshots.
+func (app *Vmsingle) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
+}
+
 // APIV1AdminTSDBSnapshot creates a database snapshot by sending a query to the
 // /api/v1/admin/tsdb/snapshot endpoint.
 //
--- a/apptest/vmstorage.go
+++ b/apptest/vmstorage.go
@@ -99,8 +99,7 @@ func (app *Vmstorage) ForceMerge(t *testing.T) {
 func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	t.Helper()

-	queryURL := fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
-	data, statusCode := app.cli.Post(t, queryURL, "", nil)
+	data, statusCode := app.cli.Post(t, app.SnapshotCreateURL(), "", nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -113,6 +112,11 @@ func (app *Vmstorage) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	return &res
 }

+// SnapshotCreateURL returns the URL for creating snapshots.
+func (app *Vmstorage) SnapshotCreateURL() string {
+	return fmt.Sprintf("http://%s/snapshot/create", app.httpListenAddr)
+}
+
 // SnapshotList lists existing database snapshots by sending a query to the
 // /snapshot/list endpoint.
 //
--- a/deployment/docker/Makefile
+++ b/deployment/docker/Makefile
@@ -3,11 +3,11 @@
 DOCKER_REGISTRIES ?= docker.io quay.io
 DOCKER_NAMESPACE ?= victoriametrics

-ROOT_IMAGE ?= alpine:3.21.3
+ROOT_IMAGE ?= alpine:3.22.0
 ROOT_IMAGE_SCRATCH ?= scratch
-CERTS_IMAGE := alpine:3.21.3
+CERTS_IMAGE := alpine:3.22.0

-GO_BUILDER_IMAGE := golang:1.24.3-alpine
+GO_BUILDER_IMAGE := golang:1.24.4-alpine
 BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
 BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
 DOCKER ?= docker
--- a/deployment/docker/builder/Dockerfile
+++ b/deployment/docker/builder/Dockerfile
@@ -6,7 +6,7 @@ RUN apk add git gcc musl-dev make wget --no-cache && \
    cd /opt/cross-builder && \
    for arch in aarch64 x86_64; do \
        wget \
-            https://musl.cc/${arch}-linux-musl-cross.tgz \
+            https://github.com/VictoriaMetrics/muslcc-mirror/releases/download/v1.0.0/${arch}-linux-musl-cross.tgz \
            -O /opt/cross-builder/${arch}-musl.tgz \
            --no-verbose && \
        tar zxf ${arch}-musl.tgz -C ./  && \
--- a/deployment/docker/vmanomaly/vmanomaly-integration/compose.yml
+++ b/deployment/docker/vmanomaly/vmanomaly-integration/compose.yml
@@ -59,7 +59,7 @@ services:
      - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
    restart: always
  vmanomaly:
-    image: victoriametrics/vmanomaly:v1.23.0
+    image: victoriametrics/vmanomaly:v1.23.2
    depends_on:
      - "victoriametrics"
    ports:
--- a/deployment/logs-benchmark/docker-compose-elk.yml
+++ b/deployment/logs-benchmark/docker-compose-elk.yml
@@ -18,7 +18,7 @@ services:
      - vlogs

  generator:
-    image: golang:1.24.3-alpine
+    image: golang:1.24.4-alpine
    restart: always
    working_dir: /go/src/app
    volumes:
--- a/deployment/logs-benchmark/docker-compose-loki.yml
+++ b/deployment/logs-benchmark/docker-compose-loki.yml
@@ -2,7 +2,7 @@ version: "3"

 services:
  generator:
-    image: golang:1.24.3-alpine
+    image: golang:1.24.4-alpine
    restart: always
    working_dir: /go/src/app
    volumes:
--- a/docs/anomaly-detection/CHANGELOG.md
+++ b/docs/anomaly-detection/CHANGELOG.md
@@ -14,6 +14,20 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.

+## v1.23.2
+Released: 2025-06-09
+
+- IMPROVEMENT: Increased convergence speed for [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score), [ZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score), [MADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#mad), and [OnlineMADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) models. Now it works better for tight optimization budgets (n_trials < 10, timeout < 1s)
+
+- BUGFIX: Now mean and variance of [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) with exponential `decay` < 1 [arg](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay) are properly calculated for unbiased predictions.
+
+## v1.23.1
+Released: 2025-06-08
+
+- BUGFIX: In [sharding mode](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability) the corner case when shard number (`VMANOMALY_MEMBER_NUM`) is greater than the number of configured shards (`VMANOMALY_MEMBERS_COUNT`) is now properly handled.
+
+- BUGFIX: In [sharding mode](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability), the corner case when the number of produced [sub-configurations](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#sub-configuration) is less than the number of configured shards (`VMANOMALY_MEMBERS_COUNT`) is now properly handled. Until config hot-reload is supported, such "idle" shards will be turned off with exit code 1 and respective critical message logged.
+
 ## v1.23.0
 Released: 2025-06-05

--- a/docs/anomaly-detection/FAQ.md
+++ b/docs/anomaly-detection/FAQ.md
@@ -224,7 +224,7 @@ services:
  # ...
  vmanomaly:
    container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.23.0
+    image: victoriametrics/vmanomaly:v1.23.2
    # ...
    ports:
      - "8490:8490"
@@ -432,7 +432,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:

 ```sh
-docker pull victoriametrics/vmanomaly:v1.23.0 && docker image tag victoriametrics/vmanomaly:v1.23.0 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.23.2 && docker image tag victoriametrics/vmanomaly:v1.23.2 vmanomaly
 ```

 ```sh
--- a/docs/anomaly-detection/QuickStart.md
+++ b/docs/anomaly-detection/QuickStart.md
@@ -118,13 +118,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:

 ```sh
-docker pull victoriametrics/vmanomaly:v1.23.0
+docker pull victoriametrics/vmanomaly:v1.23.2
 ```

 2. (Optional step) tag the `vmanomaly` Docker image:

 ```sh
-docker image tag victoriametrics/vmanomaly:v1.23.0 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.23.2 vmanomaly
 ```

 3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@@ -158,7 +158,7 @@ docker run -it --user 1000:1000 \
 services:
  # ...
  vmanomaly:
-    image: victoriametrics/vmanomaly:v1.23.0
+    image: victoriametrics/vmanomaly:v1.23.2
    volumes:
        $YOUR_LICENSE_FILE_PATH:/license
        $YOUR_CONFIG_FILE_PATH:/config.yml
--- a/docs/anomaly-detection/components/models.md
+++ b/docs/anomaly-detection/components/models.md
@@ -1276,7 +1276,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:

 ```sh
-docker pull victoriametrics/vmanomaly:v1.23.0
+docker pull victoriametrics/vmanomaly:v1.23.2
 ```

 Now we can run the docker container putting as volumes both config and model file:
@@ -1290,7 +1290,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.23.0 /config.yaml \
+victoriametrics/vmanomaly:v1.23.2 /config.yaml \
 --licenseFile=/license
 ```

--- a/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
+++ b/docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md
@@ -387,7 +387,7 @@ services:
    restart: always
  vmanomaly:
    container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.23.0
+    image: victoriametrics/vmanomaly:v1.23.2
    depends_on:
      - "victoriametrics"
    ports:
--- a/docs/victorialogs/querying/README.md
+++ b/docs/victorialogs/querying/README.md
@@ -954,6 +954,7 @@ There are three modes of displaying query results:
 - `Group` - results are displayed as a table with rows grouped by [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
 - `Table` - displays query results as a table.
 - `JSON` - displays raw JSON response from [`/select/logsql/query` HTTP API](#querying-logs).
+- `Live` - displays [live tailing](#live-tailing) results for the given query.

 See also [command line interface](#command-line).

--- a/docs/victoriametrics/changelog/CHANGELOG.md
+++ b/docs/victoriametrics/changelog/CHANGELOG.md
@@ -18,6 +18,13 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel

 ## tip

+* SECURITY: upgrade Go builder from Go1.24.3 to Go1.24.4. See [the list of issues addressed in Go1.24.4](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved).
+* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
+
+## [v1.119.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.119.0)
+
+Released at 2025-06-06
+
 * FEATURE: improve performance on systems with many CPU cores by removing the top sources of [false sharing](https://en.wikipedia.org/wiki/False_sharing) at global variables. See [#8682](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8682). Thanks to @tIGO for raising this issue and for the initial attempt to fix it at the [PR #8683](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8683).
 * FEATURE: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): add an option to use mTLS for connections to `-write.url` and `-read.url`. See [#8841](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8841).
 * FEATURE: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): dynamically adjusts the concurrent dial limit between 8 and 64 based on `-search.maxConcurrentRequests`. Additionally, goroutines now have the opportunity to access available connections while awaiting the dial limit token. This enables faster connection establishment when sudden requests arrive and reduces the blocking time during the availability check for connections. See [#8922](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8922)
@@ -38,6 +45,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): enable dual-stack network mode for `vmctl`connections by default. This allows connecting to IPv6 endpoints as it was not possible previously. See [#9116](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9116).
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): fix access conflicts for the temporary test folder when multiple users run tests on the same host. Thanks to @evkuzin for the [PR 9015](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9015).
 * BUGFIX: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): fix the alerting rule `ScrapePoolHasNoTargets`. Previously, it may cause false positive in [sharding mode](https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets).
+* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): include full list of query filters to build a cache key for [multitenant read](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels) queries. Previously, cache key did not include tenancy related labels so cache entries could be shared between tenants breaking tenant isolation. See [#9002](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9002) for details.

 ## [v1.118.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.118.0)

--- a/docs/victoriametrics/goals.md
+++ b/docs/victoriametrics/goals.md
@@ -12,8 +12,7 @@ aliases:
 ---
 ## Goals

-1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) resolving issues with VictoriaMetrics components,
-   so they could use these components in the most efficient way**.
+1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics and VictoriaLogs component in the most efficient way**.
 1. Fixing bugs in the essential functionality of VictoriaMetrics components. Small usability bugs are usually the most annoying,
   so they **must be fixed first**.
 1. Improving [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) for VictoriaMetrics components,
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
 module github.com/VictoriaMetrics/VictoriaMetrics

-go 1.24.3
+go 1.24.4

 // This is needed in order to avoid vmbackup and vmrestore binary size increase by 20MB
 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8008
Author	SHA1	Message	Date
Artem Fetishev	7f5a8af464	apptest: Add basic backup/restore integration test (#9133 ) This commit adds a basic backup/restore test for vmsingle and vmcluster. A more sophisticated was originally added to the partition index PR (#8134) and was aimed to test backup/restore when switching back and forth between legacy and partition index. During the code review it was decided that it would be good to have a separate test as well since legacy code will be removed in future and so will the test. Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>	2025-06-09 12:17:54 +02:00
Fred Navruzov	181a465c89	docs/vmanomaly: release v1.23.2 (#9135 ) ### Describe Your Changes Update docs to vmanomaly release v1.23.2 ### Checklist The following checks are mandatory: - [x] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).	2025-06-09 11:28:56 +02:00
Fred Navruzov	7288adab21	docs/vmanomaly: release v1.23.1 (#9132 ) ### Describe Your Changes Update the docs to release v1.23.1 ### Checklist The following checks are mandatory: - [x] My change adheres to [VictoriaMetrics contributing guidelines](https://docs.victoriametrics.com/victoriametrics/contributing/#pull-request-checklist).	2025-06-08 23:07:02 +02:00
Aliaksandr Valialkin	993a9d92d6	deployment/docker/builder/Dockerfile: download musl archives for cross-compilation from the local repository instead of musl.cc This speeds up building the Go builder image significantly (from hours to a few minutes), since the build speed was limited by the download speed from https://musl.cc , and this speed was extremely slow (e.g. 10kb/s and slower). This also improves build security, since the local mirror of musl.cc is under our control.	2025-06-08 13:47:13 +02:00
Aliaksandr Valialkin	45c889a1cf	docs/victorialogs/querying/README.md: mention that web UI supports live tailing This is a follow-up for the commit `231bfcf4cf` Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8882 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7046	2025-06-08 09:12:54 +02:00
Aliaksandr Valialkin	dca5d44f2b	deployment: update base Docker image from alpine:3.21.3 to alpine:3.22.0 See https://alpinelinux.org/posts/Alpine-3.22.0-released.html	2025-06-08 08:08:16 +02:00
Aliaksandr Valialkin	9e4f0cc900	go.mod: update Go from 1.24.3 to 1.24.4 See https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved This is a follow-up for `54dc9cc322`	2025-06-08 08:03:03 +02:00
Aliaksandr Valialkin	54dc9cc322	deployment: update Go builder from Go1.24.3 to Go1.24.4 See https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved	2025-06-08 08:01:45 +02:00
Aliaksandr Valialkin	67e6752b82	docs/victoriametrics/goals.md: clarify the main goal	2025-06-08 07:57:54 +02:00
Zakhar Bessarab	bb54075c23	docs/victoriametrics/changelog: cut v1.119.0 Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>	2025-06-06 16:48:02 +04:00
Zakhar Bessarab	08f5220bc3	app/{vmselect,vlselect}: run `make vmui-update vmui-logs-update` Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>	2025-06-06 16:38:04 +04:00
DmitrySafonov	f9015da6eb	app/promql/rollup_result_cache: include extra_filters in rollupCache key for multi-tenant support This PR addresses two issues: When tenant labels (e.g. vm_account_id, vm_project_id) are passed via extra_filters, they were not included in the rollupCache key. This could cause cache entries to be reused across different tenants, resulting in incorrect query results. If a tenant is specified only via extra_filters, and that tenant does not exist in TenantsCached, it gets silently filtered out by GetTenantTokensFromFilters, causing the query to fall back to a global (non-tenant) query — which is likely unexpected and potentially unsafe. This fix ensures correct tenant scoping and avoids unintended data exposure or cache pollution. Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9001 --------- Co-authored-by: Zakhar Bessarab <me@zekker.dev> Co-authored-by: Max Kotliar <kotlyar.maksim@gmail.com>	2025-06-06 15:56:36 +04:00