WIP

Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/vlinsert/insertutil/common_params.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
@@ -192,6 +193,7 @@ type logMessageProcessor struct {
 
 	rowsIngestedTotal  *metrics.Counter
 	bytesIngestedTotal *metrics.Counter
+	flushDuration      *metrics.Summary
 }
 
 func (lmp *logMessageProcessor) initPeriodicFlush() {
@@ -293,6 +295,7 @@ func (lmp *logMessageProcessor) flushLocked() {
 	lmp.lastFlushTime = time.Now()
 	logRowsStorage.MustAddRows(lmp.lr)
 	lmp.lr.ResetKeepSettings()
+	lmp.flushDuration.UpdateDuration(lmp.lastFlushTime)
 }
 
 // MustClose flushes the remaining data to the underlying storage and closes lmp.
@@ -303,6 +306,7 @@ func (lmp *logMessageProcessor) MustClose() {
 	lmp.flushLocked()
 	logstorage.PutLogRows(lmp.lr)
 	lmp.lr = nil
+	messageProcessorCount.Add(-1)
 }
 
 // NewLogMessageProcessor returns new LogMessageProcessor for the given cp.
@@ -312,12 +316,14 @@ func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode
 	lr := logstorage.GetLogRows(cp.StreamFields, cp.IgnoreFields, cp.DecolorizeFields, cp.ExtraFields, *defaultMsgValue)
 	rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
 	bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
+	flushDuration := metrics.GetOrCreateSummary(fmt.Sprintf("vl_insert_flush_duration_seconds{type=%q}", protocolName))
 	lmp := &logMessageProcessor{
 		cp: cp,
 		lr: lr,
 
 		rowsIngestedTotal:  rowsIngestedTotal,
 		bytesIngestedTotal: bytesIngestedTotal,
+		flushDuration:      flushDuration,
 
 		stopCh: make(chan struct{}),
 	}
@@ -326,10 +332,13 @@ func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode
 		lmp.initPeriodicFlush()
 	}
 
+	messageProcessorCount.Add(1)
 	return lmp
 }
 
 var (
 	rowsDroppedTotalDebug         = metrics.NewCounter(`vl_rows_dropped_total{reason="debug"}`)
 	rowsDroppedTotalTooManyFields = metrics.NewCounter(`vl_rows_dropped_total{reason="too_many_fields"}`)
+	_                             = metrics.NewGauge(`vl_insert_processors_count`, func() float64 { return float64(messageProcessorCount.Load()) })
+	messageProcessorCount         atomic.Int64
 )

app/vlstorage/main.go

@@ -78,7 +78,10 @@ var localStorage *logstorage.Storage
 var localStorageMetrics *metrics.Set
 
 var netstorageInsert *netinsert.Storage
+var netstorageInsertMetrics *metrics.Set
+
 var netstorageSelect *netselect.Storage
+var netstorageSelectMetrics *metrics.Set
 
 // Init initializes vlstorage.
 //
@@ -141,9 +144,19 @@ func initNetworkStorage() {
 
 	logger.Infof("starting insert service for nodes %s", *storageNodeAddrs)
 	netstorageInsert = netinsert.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *insertConcurrency, *insertDisableCompression)
+	netstorageInsertMetrics = metrics.NewSet()
+	netstorageInsertMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		netstorageInsert.WriteMetrics(w)
+	})
+	metrics.RegisterSet(netstorageInsertMetrics)
 
 	logger.Infof("initializing select service for nodes %s", *storageNodeAddrs)
 	netstorageSelect = netselect.NewStorage(*storageNodeAddrs, authCfgs, isTLSs, *selectDisableCompression)
+	netstorageSelectMetrics = metrics.NewSet()
+	netstorageSelectMetrics.RegisterMetricsWriter(func(w io.Writer) {
+		netstorageSelect.WriteMetrics(w)
+	})
+	metrics.RegisterSet(netstorageSelectMetrics)
 
 	logger.Infof("initialized all the network services")
 }
@@ -354,6 +367,10 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	var ss logstorage.StorageStats
 	strg.UpdateStats(&ss)
 
+	if maxDiskSpaceUsageBytes.N > 0 {
+		metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_max_disk_space_usage_bytes{path=%q}`, *storageDataPath), uint64(maxDiskSpaceUsageBytes.N))
+	}
+
 	metrics.WriteGaugeUint64(w, fmt.Sprintf(`vl_free_disk_space_bytes{path=%q}`, *storageDataPath), fs.MustGetFreeSpace(*storageDataPath))
 
 	isReadOnly := uint64(0)
@@ -365,10 +382,20 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/inmemory"}`, ss.InmemoryActiveMerges)
 	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/small"}`, ss.SmallPartActiveMerges)
 	metrics.WriteGaugeUint64(w, `vl_active_merges{type="storage/big"}`, ss.BigPartActiveMerges)
+	metrics.WriteGaugeUint64(w, `vl_active_merges{type="indexdb/inmemory"}`, ss.IndexdbActiveInmemoryMerges)
+	metrics.WriteGaugeUint64(w, `vl_active_merges{type="indexdb/file"}`, ss.IndexdbActiveFileMerges)
 
 	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/inmemory"}`, ss.InmemoryMergesTotal)
 	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/small"}`, ss.SmallPartMergesTotal)
 	metrics.WriteCounterUint64(w, `vl_merges_total{type="storage/big"}`, ss.BigPartMergesTotal)
+	metrics.WriteCounterUint64(w, `vl_merges_total{type="indexdb/inmemory"}`, ss.IndexdbInmemoryMergesTotal)
+	metrics.WriteCounterUint64(w, `vl_merges_total{type="indexdb/file"}`, ss.IndexdbFileMergesTotal)
+
+	metrics.WriteCounterUint64(w, `vl_rows_merged_total{type="storage/inmemory"}`, ss.InmemoryMergeRowsTotal)
+	metrics.WriteCounterUint64(w, `vl_rows_merged_total{type="storage/small"}`, ss.SmallPartMergeRowsTotal)
+	metrics.WriteCounterUint64(w, `vl_rows_merged_total{type="storage/big"}`, ss.BigPartMergeRowsTotal)
+	metrics.WriteCounterUint64(w, `vl_rows_merged_total{type="indexdb/inmemory"}`, ss.IndexdbInmemoryItemsMerged)
+	metrics.WriteCounterUint64(w, `vl_rows_merged_total{type="indexdb/file"}`, ss.IndexdbFileItemsMerged)
 
 	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/inmemory"}`, ss.InmemoryRowsCount)
 	metrics.WriteGaugeUint64(w, `vl_storage_rows{type="storage/small"}`, ss.SmallPartRowsCount)
@@ -382,6 +409,9 @@ func writeStorageMetrics(w io.Writer, strg *logstorage.Storage) {
 	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/small"}`, ss.SmallPartBlocks)
 	metrics.WriteGaugeUint64(w, `vl_storage_blocks{type="storage/big"}`, ss.BigPartBlocks)
 
+	metrics.WriteGaugeUint64(w, `vl_pending_rows{type="storage"}`, ss.PendingRowsCount)
+	metrics.WriteGaugeUint64(w, `vl_pending_rows{type="indexdb"}`, ss.IndexdbPendingItems)
+
 	metrics.WriteGaugeUint64(w, `vl_partitions`, ss.PartitionsCount)
 	metrics.WriteCounterUint64(w, `vl_streams_created_total`, ss.StreamsCreatedTotal)
 

app/vlstorage/netinsert/netinsert.go

@@ -21,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 // the maximum size of a single data block sent to storage node.
@@ -33,6 +34,8 @@ const ProtocolVersion = "v1"
 
 // Storage is a network storage for sending data to remote storage nodes in the cluster.
 type Storage struct {
+	RemoteSendFailed atomic.Uint64 // Number of failed requests sent to remote storage nodes.
+
 	sns []*storageNode
 
 	disableCompression bool
@@ -45,6 +48,16 @@ type Storage struct {
 	wg     sync.WaitGroup
 }
 
+func (s *Storage) WriteMetrics(w io.Writer) {
+	activeStreams := s.GetActiveStreams()
+	if activeStreams == 0 {
+		return
+	}
+
+	metrics.WriteGaugeUint64(w, `vl_insert_active_streams`, activeStreams)
+	metrics.WriteCounterUint64(w, `vl_insert_remote_send_errors_total`, s.RemoteSendFailed.Load())
+}
+
 type storageNode struct {
 	// scheme is http or https scheme to communicate with addr
 	scheme string
@@ -235,9 +248,7 @@ func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) erro
 
 	resp, err := sn.c.Do(req)
 	if err != nil {
-		// Disable sn for data writing for 10 seconds.
-		sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
-
+		sn.setDisableTemporarily()
 		return fmt.Errorf("cannot send data block with the length %d to %q: %s", pendingData.Len(), reqURL, err)
 	}
 	defer resp.Body.Close()
@@ -251,8 +262,7 @@ func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) erro
 		respBody = []byte(fmt.Sprintf("%s", err))
 	}
 
-	// Disable sn for data writing for 10 seconds.
-	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
+	sn.setDisableTemporarily()
 
 	return fmt.Errorf("unexpected status code returned when sending data block to %q: %d; want 2xx; response body: %q", reqURL, resp.StatusCode, respBody)
 }
@@ -261,6 +271,11 @@ func (sn *storageNode) getRequestURL(path string) string {
 	return fmt.Sprintf("%s://%s%s?version=%s", sn.scheme, sn.addr, path, url.QueryEscape(ProtocolVersion))
 }
 
+func (sn *storageNode) setDisableTemporarily() {
+	sn.disabledUntil.Store(fasttime.UnixTimestamp() + 10)
+	sn.s.RemoteSendFailed.Add(1)
+}
+
 var zstdBufPool bytesutil.ByteBufferPool
 
 // NewStorage returns new Storage for the given addrs with the given authCfgs.
@@ -307,6 +322,15 @@ func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
 	sn.addRow(r)
 }
 
+// GetActiveStreams returns the number of log streams being tracked since the Storage start.
+func (s *Storage) GetActiveStreams() uint64 {
+	s.srt.mu.Lock()
+	n := uint64(len(s.srt.rowsPerStream))
+	s.srt.mu.Unlock()
+
+	return n
+}
+
 func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
 	startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
 	for i := range s.sns {

app/vlstorage/netselect/netselect.go

@@ -10,6 +10,7 @@ import (
 	"net/url"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -21,6 +22,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 const (
@@ -65,6 +67,9 @@ type Storage struct {
 	sns []*storageNode
 
 	disableCompression bool
+
+	// remoteSendErrors is the number of errors when sending request to the remote storage node
+	remoteSendErrors atomic.Uint64
 }
 
 type storageNode struct {
@@ -313,6 +318,10 @@ func (s *Storage) MustStop() {
 	s.sns = nil
 }
 
+func (s *Storage) WriteMetrics(w io.Writer) {
+	metrics.WriteGaugeUint64(w, `vl_select_remote_send_errors_total`, s.remoteSendErrors.Load())
+}
+
 // RunQuery runs the given q and calls writeBlock for the returned data blocks
 func (s *Storage) RunQuery(ctx context.Context, tenantIDs []logstorage.TenantID, q *logstorage.Query, writeBlock logstorage.WriteDataBlockFunc) error {
 	nqr, err := logstorage.NewNetQueryRunner(ctx, tenantIDs, q, s.RunQuery, writeBlock)
@@ -345,6 +354,9 @@ func (s *Storage) runQuery(stopCh <-chan struct{}, tenantIDs []logstorage.Tenant
 			})
 			if err != nil {
 				// Cancel the remaining parallel queries
+				if !errors.Is(err, context.Canceled) {
+					s.remoteSendErrors.Add(1)
+				}
 				cancel()
 			}
 

app/vmalert/config/types.go

@@ -89,15 +89,18 @@ func (t *Type) ValidateExpr(expr string) error {
 	return nil
 }
 
+// SupportedType is true if given datasource type is supported
+func SupportedType(dsType string) bool {
+	return dsType == "graphite" || dsType == "prometheus" || dsType == "vlogs"
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (t *Type) UnmarshalYAML(unmarshal func(any) error) error {
 	var s string
 	if err := unmarshal(&s); err != nil {
 		return err
 	}
-	switch s {
-	case "graphite", "prometheus", "vlogs":
-	default:
+	if !SupportedType(s) {
 		return fmt.Errorf("unknown datasource type=%q, want prometheus, graphite or vlogs", s)
 	}
 	t.Name = s

app/vmalert/web.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/tpl"
@@ -27,6 +28,7 @@ var (
 		// such as Grafana, and proxied via vmselect.
 		{"api/v1/rules", "list all loaded groups and rules"},
 		{"api/v1/alerts", "list all active alerts"},
+		{"api/v1/notifiers", "list all notifiers"},
 		{fmt.Sprintf("api/v1/alert?%s=<int>&%s=<int>", paramGroupID, paramAlertID), "get alert status by group and alert ID"},
 	}
 	systemLinks = [][2]string{
@@ -42,6 +44,10 @@ var (
 		{Name: "Notifiers", URL: "notifiers"},
 		{Name: "Docs", URL: "https://docs.victoriametrics.com/victoriametrics/vmalert/"},
 	}
+	ruleTypeMap = map[string]string{
+		"alert":  ruleTypeAlerting,
+		"record": ruleTypeRecording,
+	}
 )
 
 type requestHandler struct {
@@ -89,10 +95,13 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 		WriteRuleDetails(w, r, rule)
 		return true
 	case "/vmalert/groups":
-		filter := r.URL.Query().Get("filter")
-		rf := extractRulesFilter(r, filter)
+		rf, err := newRulesFilter(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		data := rh.groups(rf)
-		WriteListGroups(w, r, data, filter)
+		WriteListGroups(w, r, data, rf.filter)
 		return true
 	case "/vmalert/notifiers":
 		WriteListTargets(w, r, notifier.GetTargets())
@@ -102,23 +111,35 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 	// served without `vmalert` prefix:
 	case "/rules":
 		// Grafana makes an extra request to `/rules`
-		// handler in addition to `/api/v1/rules` calls in alerts UI,
-		var data []apiGroup
-		filter := r.URL.Query().Get("filter")
-		rf := extractRulesFilter(r, filter)
+		// handler in addition to `/api/v1/rules` calls in alerts UI
+		var data []*apiGroup
+		rf, err := newRulesFilter(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		data = rh.groups(rf)
-		WriteListGroups(w, r, data, filter)
+		WriteListGroups(w, r, data, rf.filter)
 		return true
 
+	case "/vmalert/api/v1/notifiers", "/api/v1/notifiers":
+		data, err := rh.listNotifiers()
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(data)
+		return true
 	case "/vmalert/api/v1/rules", "/api/v1/rules":
 		// path used by Grafana for ng alerting
 		var data []byte
-		var err error
-
-		filter := r.URL.Query().Get("filter")
-		rf := extractRulesFilter(r, filter)
+		rf, err := newRulesFilter(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
 		data, err = rh.listGroups(rf)
-
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -129,7 +150,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
 
 	case "/vmalert/api/v1/alerts", "/api/v1/alerts":
 		// path used by Grafana for ng alerting
-		data, err := rh.listAlerts()
+		rf, err := newRulesFilter(r)
+		if err != nil {
+			httpserver.Errorf(w, r, "%s", err)
+			return true
+		}
+		data, err := rh.listAlerts(rf)
 		if err != nil {
 			httpserver.Errorf(w, r, "%s", err)
 			return true
@@ -218,7 +244,7 @@ func (rh *requestHandler) getAlert(r *http.Request) (*apiAlert, error) {
 type listGroupsResponse struct {
 	Status string `json:"status"`
 	Data   struct {
-		Groups []apiGroup `json:"groups"`
+		Groups []*apiGroup `json:"groups"`
 	} `json:"data"`
 }
 
@@ -229,82 +255,102 @@ type rulesFilter struct {
 	ruleNames     []string
 	ruleType      string
 	excludeAlerts bool
-	onlyUnhealthy bool
-	onlyNoMatch   bool
+	filter        string
+	dsType        config.Type
 }
 
-func extractRulesFilter(r *http.Request, filter string) rulesFilter {
-	rf := rulesFilter{}
+func newRulesFilter(r *http.Request) (*rulesFilter, error) {
+	rf := &rulesFilter{}
+	query := r.URL.Query()
 
-	var ruleType string
-	ruleTypeParam := r.URL.Query().Get("type")
-	// for some reason, `type` in filter doesn't match `type` in response,
-	// so we use this matching here
-	if ruleTypeParam == "alert" {
-		ruleType = ruleTypeAlerting
-	} else if ruleTypeParam == "record" {
-		ruleType = ruleTypeRecording
+	ruleTypeParam := query.Get("type")
+	if len(ruleTypeParam) > 0 {
+		if ruleType, ok := ruleTypeMap[ruleTypeParam]; ok {
+			rf.ruleType = ruleType
+		} else {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
+		}
+	}
+
+	dsType := query.Get("datasource_type")
+	if len(dsType) > 0 {
+		if config.SupportedType(dsType) {
+			rf.dsType = config.NewRawType(dsType)
+		} else {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "datasource_type": not supported value %q`, dsType), http.StatusBadRequest)
+		}
+	}
+
+	filter := strings.ToLower(query.Get("filter"))
+	if len(filter) > 0 {
+		if filter == "nomatch" || filter == "unhealthy" {
+			rf.filter = filter
+		} else {
+			return nil, errResponse(fmt.Errorf(`invalid parameter "filter": not supported value %q`, filter), http.StatusBadRequest)
+		}
 	}
-	rf.ruleType = ruleType
 
 	rf.excludeAlerts = httputil.GetBool(r, "exclude_alerts")
 	rf.ruleNames = append([]string{}, r.Form["rule_name[]"]...)
 	rf.groupNames = append([]string{}, r.Form["rule_group[]"]...)
 	rf.files = append([]string{}, r.Form["file[]"]...)
-	switch filter {
-	case "unhealthy":
-		rf.onlyUnhealthy = true
-	case "noMatch":
-		rf.onlyNoMatch = true
-	}
-	return rf
+	return rf, nil
 }
 
-func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
+func (rf *rulesFilter) matchesGroup(group *rule.Group) bool {
+	if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
+		return false
+	}
+	if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
+		return false
+	}
+	if len(rf.dsType.Name) > 0 && rf.dsType.String() != group.Type.String() {
+		return false
+	}
+	return true
+}
+
+func (rh *requestHandler) groups(rf *rulesFilter) []*apiGroup {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
-	groups := make([]apiGroup, 0)
+	groups := make([]*apiGroup, 0)
 	for _, group := range rh.m.groups {
-		if len(rf.groupNames) > 0 && !slices.Contains(rf.groupNames, group.Name) {
+		if !rf.matchesGroup(group) {
 			continue
 		}
-		if len(rf.files) > 0 && !slices.Contains(rf.files, group.File) {
-			continue
-		}
-
 		g := groupToAPI(group)
 		// the returned list should always be non-nil
 		// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4221
 		filteredRules := make([]apiRule, 0)
-		for _, r := range g.Rules {
-			if rf.ruleType != "" && rf.ruleType != r.Type {
+		for _, rule := range g.Rules {
+			if rf.ruleType != "" && rf.ruleType != rule.Type {
 				continue
 			}
-			if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
+			if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, rule.Name) {
+				continue
+			}
+			if (rule.LastError == "" && rf.filter == "unhealthy") || (!isNoMatch(rule) && rf.filter == "nomatch") {
 				continue
 			}
 			if rf.excludeAlerts {
-				r.Alerts = nil
+				rule.Alerts = nil
 			}
-			if (r.LastError == "" && rf.onlyUnhealthy) || (!isNoMatch(r) && rf.onlyNoMatch) {
-				continue
-			}
-			if r.LastError != "" {
+			if rule.LastError != "" {
 				g.Unhealthy++
 			} else {
 				g.Healthy++
 			}
-			if isNoMatch(r) {
+			if isNoMatch(rule) {
 				g.NoMatch++
 			}
-			filteredRules = append(filteredRules, r)
+			filteredRules = append(filteredRules, rule)
 		}
 		g.Rules = filteredRules
 		groups = append(groups, g)
 	}
 	// sort list of groups for deterministic output
-	slices.SortFunc(groups, func(a, b apiGroup) int {
+	slices.SortFunc(groups, func(a, b *apiGroup) int {
 		if a.Name != b.Name {
 			return strings.Compare(a.Name, b.Name)
 		}
@@ -313,7 +359,7 @@ func (rh *requestHandler) groups(rf rulesFilter) []apiGroup {
 	return groups
 }
 
-func (rh *requestHandler) listGroups(rf rulesFilter) ([]byte, error) {
+func (rh *requestHandler) listGroups(rf *rulesFilter) ([]byte, error) {
 	lr := listGroupsResponse{Status: "success"}
 	lr.Data.Groups = rh.groups(rf)
 	b, err := json.Marshal(lr)
@@ -360,14 +406,17 @@ func (rh *requestHandler) groupAlerts() []groupAlerts {
 	return gAlerts
 }
 
-func (rh *requestHandler) listAlerts() ([]byte, error) {
+func (rh *requestHandler) listAlerts(rf *rulesFilter) ([]byte, error) {
 	rh.m.groupsMu.RLock()
 	defer rh.m.groupsMu.RUnlock()
 
 	lr := listAlertsResponse{Status: "success"}
 	lr.Data.Alerts = make([]*apiAlert, 0)
-	for _, g := range rh.m.groups {
-		for _, r := range g.Rules {
+	for _, group := range rh.m.groups {
+		if !rf.matchesGroup(group) {
+			continue
+		}
+		for _, r := range group.Rules {
 			a, ok := r.(*rule.AlertingRule)
 			if !ok {
 				continue
@@ -391,6 +440,42 @@ func (rh *requestHandler) listAlerts() ([]byte, error) {
 	return b, nil
 }
 
+type listNotifiersResponse struct {
+	Status string `json:"status"`
+	Data   struct {
+		Notifiers []*apiNotifier `json:"notifiers"`
+	} `json:"data"`
+}
+
+func (rh *requestHandler) listNotifiers() ([]byte, error) {
+	targets := notifier.GetTargets()
+
+	lr := listNotifiersResponse{Status: "success"}
+	lr.Data.Notifiers = make([]*apiNotifier, 0)
+	for protoName, protoTargets := range targets {
+		notifier := &apiNotifier{
+			Kind:    string(protoName),
+			Targets: make([]*apiTarget, 0, len(protoTargets)),
+		}
+		for _, target := range protoTargets {
+			notifier.Targets = append(notifier.Targets, &apiTarget{
+				Address: target.Notifier.Addr(),
+				Labels:  target.Labels.ToMap(),
+			})
+		}
+		lr.Data.Notifiers = append(lr.Data.Notifiers, notifier)
+	}
+
+	b, err := json.Marshal(lr)
+	if err != nil {
+		return nil, &httpserver.ErrorWithStatusCode{
+			Err:        fmt.Errorf(`error encoding list of notifiers: %w`, err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	return b, nil
+}
+
 func errResponse(err error, sc int) *httpserver.ErrorWithStatusCode {
 	return &httpserver.ErrorWithStatusCode{
 		Err:        err,

app/vmalert/web.qtpl

@@ -93,18 +93,18 @@
     {%= tpl.Footer(r) %}
 {% endfunc %}
 
-{% func ListGroups(r *http.Request, groups []apiGroup, filter string) %}
+{% func ListGroups(r *http.Request, groups []*apiGroup, filter string) %}
     {%code
         prefix := vmalertutil.Prefix(r.URL.Path)
         filters := map[string]string{
             "":          "All",
             "unhealthy": "Unhealthy",
-            "noMatch":   "No Match",
+            "nomatch":   "No Match",
         }
         icons := map[string]string{
             "":          "all",
             "unhealthy": "unhealthy",
-            "noMatch":   "nomatch",
+            "nomatch":   "nomatch",
         }
         currentText := filters[filter]
         currentIcon := icons[filter]

app/vmalert/web.qtpl.go

@@ -316,7 +316,7 @@ func Welcome(r *http.Request) string {
 }
 
 //line app/vmalert/web.qtpl:96
-func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGroup, filter string) {
+func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []*apiGroup, filter string) {
 //line app/vmalert/web.qtpl:96
 	qw422016.N().S(`
     `)
@@ -325,12 +325,12 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGr
 	filters := map[string]string{
 		"":          "All",
 		"unhealthy": "Unhealthy",
-		"noMatch":   "No Match",
+		"nomatch":   "No Match",
 	}
 	icons := map[string]string{
 		"":          "all",
 		"unhealthy": "unhealthy",
-		"noMatch":   "nomatch",
+		"nomatch":   "nomatch",
 	}
 	currentText := filters[filter]
 	currentIcon := icons[filter]
@@ -722,7 +722,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGr
 }
 
 //line app/vmalert/web.qtpl:222
-func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []apiGroup, filter string) {
+func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []*apiGroup, filter string) {
 //line app/vmalert/web.qtpl:222
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line app/vmalert/web.qtpl:222
@@ -733,7 +733,7 @@ func WriteListGroups(qq422016 qtio422016.Writer, r *http.Request, groups []apiGr
 }
 
 //line app/vmalert/web.qtpl:222
-func ListGroups(r *http.Request, groups []apiGroup, filter string) string {
+func ListGroups(r *http.Request, groups []*apiGroup, filter string) string {
 //line app/vmalert/web.qtpl:222
 	qb422016 := qt422016.AcquireByteBuffer()
 //line app/vmalert/web.qtpl:222

app/vmalert/web_test.go

@@ -19,25 +19,34 @@ import (
 func TestHandler(t *testing.T) {
 	fq := &datasource.FakeQuerier{}
 	fq.Add(datasource.Metric{
-		Values: []float64{1}, Timestamps: []int64{0},
+		Values:     []float64{1},
+		Timestamps: []int64{0},
 	})
-	g := rule.NewGroup(config.Group{
-		Name:        "group",
-		File:        "rules.yaml",
-		Concurrency: 1,
-		Rules: []config.Rule{
-			{ID: 0, Alert: "alert"},
-			{ID: 1, Record: "record"},
-		},
-	}, fq, 1*time.Minute, nil)
-	ar := g.Rules[0].(*rule.AlertingRule)
-	rr := g.Rules[1].(*rule.RecordingRule)
-
-	g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
-
-	m := &manager{groups: map[uint64]*rule.Group{
-		g.CreateID(): g,
-	}}
+	m := &manager{groups: map[uint64]*rule.Group{}}
+	var ar *rule.AlertingRule
+	var rr *rule.RecordingRule
+	for _, dsType := range []string{"prometheus", "", "graphite"} {
+		g := rule.NewGroup(config.Group{
+			Name:        "group",
+			File:        "rules.yaml",
+			Type:        config.NewRawType(dsType),
+			Concurrency: 1,
+			Rules: []config.Rule{
+				{
+					ID:    0,
+					Alert: "alert",
+				},
+				{
+					ID:     1,
+					Record: "record",
+				},
+			},
+		}, fq, 1*time.Minute, nil)
+		ar = g.Rules[0].(*rule.AlertingRule)
+		rr = g.Rules[1].(*rule.RecordingRule)
+		g.ExecOnce(context.Background(), func() []notifier.Notifier { return nil }, nil, time.Time{})
+		m.groups[g.CreateID()] = g
+	}
 	rh := &requestHandler{m: m}
 
 	getResp := func(t *testing.T, url string, to any, code int) {
@@ -54,7 +63,7 @@ func TestHandler(t *testing.T) {
 				t.Fatalf("err closing body %s", err)
 			}
 		}()
-		if to != nil {
+		if to != nil && code < 300 {
 			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
 				t.Fatalf("unexpected err %s", err)
 			}
@@ -95,14 +104,23 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/alerts", func(t *testing.T) {
 		lr := listAlertsResponse{}
 		getResp(t, ts.URL+"/api/v1/alerts", &lr, 200)
-		if length := len(lr.Data.Alerts); length != 1 {
-			t.Fatalf("expected 1 alert got %d", length)
+		if length := len(lr.Data.Alerts); length != 3 {
+			t.Fatalf("expected 3 alert got %d", length)
 		}
 
 		lr = listAlertsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/alerts", &lr, 200)
-		if length := len(lr.Data.Alerts); length != 1 {
-			t.Fatalf("expected 1 alert got %d", length)
+		if length := len(lr.Data.Alerts); length != 3 {
+			t.Fatalf("expected 3 alert got %d", length)
+		}
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+"/api/v1/alerts?datasource_type=test", &lr, 400)
+
+		lr = listAlertsResponse{}
+		getResp(t, ts.URL+"/api/v1/alerts?datasource_type=prometheus", &lr, 200)
+		if length := len(lr.Data.Alerts); length != 2 {
+			t.Fatalf("expected 2 alert got %d", length)
 		}
 	})
 	t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
@@ -138,14 +156,14 @@ func TestHandler(t *testing.T) {
 	t.Run("/api/v1/rules", func(t *testing.T) {
 		lr := listGroupsResponse{}
 		getResp(t, ts.URL+"/api/v1/rules", &lr, 200)
-		if length := len(lr.Data.Groups); length != 1 {
-			t.Fatalf("expected 1 group got %d", length)
+		if length := len(lr.Data.Groups); length != 3 {
+			t.Fatalf("expected 3 group got %d", length)
 		}
 
 		lr = listGroupsResponse{}
 		getResp(t, ts.URL+"/vmalert/api/v1/rules", &lr, 200)
-		if length := len(lr.Data.Groups); length != 1 {
-			t.Fatalf("expected 1 group got %d", length)
+		if length := len(lr.Data.Groups); length != 3 {
+			t.Fatalf("expected 3 group got %d", length)
 		}
 	})
 	t.Run("/api/v1/rule?ruleID&groupID", func(t *testing.T) {
@@ -172,10 +190,10 @@ func TestHandler(t *testing.T) {
 	})
 
 	t.Run("/api/v1/rules&filters", func(t *testing.T) {
-		check := func(url string, expGroups, expRules int) {
+		check := func(url string, statusCode, expGroups, expRules int) {
 			t.Helper()
 			lr := listGroupsResponse{}
-			getResp(t, ts.URL+url, &lr, 200)
+			getResp(t, ts.URL+url, &lr, statusCode)
 			if length := len(lr.Data.Groups); length != expGroups {
 				t.Fatalf("expected %d groups got %d", expGroups, length)
 			}
@@ -191,25 +209,31 @@ func TestHandler(t *testing.T) {
 			}
 		}
 
-		check("/api/v1/rules?type=alert", 1, 1)
-		check("/api/v1/rules?type=record", 1, 1)
+		check("/api/v1/rules?type=alert", 200, 3, 3)
+		check("/api/v1/rules?type=record", 200, 3, 3)
+		check("/api/v1/rules?type=records", 400, 0, 0)
 
-		check("/vmalert/api/v1/rules?type=alert", 1, 1)
-		check("/vmalert/api/v1/rules?type=record", 1, 1)
+		check("/vmalert/api/v1/rules?type=alert", 200, 3, 3)
+		check("/vmalert/api/v1/rules?type=record", 200, 3, 3)
+		check("/vmalert/api/v1/rules?type=recording", 400, 0, 0)
+
+		check("/vmalert/api/v1/rules?datasource_type=prometheus", 200, 2, 4)
+		check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
+		check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
 
 		// no filtering expected due to bad params
-		check("/api/v1/rules?type=badParam", 1, 2)
-		check("/api/v1/rules?foo=bar", 1, 2)
+		check("/api/v1/rules?type=badParam", 400, 0, 0)
+		check("/api/v1/rules?foo=bar", 200, 3, 6)
 
-		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 0, 0)
-		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 1, 2)
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=bar", 200, 0, 0)
+		check("/api/v1/rules?rule_group[]=foo&rule_group[]=group&rule_group[]=bar", 200, 3, 6)
 
-		check("/api/v1/rules?rule_group[]=group&file[]=foo", 0, 0)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 1, 2)
+		check("/api/v1/rules?rule_group[]=group&file[]=foo", 200, 0, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml", 200, 3, 6)
 
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 1, 0)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 1, 1)
-		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 1, 2)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=foo", 200, 3, 0)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert", 200, 3, 3)
+		check("/api/v1/rules?rule_group[]=group&file[]=rules.yaml&rule_name[]=alert&rule_name[]=record", 200, 3, 6)
 	})
 	t.Run("/api/v1/rules&exclude_alerts=true", func(t *testing.T) {
 		// check if response returns active alerts by default
@@ -259,7 +283,7 @@ func TestEmptyResponse(t *testing.T) {
 				t.Fatalf("err closing body %s", err)
 			}
 		}()
-		if to != nil {
+		if to != nil && code < 300 {
 			if err = json.NewDecoder(resp.Body).Decode(to); err != nil {
 				t.Fatalf("unexpected err %s", err)
 			}

app/vmalert/web_types.go

@@ -20,6 +20,16 @@ const (
 	paramRuleID = "rule_id"
 )
 
+type apiNotifier struct {
+	Kind    string       `json:"kind"`
+	Targets []*apiTarget `json:"targets"`
+}
+
+type apiTarget struct {
+	Address string            `json:"address"`
+	Labels  map[string]string `json:"labels"`
+}
+
 // apiAlert represents a notifier.AlertingRule state
 // for WEB view
 // https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
@@ -108,7 +118,7 @@ type apiGroup struct {
 
 // groupAlerts represents a group of alerts for WEB view
 type groupAlerts struct {
-	Group  apiGroup
+	Group  *apiGroup
 	Alerts []*apiAlert
 }
 
@@ -327,7 +337,7 @@ func newAlertAPI(ar *rule.AlertingRule, a *notifier.Alert) *apiAlert {
 	return aa
 }
 
-func groupToAPI(g *rule.Group) apiGroup {
+func groupToAPI(g *rule.Group) *apiGroup {
 	g = g.DeepCopy()
 	ag := apiGroup{
 		// encode as string to avoid rounding
@@ -353,7 +363,7 @@ func groupToAPI(g *rule.Group) apiGroup {
 	for _, r := range g.Rules {
 		ag.Rules = append(ag.Rules, ruleToAPI(r))
 	}
-	return ag
+	return &ag
 }
 
 func urlValuesToStrings(values url.Values) []string {

app/vmctl/flags.go

@@ -70,7 +70,7 @@ var (
 			Usage: "VictoriaMetrics address to perform import requests. \n" +
 				"Should be the same as --httpListenAddr value for single-node version or vminsert component. \n" +
 				"When importing into the clustered version do not forget to set additionally --vm-account-id flag. \n" +
-				"Please note, that `vmctl` performs initial readiness check for the given address by checking `/health` endpoint.",
+				"Please note, that vmctl performs initial readiness check for the given address by checking /health endpoint.",
 		},
 		&cli.StringFlag{
 			Name:    vmUser,
@@ -514,27 +514,27 @@ var (
 		},
 		&cli.StringFlag{
 			Name:  vmNativeSrcBearerToken,
-			Usage: "Optional bearer auth token to use for the corresponding `--vm-native-src-addr`",
+			Usage: "Optional bearer auth token to use for the corresponding --vm-native-src-addr",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeSrcCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to `--vm-native-src-addr`",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to --vm-native-src-addr",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeSrcKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to `--vm-native-src-addr`",
+			Usage: "Optional path to client-side TLS key to use when connecting to --vm-native-src-addr",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeSrcCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to `--vm-native-src-addr`. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to --vm-native-src-addr. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeSrcServerName,
-			Usage: "Optional TLS server name to use for connections to `--vm-native-src-addr`. By default, the server name from `--vm-native-src-addr` is used",
+			Usage: "Optional TLS server name to use for connections to --vm-native-src-addr. By default, the server name from --vm-native-src-addr is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmNativeSrcInsecureSkipVerify,
-			Usage: "Whether to skip TLS certificate verification when connecting to `--vm-native-src-addr`",
+			Usage: "Whether to skip TLS certificate verification when connecting to --vm-native-src-addr",
 			Value: false,
 		},
 
@@ -563,27 +563,27 @@ var (
 		},
 		&cli.StringFlag{
 			Name:  vmNativeDstBearerToken,
-			Usage: "Optional bearer auth token to use for the corresponding `--vm-native-dst-addr`",
+			Usage: "Optional bearer auth token to use for the corresponding --vm-native-dst-addr",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeDstCertFile,
-			Usage: "Optional path to client-side TLS certificate file to use when connecting to `--vm-native-dst-addr`",
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to --vm-native-dst-addr",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeDstKeyFile,
-			Usage: "Optional path to client-side TLS key to use when connecting to `--vm-native-dst-addr`",
+			Usage: "Optional path to client-side TLS key to use when connecting to --vm-native-dst-addr",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeDstCAFile,
-			Usage: "Optional path to TLS CA file to use for verifying connections to `--vm-native-dst-addr`. By default, system CA is used",
+			Usage: "Optional path to TLS CA file to use for verifying connections to --vm-native-dst-addr. By default, system CA is used",
 		},
 		&cli.StringFlag{
 			Name:  vmNativeDstServerName,
-			Usage: "Optional TLS server name to use for connections to `--vm-native-dst-addr`. By default, the server name from `--vm-native-dst-addr` is used",
+			Usage: "Optional TLS server name to use for connections to --vm-native-dst-addr. By default, the server name from --vm-native-dst-addr is used",
 		},
 		&cli.BoolFlag{
 			Name:  vmNativeDstInsecureSkipVerify,
-			Usage: "Whether to skip TLS certificate verification when connecting to `--vm-native-dst-addr`",
+			Usage: "Whether to skip TLS certificate verification when connecting to --vm-native-dst-addr",
 			Value: false,
 		},
 
@@ -597,7 +597,7 @@ var (
 			Name: vmRateLimit,
 			Usage: "Optional data transfer rate limit in bytes per second.\n" +
 				"By default, the rate limit is disabled. It can be useful for limiting load on source or destination databases. \n" +
-				"Rate limit is applied per worker, see `--vm-concurrency`.",
+				"Rate limit is applied per worker, see --vm-concurrency.",
 		},
 		&cli.BoolFlag{
 			Name: vmInterCluster,

deployment/docker/compose-vl-cluster.yml

@@ -1,7 +1,7 @@
 services:
   # Grafana instance configured with VictoriaLogs as datasource
   grafana:
-    image: grafana/grafana:11.5.0
+    image: grafana/grafana:12.0.2
     depends_on:
       - "victoriametrics"
       - "vmauth"
@@ -68,7 +68,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # scraping, storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     volumes:
       - vmdata:/storage
       - ./prometheus-vl-cluster.yml:/etc/prometheus/prometheus.yml
@@ -81,7 +81,7 @@ services:
   # It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
   # depending on the requested path.
   vmauth:
-    image: victoriametrics/vmauth:v1.119.0
+    image: victoriametrics/vmauth:v1.120.0
     depends_on:
       - "victoriametrics"
       - "vlselect-1"
@@ -97,7 +97,7 @@ services:
 
   # vmalert executes alerting and recording rules according to given rule type.
   vmalert:
-    image: victoriametrics/vmalert:v1.119.0
+    image: victoriametrics/vmalert:v1.120.0
     depends_on:
       - "vmauth"
       - "alertmanager"

deployment/docker/compose-vl-single.yml

@@ -1,7 +1,7 @@
 services:
   # Grafana instance configured with VictoriaLogs as datasource
   grafana:
-    image: grafana/grafana:11.5.0
+    image: grafana/grafana:12.0.2
     depends_on:
       - "victoriametrics"
       - "victorialogs"
@@ -49,7 +49,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # scraping, storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     ports:
       - "8428:8428"
     volumes:
@@ -64,7 +64,7 @@ services:
   # It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
   # depending on the requested path.
   vmauth:
-    image: victoriametrics/vmauth:v1.119.0
+    image: victoriametrics/vmauth:v1.120.0
     depends_on:
       - "victoriametrics"
       - "victorialogs"
@@ -78,7 +78,7 @@ services:
 
   # vmalert executes alerting and recording rules according to the given rule type.
   vmalert:
-    image: victoriametrics/vmalert:v1.119.0
+    image: victoriametrics/vmalert:v1.120.0
     depends_on:
       - "vmauth"
       - "alertmanager"

deployment/docker/compose-vm-cluster.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.119.0
+    image: victoriametrics/vmagent:v1.120.0
     depends_on:
       - "vmauth"
     ports:
@@ -17,7 +17,7 @@ services:
     restart: always
 
   grafana:
-    image: grafana/grafana:11.5.0
+    image: grafana/grafana:12.0.2
     depends_on:
       - "vmauth"
     ports:
@@ -35,14 +35,14 @@ services:
   # vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
   # where N is number of vmstorages (2 in this case).
   vmstorage-1:
-    image: victoriametrics/vmstorage:v1.119.0-cluster
+    image: victoriametrics/vmstorage:v1.120.0-cluster
     volumes:
       - strgdata-1:/storage
     command:
       - "--storageDataPath=/storage"
     restart: always
   vmstorage-2:
-    image: victoriametrics/vmstorage:v1.119.0-cluster
+    image: victoriametrics/vmstorage:v1.120.0-cluster
     volumes:
       - strgdata-2:/storage
     command:
@@ -52,7 +52,7 @@ services:
   # vminsert is ingestion frontend. It receives metrics pushed by vmagent,
   # pre-process them and distributes across configured vmstorage shards.
   vminsert-1:
-    image: victoriametrics/vminsert:v1.119.0-cluster
+    image: victoriametrics/vminsert:v1.120.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -61,7 +61,7 @@ services:
       - "--storageNode=vmstorage-2:8400"
     restart: always
   vminsert-2:
-    image: victoriametrics/vminsert:v1.119.0-cluster
+    image: victoriametrics/vminsert:v1.120.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -73,7 +73,7 @@ services:
   # vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
   # vmselect collects results from configured `--storageNode` shards.
   vmselect-1:
-    image: victoriametrics/vmselect:v1.119.0-cluster
+    image: victoriametrics/vmselect:v1.120.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -83,7 +83,7 @@ services:
       - "--vmalert.proxyURL=http://vmalert:8880"
     restart: always
   vmselect-2:
-    image: victoriametrics/vmselect:v1.119.0-cluster
+    image: victoriametrics/vmselect:v1.120.0-cluster
     depends_on:
       - "vmstorage-1"
       - "vmstorage-2"
@@ -98,7 +98,7 @@ services:
   # read requests from Grafana, vmui, vmalert among vmselects.
   # It can be used as an authentication proxy.
   vmauth:
-    image: victoriametrics/vmauth:v1.119.0
+    image: victoriametrics/vmauth:v1.120.0
     depends_on:
       - "vmselect-1"
       - "vmselect-2"
@@ -112,7 +112,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.119.0
+    image: victoriametrics/vmalert:v1.120.0
     depends_on:
       - "vmauth"
     ports:

deployment/docker/compose-vm-single.yml

@@ -3,7 +3,7 @@ services:
   #  It scrapes targets defined in --promscrape.config
   #  And forward them to --remoteWrite.url
   vmagent:
-    image: victoriametrics/vmagent:v1.119.0
+    image: victoriametrics/vmagent:v1.120.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -18,7 +18,7 @@ services:
   # VictoriaMetrics instance, a single process responsible for
   # storing metrics and serve read requests.
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     ports:
       - 8428:8428
       - 8089:8089
@@ -38,7 +38,7 @@ services:
     restart: always
 
   grafana:
-    image: grafana/grafana:11.5.0
+    image: grafana/grafana:12.0.2
     depends_on:
       - "victoriametrics"
     ports:
@@ -54,7 +54,7 @@ services:
 
   # vmalert executes alerting and recording rules
   vmalert:
-    image: victoriametrics/vmalert:v1.119.0
+    image: victoriametrics/vmalert:v1.120.0
     depends_on:
       - "victoriametrics"
       - "alertmanager"

deployment/docker/victorialogs/compose-base.yml

@@ -19,7 +19,7 @@ services:
       retries: 10
 
   dd-proxy:
-    image: docker.io/victoriametrics/vmauth:v1.119.0
+    image: docker.io/victoriametrics/vmauth:v1.120.0
     restart: on-failure
     volumes:
       - ./:/etc/vmauth

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -1,6 +1,6 @@
 services:
   vmagent:
-    image: victoriametrics/vmagent:v1.119.0
+    image: victoriametrics/vmagent:v1.120.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -14,7 +14,7 @@ services:
     restart: always
 
   victoriametrics:
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     ports:
       - 8428:8428
     volumes:
@@ -27,7 +27,7 @@ services:
     restart: always
 
   grafana:
-    image: grafana/grafana-oss:10.2.1
+    image: grafana/grafana:12.0.2
     depends_on:
       - "victoriametrics"
     ports:
@@ -40,7 +40,7 @@ services:
     restart: always
 
   vmalert:
-    image: victoriametrics/vmalert:v1.119.0
+    image: victoriametrics/vmalert:v1.120.0
     depends_on:
       - "victoriametrics"
     ports:

deployment/logs-benchmark/docker-compose.yml

@@ -58,7 +58,7 @@ services:
       - ./vmsingle/promscrape.yml:/promscrape.yml
 
   grafana:
-    image: grafana/grafana:11.5.0
+    image: grafana/grafana:12.0.2
     depends_on: [vmsingle]
     ports:
       - 3000:3000

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -2,9 +2,9 @@
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.119.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.119.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.119.0)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.120.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.120.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.120.0)
 - [Grafana](https://grafana.com/) (v.10.2.1)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
 - [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
@@ -315,7 +315,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.119.0
+    image: victoriametrics/vmagent:v1.120.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -332,7 +332,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     ports:
       - 8428:8428
     volumes:
@@ -365,7 +365,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.119.0
+    image: victoriametrics/vmalert:v1.120.0
     depends_on:
       - "victoriametrics"
     ports:

docs/guides/grafana-vmgateway-openid-configuration/README.md

@@ -241,27 +241,27 @@ services:
       - grafana_data:/var/lib/grafana/
 
   vmsingle:
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     command:
       - -httpListenAddr=0.0.0.0:8429
 
   vmstorage:
-    image: victoriametrics/vmstorage:v1.119.0-cluster
+    image: victoriametrics/vmstorage:v1.120.0-cluster
 
   vminsert:
-    image: victoriametrics/vminsert:v1.119.0-cluster
+    image: victoriametrics/vminsert:v1.120.0-cluster
     command:
       - -storageNode=vmstorage:8400
       - -httpListenAddr=0.0.0.0:8480
 
   vmselect:
-    image: victoriametrics/vmselect:v1.119.0-cluster
+    image: victoriametrics/vmselect:v1.120.0-cluster
     command:
       - -storageNode=vmstorage:8401
       - -httpListenAddr=0.0.0.0:8481
 
   vmagent:
-    image: victoriametrics/vmagent:v1.119.0
+    image: victoriametrics/vmagent:v1.120.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
     command:
@@ -270,7 +270,7 @@ services:
       - -remoteWrite.url=http://vmsingle:8429/api/v1/write
 
   vmgateway-cluster:
-    image: victoriametrics/vmgateway:v1.119.0-enterprise
+    image: victoriametrics/vmgateway:v1.120.0-enterprise
     ports:
       - 8431:8431
     volumes:
@@ -286,7 +286,7 @@ services:
       - -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
 
   vmgateway-single:
-    image: victoriametrics/vmgateway:v1.119.0-enterprise
+    image: victoriametrics/vmgateway:v1.120.0-enterprise
     ports:
       - 8432:8431
     volumes:
@@ -397,7 +397,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
 
 ```yaml
   vmagent:
-    image: victoriametrics/vmagent:v1.119.0
+    image: victoriametrics/vmagent:v1.120.0
     volumes:
       - ./scrape.yaml:/etc/vmagent/config.yaml
       - ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret

docs/guides/migrate-from-influx/README.md

@@ -209,8 +209,8 @@ Migrating data from other databases to VictoriaMetrics is as simple as importing
 [supported ingestion formats](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model).
 
 But migration from InfluxDB might get easier with [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/). See more about
-migrating [from InfluxDB v1.x versions](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x).
-Migrating data from InfluxDB v2.x is not supported. But there is a useful [3rd party solution](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-2x) 
+migrating [from InfluxDB v1.x versions](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/).
+Migrating data from InfluxDB v2.x is not supported. But there is a useful [3rd party solution](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/#influxdb-v2) 
 for this.
 
 Please note, data migration is a backfilling process, so read about [backfilling tips](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#backfilling).

docs/victorialogs/CHANGELOG.md

@@ -570,7 +570,7 @@ Released at 2024-09-30
 Released at 2024-09-29
 
 * FEATURE: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): accept Unix timestamps in seconds in the ingested logs. This simplifies integration with systems, which prefer Unix timestamps over text-based representation of time.
-* FEATURE: [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe): allow using `order` alias instead of `sort`. For example, `_time:5s | order by (_time)` query works the same as `_time:5s | sort by (_time)`. This simplifies the to [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) transition from SQL-like query languages.
+* FEATURE: [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe): allow using `order` alias instead of `sort`. For example, `_time:5s | order by (_time)` query works the same as `_time:5s | sort by (_time)`. This simplifies [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) transition from SQL-like query languages.
 * FEATURE: [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe): allow using multiple identical [stats functions](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions) with distinct [filters](https://docs.victoriametrics.com/victorialogs/logsql/#stats-with-additional-filters) and automatically generated result names. For example, `_time:5m | count(), count() if (error)` query works as expected now, e.g. it returns two results over the last 5 minutes: the total number of logs and the number of logs with `error` [word](https://docs.victoriametrics.com/victorialogs/logsql/#word). Previously this query couldn't be executed because the `if (...)` condition wasn't included in the automatically generate result name, so both results had the same name - `count(*)`.
 
 * BUGFIX: properly calculate [`uniq`](https://docs.victoriametrics.com/victorialogs/logsql/#uniq-pipe) and [`top`](https://docs.victoriametrics.com/victorialogs/logsql/#top-pipe) pipes. Previously they could return invalid results in some cases.
@@ -650,7 +650,7 @@ Released at 2024-07-05
 
 Released at 2024-07-02
 
-* FEATURE: add `-syslog.useLocalTimestamp.tcp` and `-syslog.useLocalTimestamp.udp` command-line flags, which could be used for using the local timestamp as [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) for the logs ingested via the corresponding `-syslog.listenAddr.tcp` / `-syslog.listenAddr.udp`. By default the timestamp from the syslog message is used as [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field). See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/).
+* FEATURE: add `-syslog.useLocalTimestamp.tcp` and `-syslog.useLocalTimestamp.udp` command-line flags, which could be used for using the local timestamp as [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field) for the logs ingested via the corresponding `-syslog.listenAddr.tcp` / `-syslog.listenAddr.udp`. By default, the timestamp from the syslog message is used as [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field). See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/).
 
 * BUGFIX: make slowly ingested logs visible for search as soon as they are ingested into VictoriaLogs. Previously slowly ingested logs could remain invisible for search for long time.
 

docs/victorialogs/FAQ.md

@@ -207,7 +207,7 @@ via `-insert.maxLineSizeBytes` command-line flag.
 VictoriaLogs limits [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) name length to 128 bytes -
 Log entries with longer field names are ignored during [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/).
 
-The maximum length of a field name is hardcoded and is unikely to increase, since this may increase RAM and CPU usage.
+The maximum length of a field name is hardcoded and is unlikely to increase, since this may increase RAM and CPU usage.
 
 ## How many fields a single log entry may contain
 
@@ -243,7 +243,7 @@ is returned in the `total_bytes` field.
 
 If you use [VictoriaLogs web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui)
 or [Grafana plugin for VictoriaLogs](https://docs.victoriametrics.com/victorialogs/victorialogs-datasource/),
-then make sure the selected time range covers the last day. Otherwise the query above returns
+then make sure the selected time range covers the last day. Otherwise, the query above returns
 results on the intersection of the last day and the selected time range.
 
 See [why the log field occupies a lot of disk space](#why-the-log-field-occupies-a-lot-of-disk-space).
@@ -334,7 +334,7 @@ The query works in the following way:
 
 The needed storage space depends on the following factors:
 
-- Data compressibility. VictoraLogs compresses the ingested logs before storing them to disk. The compression ratio depends on the "randomness" of the ingested logs.
+- Data compressibility. VictoriaLogs compresses the ingested logs before storing them to disk. The compression ratio depends on the "randomness" of the ingested logs.
   Less "random" logs with many repeated field values and small differences between log messages compress the best (up to 100x and more).
   More "random" logs with many unique field values may have very low compression rate.
 

docs/victorialogs/LogsQL.md

@@ -1319,7 +1319,7 @@ This query matches the following log messages, since their length is in the requ
 This query doesn't match the following log messages:
 
 - `foo`, since it is too short
-- `foo bar baz abc`, sinc it is too long
+- `foo bar baz abc`, since it is too long
 
 It is possible to use `inf` as the upper bound. For example, the following query matches [log messages](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field)
 with the length bigger or equal to 5 chars:
@@ -3362,7 +3362,7 @@ It understands the following Syslog formats:
 
 The following fields are unpacked:
 
-- `level` - optained from `PRI`.
+- `level` - obtained from `PRI`.
 - `priority` - obtained from `PRI`.
 - `facility` - calculated as `PRI / 8`.
 - `facility_keyword` - string representation of the `facility` field according to [these docs](https://en.wikipedia.org/wiki/Syslog#Facility).
@@ -3497,7 +3497,7 @@ See also:
 
 #### Conditional unroll
 
-If the [`unroll` pipe](#unroll-pipe) must be applied only to some [log enties](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
+If the [`unroll` pipe](#unroll-pipe) must be applied only to some [log entries](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
 then add `if (<filters>)` after `unroll`.
 The `<filters>` can contain arbitrary [filters](#filters). For example, the following query unrolls `value` field only if `value_type` field equals to `json_array`:
 
@@ -4127,7 +4127,7 @@ LogsQL supports the following string literals:
 
 - `"double quoted"`. Double quote and backslash inside such a string must be escaped with `\`: `"escape\"doublequote and \\ backslash"`.
   Double-quoted strings may contain special sequences such as `\n`, `\t`, `\f`, `\x8c`, etc. They are decoded according to [these docs](https://go.dev/ref/spec#String_literals).
-- `'single quoted'`. Single quote and backsliash inside such a string must be escaped with `\`: `'escape\'singlequote and \\ backslash'`.
+- `'single quoted'`. Single quote and backslash inside such a string must be escaped with `\`: `'escape\'singlequote and \\ backslash'`.
 - ``` `backtick quoted` ```. Strings with backslashes, double quotes and single quotes shouldn't be escaped inside backtick-quoted strings.
 
 ## Comments

docs/victorialogs/cluster.md

@@ -31,41 +31,126 @@ See [quick start guide](#quick-start) on how to start working with VictoriaLogs
 
 ## Architecture
 
-VictoriaLogs in cluster mode consists of `vlinsert`, `vlselect` and `vlstorage` components:
+VictoriaLogs in cluster mode is composed of three main components: `vlinsert`, `vlselect`, and `vlstorage`.
 
-- `vlinsert` accepts the ingested logs via [all the supported data ingestion protocols](https://docs.victoriametrics.com/victorialogs/data-ingestion/)
-  and spreads them evenly among the `vlstorage` nodes listed via the `-storageNode` command-line flag.
+```mermaid
+sequenceDiagram
+    participant LS as Log Sources
+    participant VI as vlinsert
+    participant VS1 as vlstorage-1
+    participant VS2 as vlstorage-2
+    participant VL as vlselect
+    participant QC as Query Client
+    
+    Note over LS,VS2: Log Ingestion Flow
+    LS->>VI: Send logs via supported protocols
+    VI->>VS1: POST /internal/insert (HTTP)
+    VI->>VS2: POST /internal/insert (HTTP)
+    Note right of VI: Distributes logs evenly<br/>across vlstorage nodes
+    
+    Note over VS1,QC: Query Flow
+    QC->>VL: Query via HTTP endpoints
+    VL->>VS1: GET /internal/select/* (HTTP)
+    VL->>VS2: GET /internal/select/* (HTTP)
+    VS1-->>VL: Return local results
+    VS2-->>VL: Return local results
+    VL->>QC: Processed & aggregated results
+```
 
-- `vlselect` accepts incoming queries via [all the supported HTTP querying endpoints](https://docs.victoriametrics.com/victorialogs/querying/),
-  requests the needed data from `vlstorage` nodes listed via the `-storageNode` command-line flag, processes the queries and returns the corresponding responses.
+- `vlinsert` handles log ingestion via [all supported protocols](https://docs.victoriametrics.com/victorialogs/data-ingestion/).  
+  It distributes incoming logs evenly across `vlstorage` nodes, as specified by the `-storageNode` command-line flag.
 
-- `vlstorage` is responsible for two tasks:
+- `vlselect` receives queries through [all supported HTTP query endpoints](https://docs.victoriametrics.com/victorialogs/querying/).  
+  It fetches the required data from the configured `vlstorage` nodes, processes the queries, and returns the results.
 
-  - It accepts logs from `vlinsert` nodes and stores them at the directory specified via `-storageDataPath` command-line flag.
-    See [these docs](https://docs.victoriametrics.com/victorialogs/#storage) for details about this flag.
+- `vlstorage` performs two key roles:
+  - It stores logs received from `vlinsert` at the directory defined by the `-storageDataPath` flag.  
+    See [storage configuration docs](https://docs.victoriametrics.com/victorialogs/#storage) for details.
+  - It handles queries from `vlselect` by retrieving and transforming the requested data locally before returning results.
 
-  - It processes requests from `vlselect` nodes. It selects the requested logs and performs all data transformations and calculations,
-    which can be executed locally, before sending the results to `vlselect`.
+Each `vlstorage` node operates as a self-contained VictoriaLogs instance.  
+Refer to the [single-node and cluster mode duality](#single-node-and-cluster-mode-duality) documentation for more information.  
+This design allows you to reuse existing single-node VictoriaLogs instances by listing them in the `-storageNode` flag for `vlselect`, enabling unified querying across all nodes.
 
-`vlstorage` is basically a single-node version of VictoriaLogs. See [these docs](#single-node-and-cluster-mode-duality) for details.
-This means that the existing single-node VictoriaLogs instances can be added to the list of `vlstorage` nodes via `-storageNode` command-line flag at `vlselect`
-in order to get global querying view over all the logs across all the single-node VictoriaLogs instances.
+All VictoriaLogs components are horizontally scalable and can be deployed on hardware best suited to their respective workloads.  
+`vlinsert` and `vlselect` can be run on the same node, which allows the minimal cluster to consist of just one `vlstorage` node and one node acting as both `vlinsert` and `vlselect`.  
+However, for production environments, it is recommended to separate `vlinsert` and `vlselect` roles to avoid resource contention — for example, to prevent heavy queries from interfering with log ingestion.
 
-Every component of the VictoriaLogs cluster can scale from a single node to arbitrary number of nodes and can run on the most suitable hardware for the given workload.
-`vlinsert` nodes can be used as `vlselect` nodes, so the minimum VictoriaLogs cluster must contain a `vlstorage` node plus a node, which plays both `vlinsert` and `vlselect` roles.
-It isn't recommended sharing `vlinsert` and `vlselect` responsibilities in a single node, since this increases chances that heavy queries can negatively affect data ingestion
-and vice versa.
+Communication between `vlinsert` / `vlselect` and `vlstorage` is done via HTTP over the port specified by the `-httpListenAddr` flag:
 
-`vlselect` and `vlinsert` communicate with `vlstorage` via HTTP at the TCP port specified via `-httpListenAddr` command-line flag:
+- `vlinsert` sends data to the `/internal/insert` endpoint on `vlstorage`.
+- `vlselect` sends queries to endpoints under `/internal/select/` on `vlstorage`.
 
-- `vlinsert` sends requests to `/internal/insert` HTTP endpoint at `vlstorage`.
-- `vlselect` sends requests to HTTP endpoints at `vlstorage` starting with `/internal/select/`.
+This HTTP-based communication model allows you to use reverse proxies for authorization, routing, and encryption between components.  
+Use of [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) is recommended for managing access control.
 
-This allows using various http proxies for authorization, routing and encryption of requests between these components.
-It is recommended to use [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/).
+For advanced setups, refer to the [multi-level cluster setup](#multi-level-cluster-setup) documentation.
 
-See also [multi-level cluster setup](#multi-level-cluster-setup).
+## High availability
 
+In the cluster setup, the following rules apply:
+
+- The `vlselect` component requires **all relevant vlstorage nodes to be available** in order to return complete and correct query results. 
+
+  - If even one of the vlstorage nodes is temporarily unavailable, `vlselect` cannot safely return a full response, since some of the required data may reside on the missing node. Rather than risk delivering partial or misleading query results, which can cause confusion, trigger false alerts, or produce incorrect metrics, VictoriaLogs chooses to return an error instead. 
+
+- The `vlinsert` component continues to function normally when some vlstorage nodes are unavailable. It automatically routes new logs to the remaining available nodes to ensure that data ingestion remains uninterrupted and newly received logs are not lost.
+
+> [!NOTE] Insight  
+> In most real-world cases, `vlstorage` nodes become unavailable during planned maintenance such as upgrades, config changes, or rolling restarts. These are typically infrequent (weekly or monthly) and brief (a few minutes).  
+> A short period of query downtime during such events is acceptable and fits well within most SLAs. For example, 60 minutes of downtime per month still provides around 99.86% availability, which often outperforms complex HA setups that rely on opaque auto-recovery and may fail unpredictably.
+
+VictoriaLogs itself does not handle replication at the storage level. Instead, it relies on an external log shipper, such as [vector](https://docs.victoriametrics.com/victorialogs/data-ingestion/vector/) or [vlagent](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9034), to send the same log stream to multiple independent VictoriaLogs instances:
+
+```mermaid
+graph TD
+    subgraph "HA Solution"
+        subgraph "Ingestion Layer"
+            LS["Log Sources<br/>(Applications)"]
+            VECTOR["Log Collector<br/>• Buffering<br/>• Replication<br/>• Delivery Guarantees"]
+        end
+        
+        subgraph "Storage Layer"
+            subgraph "Zone A"
+                VLA["VictoriaLogs Cluster A"]
+            end
+            
+            subgraph "Zone B"
+                VLB["VictoriaLogs Cluster B"]
+            end
+        end
+        
+        subgraph "Query Layer"
+            LB["Load Balancer<br/>(vmauth)<br/>• Health Checks<br/>• Failover<br/>• Query Distribution"]
+            QC["Query Clients<br/>(Grafana, API)"]
+        end
+        
+        LS --> VECTOR
+        VECTOR -->|"Replicate logs to<br/>Zone A cluster"| VLA
+        VECTOR -->|"Replicate logs to<br/>Zone B cluster"| VLB
+        
+        VLA -->|"Serve queries from<br/>Zone A cluster"| LB
+        VLB -->|"Serve queries from<br/>Zone B cluster"| LB
+        LB --> QC
+        
+        style VECTOR fill:#e8f5e8
+        style VLA fill:#d5e8d4
+        style VLB fill:#d5e8d4
+        style LB fill:#e1f5fe
+        style QC fill:#fff2cc
+        style LS fill:#fff2cc
+    end
+```
+
+In this HA solution:
+
+- A log shipper at the top receives logs and replicates them in parallel to two VictoriaLogs clusters.
+  - If one cluster fails completely (i.e., **all** of its storage nodes become unavailable), the log shipper continues to send logs to the remaining healthy cluster and buffers any logs that cannot be delivered. When the failed cluster becomes available again, the log shipper resumes sending both buffered and new logs to it.
+- On the read path, a load balancer (e.g., vmauth) sits in front of the VictoriaLogs clusters and routes query requests to any healthy cluster.
+  - If one cluster fails (i.e., **at least one** of its storage nodes is unavailable), the load balancer detects this and automatically redirects all query traffic to the remaining healthy cluster.
+
+There's no hidden coordination logic or consensus algorithm. You can scale it horizontally and operate it safely, even in bare-metal Kubernetes clusters using local PVs, as long as the log shipper handles reliable replication and buffering.
+  
 ## Single-node and cluster mode duality
 
 Every `vlstorage` node can be used as a single-node VictoriaLogs instance:
@@ -189,7 +274,7 @@ Start `vlselect` node, which [accepts incoming queries](https://docs.victoriamet
 
 Note that all the VictoriaLogs cluster components - `vlstorage`, `vlinsert` and `vlselect` - share the same executable - `victoria-logs-prod`.
 Their roles depend on whether the `-storageNode` command-line flag is set - if this flag is set, then the executable runs in `vlinsert` and `vlselect` modes.
-Otherwise it runs in `vlstorage` mode, which is identical to a [single-node VictoriaLogs mode](https://docs.victoriametrics.com/victorialogs/).
+Otherwise, it runs in `vlstorage` mode, which is identical to a [single-node VictoriaLogs mode](https://docs.victoriametrics.com/victorialogs/).
 
 Let's ingest some logs (aka [wide events](https://jeremymorrell.dev/blog/a-practitioners-guide-to-wide-events/))
 from [GitHub archive](https://www.gharchive.org/) into the VictoriaLogs cluster with the following command:

docs/victorialogs/data-ingestion/Journald.md

@@ -28,12 +28,12 @@ See [the list of supported Journald fields](https://www.freedesktop.org/software
 
 ## Level field
 
-VictoriaLogs atomatically sets the `level` log field according to the [`PRIORITY` field falue](https://wiki.archlinux.org/title/Systemd/Journal).
+VictoriaLogs automatically sets the `level` log field according to the [`PRIORITY` field value](https://wiki.archlinux.org/title/Systemd/Journal).
 
 ## Stream fields
 
 VictoriaLogs uses `(_MACHINE_ID, _HOSTNAME, _SYSTEMD_UNIT)` as [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
-for logs ingested via jorunald protocol. The list of log stream fields can be changed via `-journald.streamFields` command-line flag if needed,
+for logs ingested via journald protocol. The list of log stream fields can be changed via `-journald.streamFields` command-line flag if needed,
 by providing comma-separated list of journald fields form [this list](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
 
 Please make sure that the log stream fields passed to `-journlad.streamFields` do not contain fields with high number or unbound number of unique values,

docs/victorialogs/logql-to-logsql.md

@@ -94,7 +94,7 @@ Loki allows applying filters to log labels with `{...} | label op value` syntax:
   according to [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#logical-filter).
 
 * `{...} | label > value`, `{...} label >= value`, `{...} label < value` and `{...} label <= value`.
-  This is eqvalent to `{...} label:>value`, `{...} label:>=value`, `{...} label:<value` and `{...} label:<=value`
+  This is equivalent to `{...} label:>value`, `{...} label:>=value`, `{...} label:<value` and `{...} label:<=value`
   in VictoriaLogs. See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#range-comparison-filter).
 
 * `{...} | label ~= value`. This is equivalent to `{...} label:~value` in VictoriaLogs.
@@ -192,7 +192,7 @@ Loki provides the ability to drop log labels with the `{...} | drop label1, ...,
 according to [these docs](https://grafana.com/docs/loki/latest/query/log_queries/#drop-labels-expression).
 The similar syntax is also supported by VictoriaLogs. See [these docs](https://docs.victoriametrics.com/victorialogs/logsql/#delete-pipe).
 
-Lokis supports conditional dropping of labels with the `{...} | drop label="value"` syntax.
+Loki supports conditional dropping of labels with the `{...} | drop label="value"` syntax.
 This can be replaced with [conditional format](https://docs.victoriametrics.com/victorialogs/logsql/#conditional-format) at VictoriaLogs:
 
 ```logsql
@@ -266,7 +266,7 @@ can be substituted with the simple `{...} | count()` query at VictoriaLogs.
 
 ### Unwrapped range aggregations
 
-Loki allows calculating metrics from label values by using the `func_name({...} | unwrap label_name)` syntax. There is no need in unrapping any labels in VictoriaLogs -
+Loki allows calculating metrics from label values by using the `func_name({...} | unwrap label_name)` syntax. There is no need in unwrapping any labels in VictoriaLogs -
 just pass the needed label names into the needed [`stats` pipe function](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe-functions).
 
 VictoriaLogs aggregates all the selected logs by default, while Loki groups stats by log stream. Use `... | stats by (_stream) ...`

docs/victorialogs/logsql-examples.md

@@ -128,7 +128,7 @@ See also:
 
 ## How to select logs with all the given words in log message?
 
-Just enumerate the needed [words](https://docs.victoriametrics.com/victorialogs/logsql/#word) in the query, by deliming them with whitespace.
+Just enumerate the needed [words](https://docs.victoriametrics.com/victorialogs/logsql/#word) in the query, by delimiting them with whitespace.
 For example, the following query selects logs containing both `error` and `kubernetes` [words](https://docs.victoriametrics.com/victorialogs/logsql/#word)
 in the [log message](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field):
 

docs/victorialogs/sql-to-logsql.md

@@ -56,7 +56,7 @@ FROM <table>
 ```
 
 The `<filters>` part selects the needed logs (rows) according to the provided [filters](https://docs.victoriametrics.com/victorialogs/logsql/#filters).
-Then the provided [pipes](https://docs.victoriametrics.com/victorialogs/logsql/#pipes) are executed sequentlially.
+Then the provided [pipes](https://docs.victoriametrics.com/victorialogs/logsql/#pipes) are executed sequentially.
 Every such pipe receives all the rows from the previous stage, performs some calculations and/or transformations,
 and then pushes the resulting rows to the next stage. This simplifies reading and understanding the query - just read it from the beginning
 to the end in order to understand what does it do at every stage.

docs/victoriametrics-cloud/integrations/grafana.md

@@ -3,6 +3,7 @@ title : "Grafana"
 menu:
   docs:
     parent: "integrations"
+    identifier: "cloud-integrations-grafana"
 ---
 
 [Grafana](https://grafana.com/) is a popular open-source visualization and dashboarding tool. You can

docs/victoriametrics/Cluster-VictoriaMetrics.md

@@ -34,7 +34,7 @@ Enterprise binaries can be downloaded and evaluated for free
 from [the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
 See how to request a free trial license [here](https://victoriametrics.com/products/enterprise/trial/).
 
-VictoriaMetrics is developed at a fast pace, so it is recommended periodically checking the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/) and performing [regular upgrades](#how-to-upgrade-victoriametrics).
+VictoriaMetrics is developed at a fast pace, so it is recommended periodically checking the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/) and performing [regular upgrades](https://docs.victoriametrics.com/victoriametrics/#how-to-upgrade-victoriametrics).
 
 VictoriaMetrics has achieved security certifications for Database Software Development and Software-Based Monitoring Services. We apply strict security measures in everything we do. See our [Security page](https://victoriametrics.com/security/) for more details.
 

docs/victoriametrics/FAQ.md

@@ -417,7 +417,7 @@ Cluster version of VictoriaMetrics may be preferred over single-node VictoriaMet
 
 Single-node VictoriaMetrics stores data on disk in slightly different format comparing to cluster version of VictoriaMetrics.
 So it is impossible to just copy the on-disk data from `-storageDataPath` directory from single-node VictoriaMetrics to a `vmstorage` node in VictoriaMetrics cluster.
-If you need migrating data from single-node VictoriaMetrics to cluster version, then [follow these instructions](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics).
+If you need migrating data from single-node VictoriaMetrics to cluster version, then [follow these instructions](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
 ## Why isn't MetricsQL 100% compatible with PromQL?
 
@@ -425,15 +425,15 @@ If you need migrating data from single-node VictoriaMetrics to cluster version,
 
 ## How to migrate data from Prometheus to VictoriaMetrics?
 
-Please see [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-prometheus).
+Please see [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/).
 
 ## How to migrate data from InfluxDB to VictoriaMetrics?
 
-Please see [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x).
+Please see [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/).
 
 ## How to migrate data from OpenTSDB to VictoriaMetrics?
 
-Please see [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-opentsdb).
+Please see [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/).
 
 ## How to migrate data from Graphite to VictoriaMetrics?
 

docs/victoriametrics/LTS-releases.md

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
 
 ## Currently supported LTS release lines
 
-- v1.110.x - the latest one is [v1.110.9 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.9)
-- v1.102.x - the latest one is [v1.102.22 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.22)
+- v1.110.x - the latest one is [v1.110.12 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.12)
+- v1.102.x - the latest one is [v1.102.24 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.24)

docs/victoriametrics/MetricsQL.md

@@ -746,7 +746,7 @@ See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rat
 
 #### rate_prometheus
 
-`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
+`rate_prometheus(series_selector[d])` {{% available_from "v1.120.0" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
 increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
 The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
 

docs/victoriametrics/Quick-Start.md

@@ -57,9 +57,9 @@ and performing [regular upgrades](https://docs.victoriametrics.com/victoriametri
 Download the newest available [VictoriaMetrics release](https://docs.victoriametrics.com/victoriametrics/changelog/)
 from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
 ```sh
-docker pull victoriametrics/victoria-metrics:v1.119.0
+docker pull victoriametrics/victoria-metrics:v1.120.0
 docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
- victoriametrics/victoria-metrics:v1.119.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ victoriametrics/victoria-metrics:v1.120.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
 ```
 _For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
 
@@ -377,11 +377,14 @@ Migrating data from other TSDBs to VictoriaMetrics is as simple as importing dat
 
 The migration might get easier when using [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/) - VictoriaMetrics
 command line tool. It supports the following databases for migration to VictoriaMetrics:
-* [Prometheus using snapshot API](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-prometheus);
-* [Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-thanos);
-* [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x);
-* [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-opentsdb);
-* [Migrate data between VictoriaMetrics single and cluster versions](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics).
+* [Prometheus using snapshot API](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/);
+* [Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos/);
+* [Mimir](https://docs.victoriametrics.com/victoriametrics/vmctl/mimir/);
+* [Promscale](https://docs.victoriametrics.com/victoriametrics/vmctl/promscale/);
+* [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/);
+* [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/);
+* [Migrate data between VictoriaMetrics single and cluster versions](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
+* [Migrate data via Prometheus Remote Read protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/).
 
 ## Productionization
 

docs/victoriametrics/README.md

@@ -2173,7 +2173,7 @@ Things to consider when copying data:
 1. Copying data folder means complete replacement of the previous data on destination VictoriaMetrics.
 
 For more complex scenarios like single-to-cluster, cluster-to-single, re-sharding or migrating only a fraction
-of data - see [vmctl. Migrating data from VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics).
+of data - see [vmctl. Migrating data from VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
 
 ### From other systems
 
@@ -2273,14 +2273,6 @@ The command for collecting CPU profile waits for 30 seconds before returning.
 The collected profiles may be analyzed with [go tool pprof](https://github.com/google/pprof).
 It is safe sharing the collected profiles from security point of view, since they do not contain sensitive information.
 
-## Integrations
-
-* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi) can use VictoriaMetrics as time series backend.
-  See [this example](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml).
-* [netdata](https://github.com/netdata/netdata) can push data into VictoriaMetrics via `Prometheus remote_write API`.
-  See [these docs](https://github.com/netdata/netdata#integrations).
-* [vmalert-cli](https://github.com/aorfanos/vmalert-cli) - a CLI application for managing [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/).
-
 ## Third-party contributions
 
 * [Prometheus -> VictoriaMetrics exporter #1](https://github.com/ryotarai/prometheus-tsdb-dump)
@@ -3045,3 +3037,7 @@ Moved to [integrations/graphite/#metrics-api](https://docs.victoriametrics.com/v
 ###### Graphite Tags API usage
 
 Moved to [integrations/graphite/#tags-api](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#tags-api).
+
+###### Integrations
+
+Moved to [integrations](https://docs.victoriametrics.com/victoriametrics/integrations).

docs/victoriametrics/changelog/CHANGELOG.md

@@ -18,6 +18,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): remove duplicate kubernetes targets from [service-discovery-debug](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging) page. See [8626](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8626) issue for details.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `/api/v1/notifiers` API endpoint for returning list of configured or discovered notifiers.
+* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `datasource_type` query argument for `/api/v1/rules` and `/api/v1/alerts` endpoints to filter response by rule's datasource [type](https://docs.victoriametrics.com/victoriametrics/vmalert/#groups). See [#8537](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8537).
+
 ## [v1.120.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.120.0)
 
 Released at 2025-06-20
@@ -243,7 +247,7 @@ If you are impacted by this, please upgrade to [v1.114.0](https://github.com/Vic
 * FEATURE: [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): add panel `Partial query results` that shows the number of served [partial responses](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-availability) by vmselects.
 
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly apply global relabeling configuration, defined with  `-relabelConfig` flag, for metrics scraped with `-promscrape.config`. Bug was introduces in [v1.108.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.108.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8389).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): properly propagate an error message when applying retention policy fails. Previously, an actual error message was discarded.
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): fix data query in [rate limiter](https://docs.victoriametrics.com/victoriametrics/vmgateway/#rate-limiter). The bug was introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/68bad22fd26d1436ad0236b1f3ced8604c5d851c) starting from [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0).
@@ -305,6 +309,23 @@ Released at 2025-02-10
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix discrepancies when using `or` binary operator. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640) issues for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly update number of unique series for [cardinality limiter](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter) on ingestion. Previously, limit could undercount the real number of the ingested unique series. 
 
+## [v1.110.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.12)
+
+Released at 2025-06-20
+
+**v1.110.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/changelog/#v11100) release**
+
+* SECURITY: upgrade Go builder from Go1.24.3 to Go1.24.4. See [the list of issues addressed in Go1.24.4](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved).
+* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): fixed duplication in Datadog sketches aggregation metrics. See [#8836](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8836) for details.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect sorting of tag filters, which led to suboptimal tag filter evaluation order and potentially degraded query performance in rare cases. See [#9127](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9127) for details.
+* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `/-/healthy` and `/-/ready` endpoints as Prometheus does on vmselect endpoints, thus achieving full Prometheus compatibility for external dependencies.
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix server-side copying of objects for Azure Blob Storage when using managed identity for authentication. Previously, it wasn't possible to use [smart backups](https://docs.victoriametrics.com/victoriametrics/vmbackup/#smart-backups) strategy for `vmbackup` as server-side copy would fail. See [#9131](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9131).
+* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): increase startup healthcheck delay for storage reachability from 30 seconds to 3 minutes. This is required to avoid vmbackupmanager restarts when storage node startup take more than 30 seconds (e.g. when storage nodes stores more than 5Tb of data).
+
 ## [v1.110.11](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.11)
 
 Released at 2025-06-09
@@ -431,7 +452,7 @@ The v1.110.x line will be supported for at least 12 months since [v1.110.0](http
 **Update note 1: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) include a fix which enforces IPv6 addresses escaping for containers discovered with [Kubernetes service-discovery](https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs) and `role: pod` which do not have exposed ports defined. This means that `address` for these containers will always be wrapped in square brackets, this might affect some relabeling rules which were relying on previous behaviour.**
 
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly apply global relabeling configuration, defined with  `-relabelConfig` flag, for metrics scrapped with `-promscrape.config`. Bug was introduces in [v1.108.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.108.0). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8389).
 * BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): properly propagate an error message when applying retention policy fails. Previously, an actual error messages was discarded.
 * BUGFIX: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): fix data query in [rate limiter](https://docs.victoriametrics.com/victoriametrics/vmgateway/#rate-limiter). The bug was introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/68bad22fd26d1436ad0236b1f3ced8604c5d851c) starting from [v1.106.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.106.0).
@@ -568,6 +589,22 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
 
 See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2024/#v11030)
 
+## [v1.102.24](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.24)
+
+Released at 2025-06-20
+
+**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
+All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
+The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
+
+* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
+* SECURITY: upgrade Go builder from Go1.23.9 to Go1.23.10. See [the list of issues addressed in Go1.23.10](https://github.com/golang/go/issues?q=milestone%3AGo1.23.10+label%3ACherryPickApproved).
+
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect sorting of tag filters, which led to suboptimal tag filter evaluation order and potentially degraded query performance in rare cases. See [#9127](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9127) for details.
+* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `/-/healthy` and `/-/ready` endpoints as Prometheus does on vmselect endpoints, thus achieving full Prometheus compatibility for external dependencies.
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix server-side copying of objects for Azure Blob Storage when using managed identity for authentication. Previously, it wasn't possible to use [smart backups](https://docs.victoriametrics.com/victoriametrics/vmbackup/#smart-backups) strategy for `vmbackup` as server-side copy would fail. See [#9131](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9131).
+* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): increase startup healthcheck delay for storage reachability from 30 seconds to 3 minutes. This is required to avoid vmbackupmanager restarts when storage node startup take more than 30 seconds (e.g. when storage nodes stores more than 5Tb of data).
+
 ## [v1.102.23](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.23)
 
 Released at 2025-06-06
@@ -693,7 +730,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
 * SECURITY: upgrade golang.org/x/net from v0.31.0 to v0.33.0 to address [GHSA-w32m-9786-jp63](https://github.com/advisories/GHSA-w32m-9786-jp63).
 
 * BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix the incorrect caching of extMetricsIDs when a query timeout error occurs. This can lead to incorrect query results. Thanks to @changshun-shi for [the bug report issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8345).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): respect time filter when exploring time series for [influxdb mode](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8259) for details.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly escape IPv6 address in [Kubernetes service-discovery](https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs) with `role: pod` for containers without exposed ports. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8374).
 
 ## [v1.102.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.14)

docs/victoriametrics/changelog/CHANGELOG_2021.md

@@ -376,7 +376,7 @@ Released at 2021-05-01
 * FEATURE: improved new time series registration speed on systems with many CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1244). Thanks to @waldoweng for the idea and [draft implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1243).
 * FEATURE: vmalert: use the same technique as Grafana for determining evaluation timestamps for recording rules. This should make consistent graphs for series generated by recording rules compared to graphs generated for queries from recording rules in Grafana. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1232).
 * FEATURE: vmauth: add ability to set mandatory query args in `url_prefix`. For example, `url_prefix: http://vm:8428/?extra_label=team=dev` would add `extra_label=team=dev` query arg to all the incoming requests. See [the example](https://docs.victoriametrics.com/victoriametrics/vmauth/#auth-config) for more details.
-* FEATURE: vmctl: add OpenTSDB migration option. See more details [here](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-opentsdb).
+* FEATURE: vmctl: add OpenTSDB migration option. See more details [here](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/).
 Thanks to @johnseekins!
 * FEATURE: log metrics with dropped labels if the number of labels in the ingested metric exceeds `-maxLabelsPerTimeseries`. This should simplify debugging for this case.
 * FEATURE: vmagent: list user-visible endpoints at `http://vmagent:8429/`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1251).

docs/victoriametrics/changelog/CHANGELOG_2022.md

@@ -116,7 +116,7 @@ Released at 2022-12-11
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): reduce memory and CPU usage by up to 50% on setups with thousands of recording/alerting groups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3464).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-remoteWrite.sendTimeout` command-line flag, which allows configuring timeout for sending data to `-remoteWrite.url`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3408).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add ability to migrate data between VictoriaMetrics clusters with automatic tenants discovery. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/#cluster-to-cluster-migration-mode) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2930).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add ability to copy data from sources via Prometheus `remote_read` protocol. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-by-remote-read-protocol). The related issues: [one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3132) and [two](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1101).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add ability to copy data from sources via Prometheus `remote_read` protocol. See [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). The related issues: [one](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3132) and [two](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1101).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): allow changing timezones for the requested data. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3075).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): provide fast path for hiding results for all the queries except the given one by clicking `eye` icon with `ctrl` key pressed. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3446).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add `range_trim_spikes(phi, q)` function for trimming `phi` percent of the largest spikes per each time series returned by `q`. See [these docs](https://docs.victoriametrics.com/victoriametrics/metricsql/#range_trim_spikes).

docs/victoriametrics/changelog/CHANGELOG_2023.md

@@ -50,7 +50,7 @@ Released at 2023-12-13
 * FEATURE: add field `version` to the response for `/api/v1/status/buildinfo` API for using more efficient API in Grafana for receiving label values. Add additional info about setup Grafana datasource. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5370) and [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/) for details.
 * FEATURE: add `-search.maxResponseSeries` command-line flag for limiting the number of time series a single query to [`/api/v1/query`](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#instant-query) or [`/api/v1/query_range`](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) can return. This limit can protect Grafana from high memory usage when the query returns too many series. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5372).
 * FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): ease aggregation for certain alerting rules to keep more useful labels for the context. Before, all extra labels except `job` and `instance` were ignored. See this [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5429) and this [follow-up commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/8fb68152e67712ed2c16dcfccf7cf4d0af140835). Thanks to @7840vz.
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow reversing the migrating order from the newest to the oldest data for [vm-native](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics) and [remote-read](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-by-remote-read-protocol) modes via `--vm-native-filter-time-reverse` and `--remote-read-filter-time-reverse` command-line flags respectively. See: https://docs.victoriametrics.com/victoriametrics/vmctl/#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow reversing the migrating order from the newest to the oldest data for [vm-native](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/) and [remote-read](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) modes via `--vm-native-filter-time-reverse` and `--remote-read-filter-time-reverse` command-line flags respectively. See: https://docs.victoriametrics.com/victoriametrics/vmctl/#using-time-based-chunking-of-migration and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5376).
 
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly calculate values for the first point on the graph for queries, which do not use [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions). For example, previously `count(up)` could return lower than expected values for the first point on the graph. This also could result in lower than expected values in the middle of the graph like in [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388) when the response caching isn't disabled. The issue has been introduced in [v1.95.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v1950).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent from `FATAL: cannot flush metainfo` panic when [`-remoteWrite.multitenantURL`](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy) command-line flag is set. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5357).
@@ -362,7 +362,7 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow configuring deadline for a backend to be excluded from the rotation on errors via `-failTimeout` cmd-line flag. This feature could be useful when it is expected for backends to be not available for significant periods of time. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4415) for details. Thanks to @SunKyu for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4416).  
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): remove support of deprecated web links of `/api/v1/<groupID>/<alertID>/status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1/<groupID>/<alertID>/status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details.
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side.
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side.
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/victoriametrics/vmctl/#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow specifying custom full url at `--remote-read-src-addr` command-line flag if `--remote-read-disable-path-append` command-line flag is set. This allows importing data from urls, which do not end with `/api/v1/read`. For example, from Promscale. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4655).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add warning in query field of vmui for partial data responses. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4721).
@@ -572,7 +572,7 @@ Released at 2023-05-18
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix the text display on buttons in Safari 16.4.
 * BUGFIX: [alerts-health](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml): update threshold for `TooHighMemoryUsage` alert from 90% to 80%, since 90% is too high for production environments.
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/): fix compatibility with Windows OS. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix bug with duplicated labels during stream aggregation via single-node VictoriaMetrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4277).
 
 ## [v1.90.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.90.0)
@@ -640,10 +640,10 @@ Another option is to upgrade to [v1.89.1](https://docs.victoriametrics.com/victo
 * SECURITY: upgrade Go builder from Go1.20.1 to Go1.20.2. See [the list of issues addressed in Go1.20.2](https://github.com/golang/go/issues?q=milestone%3AGo1.20.2+label%3ACherryPickApproved).
 
 * FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): increase the default value for `--remote-read-http-timeout` command-line option from 30s (30 seconds) to 5m (5 minutes). This reduces the probability of timeout errors when migrating big number of time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3879).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): migrate series one-by-one in [vm-native mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). This allows better tracking the migration progress and resuming the migration process from the last migrated time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3859) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3600).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--vm-native-src-headers` and `--vm-native-dst-headers` command-line flags, which can be used for setting custom HTTP headers during [vm-native migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3906).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--vm-native-src-bearer-token` and `--vm-native-dst-bearer-token` command-line flags, which can be used for setting Bearer token headers for the source and the destination storage during [vm-native migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3835).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--vm-native-disable-http-keep-alive` command-line flag to allow `vmctl` to use non-persistent HTTP connections in [vm-native migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3909).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): migrate series one-by-one in [vm-native mode](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). This allows better tracking the migration progress and resuming the migration process from the last migrated time series. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3859) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3600).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--vm-native-src-headers` and `--vm-native-dst-headers` command-line flags, which can be used for setting custom HTTP headers during [vm-native migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3906).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--vm-native-src-bearer-token` and `--vm-native-dst-bearer-token` command-line flags, which can be used for setting Bearer token headers for the source and the destination storage during [vm-native migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3835).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `--vm-native-disable-http-keep-alive` command-line flag to allow `vmctl` to use non-persistent HTTP connections in [vm-native migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). Thanks to @baconmania for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3909).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): log number of configuration files found for each specified `-rule` command-line flag.
 * FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/victoriametrics/vmalert/): concurrently [read config files from S3, GCS or S3-compatible object storage](https://docs.victoriametrics.com/victoriametrics/vmalert/#reading-rules-from-object-storage). This significantly improves config load speed for cases when there are thousands of files to read from the object storage.
 

docs/victoriametrics/changelog/CHANGELOG_2024.md

@@ -632,9 +632,9 @@ Released at 2024-04-04
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): automatically reload updated root CA certificates from files without the need to restart `vmagent`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5526).
 * FEATURE: optimize [`/api/v1/labels`](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels) and [`/api/v1/label/.../values`](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues) when `match[]` filters contains metric name. For example, `/api/v1/label/instance/values?match[]=up` now works much faster than before. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5055).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): accept Prometheus remote write protocol data at `/api/v1/push` endpoint. This simplifies writing data to `vmagent` from Grafana Mimir. Thanks to @edma2 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5990).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support client-side TLS configuration for [native protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5824).
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support client-side TLS configuration for VictoriaMetrics destination specified via `--vm-*` cmd-line flags used in [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-by-remote-read-protocol), [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-opentsdb), [Prometheus](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-prometheus) and [Promscale](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-promscale) migration modes.
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): split [explore phase](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-victoriametrics) in `vm-native` mode by time intervals when [--vm-native-step-interval](https://docs.victoriametrics.com/victoriametrics/vmctl/#using-time-based-chunking-of-migration) is specified. This should reduce probability of exceeding complexity limits for number of selected series during explore phase. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5369).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support client-side TLS configuration for [native protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5824).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support client-side TLS configuration for VictoriaMetrics destination specified via `--vm-*` cmd-line flags used in [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/), [Remote Read protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/), [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/), [Prometheus](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/) and [Promscale](https://docs.victoriametrics.com/victoriametrics/vmctl/promscale/) migration modes.
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): split [explore phase](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/) in `vm-native` mode by time intervals when [--vm-native-step-interval](https://docs.victoriametrics.com/victoriametrics/vmctl/#using-time-based-chunking-of-migration) is specified. This should reduce probability of exceeding complexity limits for number of selected series during explore phase. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5369).
 * FEATURE: [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/): add `-logInvalidAuthTokens` command-line flag, which can be used for logging invalid auth tokens. This is useful for debugging of auth token format issues. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6029).
 * FEATURE: [graphite](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#render-api): add support for [aggregateSeriesLists](https://graphite.readthedocs.io/en/latest/functions.html#graphite.render.functions.aggregateSeriesLists), [diffSeriesLists](https://graphite.readthedocs.io/en/latest/functions.html#graphite.render.functions.diffSeriesLists), [multiplySeriesLists](https://graphite.readthedocs.io/en/latest/functions.html#graphite.render.functions.multiplySeriesLists) and [sumSeriesLists](https://graphite.readthedocs.io/en/latest/functions.html#graphite.render.functions.sumSeriesLists) functions. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5809).
 * FEATURE: [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#sending-data-via-opentelemetry): add `-opentelemetry.usePrometheusNaming` command-line flag, which can be used for enabling automatic conversion of the ingested metric names and labels into Prometheus-compatible format. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#sending-data-via-opentelemetry) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6037).
@@ -644,7 +644,7 @@ Released at 2024-04-04
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly set `Host` header in requests to scrape targets when [`server_name` option at `tls_config`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config) is set. Previously the `Host` header was set incorrectly to the target hostname in this case.
 * BUGFIX: do not drop `match[]` filter at [`/api/v1/series`](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1series) if `-search.ignoreExtraFiltersAtLabelsAPI` command-line flag is set, since missing `match[]` filter breaks `/api/v1/series` requests.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return proper responses for [AWS Firehose](https://docs.aws.amazon.com/firehose/latest/dev/httpdeliveryrequestresponse.html#requestformat) requests according to [these docs](https://docs.aws.amazon.com/firehose/latest/dev/httpdeliveryrequestresponse.html#responseformat). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6016) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6037).
-* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse TLS key and CA files for [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x) and [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-opentsdb) migration modes.
+* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): properly parse TLS key and CA files for [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/) and [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/) migration modes.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix VictoriaLogs UI query handling to correctly apply `_time` filter across all queries. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5920).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix issue where `step` parameter wasn't updated after moving the focus outside the `step` input field in Firefox.
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix freezing when pasting a query with autocomplete enabled. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5923).
@@ -670,7 +670,7 @@ Released at 2024-03-01
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): support reading [Amazon CloudWatch](https://aws.amazon.com/cloudwatch/) metrics in [OpenTelemetry](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-formats-opentelemetry-100.html) format from [Amazon Data Firehose](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Metric-Streams.html).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for `client_id` option into [kuma_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#kuma_sd_configs) in the same way as Prometheus does. See [this pull request](https://github.com/prometheus/prometheus/pull/13278).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for `enable_compression` option in [scrape_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs) in order to be compatible with Prometheus scrape configs. See [this pull request](https://github.com/prometheus/prometheus/pull/13166) and [this feature request](https://github.com/prometheus/prometheus/issues/12319). Note that `vmagent` was always supporting [`disable_compression` option](https://docs.victoriametrics.com/victoriametrics/vmagent/#scrape_config-enhancements) before Prometheus added `enable_compression` option.
-* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support client-side TLS configuration for [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-by-remote-read-protocol) and [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/#migrating-data-from-opentsdb). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for pull requests [1](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5783), [2](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5798), [3](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5797).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support client-side TLS configuration for [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/), [Remote Read protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) and [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for pull requests [1](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5783), [2](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5798), [3](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5797).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve [`WITH` templates](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/expand-with-exprs) when clicking the `prettify query` button at the right side of query input field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5383).
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): allow filling gaps on graphs with interpolated lines as Grafana does. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5152) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5862).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert): support filtering by group, rule or labels in [vmalert's UI](https://docs.victoriametrics.com/victoriametrics/vmalert/#web) for `/groups` and `/alerts` pages. See [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5791) by @victoramsantos.

docs/victoriametrics/enterprise.md

@@ -89,7 +89,7 @@ VictoriaMetrics Enterprise components are available in the following forms:
 It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Binary releases of VictoriaMetrics Enterprise are available [at the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
-Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz`.
+Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz`.
 
 In order to run binary release of VictoriaMetrics Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and unpack it. Then run the unpacked binary.
@@ -107,8 +107,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
 obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
 
 ```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz
-tar -xzf victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz
+tar -xzf victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz
 ./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
@@ -123,7 +123,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for VictoriaMetrics Enterprise are available at VictoriaMetrics [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
-Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.119.0-enterprise`.
+Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.120.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via command-line
 flag as described [here](#binary-releases).
@@ -133,13 +133,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
 For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
 
 ```sh
-docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.119.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
+docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.120.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
 ```
 
 Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
 
 ```sh
-docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.119.0-enterprise -licenseFile=/path/to/vm-license
+docker run --name=victoria-metrics -v /vm-license:/vm-license  victoriametrics/victoria-metrics:v1.120.0-enterprise -licenseFile=/path/to/vm-license
 ```
 
 Example docker-compose configuration:
@@ -148,7 +148,7 @@ version: "3.5"
 services:
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.119.0
+    image: victoriametrics/victoria-metrics:v1.120.0
     ports:
       - 8428:8428
     volumes:
@@ -180,7 +180,7 @@ is used to provide key in plain-text:
 ```yaml
 server:
   image:
-    tag: v1.119.0-enterprise
+    tag: v1.120.0-enterprise
 
 license:
   key: {BASE64_ENCODED_LICENSE_KEY}
@@ -191,7 +191,7 @@ In order to provide key via existing secret, the following values file is used:
 ```yaml
 server:
   image:
-    tag: v1.119.0-enterprise
+    tag: v1.120.0-enterprise
 
 license:
   secret:
@@ -240,7 +240,7 @@ spec:
   license:
     key: {BASE64_ENCODED_LICENSE_KEY}
   image:
-    tag: v1.119.0-enterprise
+    tag: v1.120.0-enterprise
 ```
 
 In order to provide key via existing secret, the following custom resource is used:
@@ -257,7 +257,7 @@ spec:
       name: vm-license
       key: license
   image:
-    tag: v1.119.0-enterprise
+    tag: v1.120.0-enterprise
 ```
 
 Example secret with license key:
@@ -283,10 +283,10 @@ See full list of CRD specifications [here](https://docs.victoriametrics.com/oper
 ### FIPS compliance
 
 VictoriaMetrics Enterprise components can be run in FIPS compliant mode {{% available_from "v1.118.0" %}}. Binary releases and Docker images
-of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz`
+of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz`
 archive includes `victoria-metrics-prod` and `victoria-metrics-fips` binaries. The latter binary is FIPS compliant.
 
-Docker images of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.119.0-enterprise-fips`
+Docker images of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.120.0-enterprise-fips`
 image uses FIPS compliant binary version.
 
 ## Monitoring license expiration

docs/victoriametrics/integrations/_index.md

@@ -17,6 +17,8 @@ VictoriaMetrics integrates with many popular monitoring solutions as remote stor
 * [InfluxDB](https://docs.victoriametrics.com/victoriametrics/integrations/influxdb/) (write)
 * [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/) (write)
 * [NewRelic](https://docs.victoriametrics.com/victoriametrics/integrations/newrelic/) (write)
+* [Netdata](https://victoriametrics.com/blog/using-victoriametrics-and-netdata/) (write)
+* [go-graphite/carbonapi](https://github.com/go-graphite/carbonapi/blob/main/cmd/carbonapi/carbonapi.example.victoriametrics.yaml) (read)
 
 If you think that community will benefit from new integrations, open a [feature request on GitHub](https://github.com/VictoriaMetrics/VictoriaMetrics/issues).
 

docs/victoriametrics/integrations/prometheus.md

@@ -4,6 +4,7 @@ weight: 2
 menu:
   docs:
     parent: "integrations-vm"
+    identifier: "integrations-prometheus-vm"
     weight: 2
 aliases:
   - /victoriametrics/data-ingestion/prometheus/

docs/victoriametrics/scrape_config_examples.md

@@ -36,8 +36,8 @@ scrape_configs:
 After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to the same directory:
 
 ```
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.119.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/victoria-metrics-linux-amd64-v1.120.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.120.0.tar.gz
 ```
 
 Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -152,8 +152,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/) acco
 
 ```yaml
 # Download and unpack single-node VictoriaMetrics
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.119.0.tar.gz
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/victoria-metrics-linux-amd64-v1.120.0.tar.gz
+tar xzf victoria-metrics-linux-amd64-v1.120.0.tar.gz
 
 # Run single-node VictoriaMetrics with the given scrape.yaml
 ./victoria-metrics-prod -promscrape.config=scrape.yaml

docs/victoriametrics/vmalert.md

@@ -735,6 +735,7 @@ or time series modification via [relabeling](https://docs.victoriametrics.com/vi
 * `http://<vmalert-addr>` - UI;
 * `http://<vmalert-addr>/api/v1/rules` - list of all loaded groups and rules. Supports additional [filtering](https://prometheus.io/docs/prometheus/2.53/querying/api/#rules);
 * `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
+* `http://<vmalert-addr>/api/v1/notifiers` - list all available notifiers;
 * `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in JSON format.
   Used as alert source in AlertManager.
 * `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in web UI.

docs/victoriametrics/vmauth.md

@@ -604,7 +604,7 @@ unauthorized_user:
 
 There are the following solutions for this issue:
 
-- To enumerate every `vmselect` hosname or IP in the `url_prefix` list:
+- To enumerate every `vmselect` hostname or IP in the `url_prefix` list:
 
   ```yaml
   unauthorized_user:

docs/victoriametrics/vmctl/_index.md

@@ -0,0 +1,15 @@
+---
+weight: 8
+menu:
+  docs:
+    parent: victoriametrics
+    weight: 8
+title: vmctl
+tags:
+  - metrics
+aliases:
+  - /vmctl.html
+  - /vmctl/index.html
+  - /vmctl/
+---
+{{% content "vmctl.md" %}}

docs/victoriametrics/vmctl/cortex.md

@@ -0,0 +1,68 @@
+---
+title: Cortex
+weight: 6
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-cortex"
+    weight: 6
+---
+
+Cortex supports [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+`vmctl` in [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) can be used 
+for historical data migration from Cortex.
+
+Check Cortex configuration in the `api` section:
+```yaml
+api:
+  prometheus_http_prefix:
+```
+
+If you defined some prometheus prefix, you should use it when you define flag `--remote-read-src-addr=http://127.0.0.1:9009/{prometheus_http_prefix}`.
+By default, Cortex uses the `prometheus` path prefix, so you should define the flag `--remote-read-src-addr=http://127.0.0.1:9009/prometheus`.
+
+By default, Cortex exposes HTTP port on `:9009 `. The importing process example for the local installation of Cortex
+and single-node VictoriaMetrics(`http://localhost:8428`):
+
+```sh
+./vmctl remote-read \ 
+--remote-read-src-addr=http://127.0.0.1:9009/prometheus \
+--remote-read-filter-time-start=2021-10-18T00:00:00Z \
+--remote-read-step-interval=hour \
+--vm-addr=http://127.0.0.1:8428 \
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+When the process finishes, you will see the following:
+
+```sh
+Split defined times into 8842 ranges to import. Continue? [Y/n]
+VM worker 0:↗ 3863 samples/s
+VM worker 1:↗ 2686 samples/s
+VM worker 2:↗ 2620 samples/s
+VM worker 3:↗ 2705 samples/s
+VM worker 4:↗ 2643 samples/s
+VM worker 5:↗ 2593 samples/s
+Processing ranges: 8842 / 8842 [█████████████████████████████████████████████████████████████████████████████] 100.00%
+2022/10/21 12:09:49 Import finished!
+2022/10/21 12:09:49 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 3.82640757s;
+  total samples: 160232;
+  samples/s: 41875.31;
+  total bytes: 11.3 MB;
+  bytes/s: 3.0 MB;
+  import requests: 6;
+  import requests retries: 0;
+2022/10/21 12:09:49 Total time: 4.71824253s
+```
+
+## Configuration
+
+If you run Cortex installation in multi-tenant mode, remote read protocol requires an Authentication header like `X-Scope-OrgID`.
+You can define it via the flag `--remote-read-headers=X-Scope-OrgID:demo`.
+
+See [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) for more details.
+
+See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).

docs/victoriametrics/vmctl/influxdb.md

@@ -0,0 +1,133 @@
+---
+title: InfluxDB
+weight: 2
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-influxdb"
+    weight: 2
+---
+`vmctl` can migrate historical data from InfluxDB (v1) to VictoriaMetrics. See `./vmctl influx --help` for details and 
+full list of flags. Also see [migrating data from InfluxDB to VictoriaMetrics](https://docs.victoriametrics.com/guides/migrate-from-influx/) article.
+
+To start migration, specify the InfluxDB address `--influx-addr`, database `--influx-database` and VictoriaMetrics address `--vm-addr`:
+```sh
+./vmctl influx --influx-addr=http://<influx-addr>:8086 \
+  --influx-database=benchmark \
+  --vm-addr=http://<victoriametrics-addr>:8428
+InfluxDB import mode
+2020/01/18 20:47:11 Exploring scheme for database "benchmark"
+2020/01/18 20:47:11 fetching fields: command: "show field keys"; database: "benchmark"; retention: "autogen"
+2020/01/18 20:47:11 found 10 fields
+2020/01/18 20:47:11 fetching series: command: "show series "; database: "benchmark"; retention: "autogen"
+Found 40000 timeseries to import. Continue? [Y/n] y
+40000 / 40000 [----------------------------------------------------------------------------------------] 100.00% 21 p/s
+2020/01/18 21:19:00 Import finished!
+2020/01/18 21:19:00 VictoriaMetrics importer stats:
+  idle duration: 13m51.461434876s;
+  time spent while importing: 17m56.923899847s;
+  total samples: 345600000;
+  samples/s: 320914.04;
+  total bytes: 5.9 GB;
+  bytes/s: 5.4 MB;
+  import requests: 40001;
+2020/01/18 21:19:00 Total time: 31m48.467044016s
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+## Data mapping
+
+vmctl modifies InfluxDB data by using the following rules:
+- Field values are mapped to time series values.
+- Tags are mapped to [labels](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels) format as-is.
+- `influx-database` is mapped into `db` label value, unless `db` tag already exists in the InfluxDB line.
+    To skip this mapping, enable flag `influx-skip-database-label`.
+- Field names are mapped to time series names prefixed with `{measurement}{separator}` value,
+  where `{separator}` equals to `_` by default. It can be changed with `--influx-measurement-field-separator` cmd-line flag.
+
+For example, the following InfluxDB line:
+```text
+foo,tag1=value1,tag2=value2 field1=12,field2=40
+```
+
+is converted into the following format in VictoriaMetrics:
+```text
+foo_field1{tag1="value1", tag2="value2"} 12
+foo_field2{tag1="value1", tag2="value2"} 40
+```
+
+See more about [data model differences](https://docs.victoriametrics.com/guides/migrate-from-influx/#data-model-differences) 
+between VictoriaMetrics and InfluxDB.
+
+## Filtering
+
+Additional filtering for exported data from InfluxDB can be applied via `--influx-filter-series` flag. For example:
+```sh
+./vmctl influx --influx-database benchmark \
+  --influx-filter-series "on benchmark from cpu where hostname='host_1703'"
+InfluxDB import mode
+2020/01/26 14:23:29 Exploring scheme for database "benchmark"
+2020/01/26 14:23:29 fetching fields: command: "show field keys"; database: "benchmark"; retention: "autogen"
+2020/01/26 14:23:29 found 12 fields
+2020/01/26 14:23:29 fetching series: command: "show series on benchmark from cpu where hostname='host_1703'"; database: "benchmark"; retention: "autogen"
+Found 10 timeseries to import. Continue? [Y/n]
+```
+
+The timeseries select query would be following:
+`fetching series: command: "show series on benchmark from cpu where hostname='host_1703'"; database: "benchmark"; retention: "autogen"`
+
+To filter by time specify the following flags:
+- `--influx-filter-time-start`
+- `--influx-filter-time-end`
+
+Here's an example of importing timeseries for one day only:
+```sh
+./vmctl influx --influx-database benchmark \
+  --influx-filter-time-start "2020-01-01T10:07:00Z" \
+  --influx-filter-time-end "2020-01-01T15:07:00Z"
+ ```
+
+See more about [time filtering in InfluxDB](https://docs.influxdata.com/influxdb/v1.7/query_language/schema_exploration#filter-meta-queries-by-time).
+
+## InfluxDB v2
+
+Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
+You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
+
+## Configuration
+
+In Influx mode, vmctl fetches data from the InfluxDB by executing read queries. The speed of migration is mostly limited 
+by capabilities of InfluxDB to respond to these queries fast enough. vmctl executes one read request at a time by default.
+Increase `--influx-concurrency` to execute more read requests concurrently. But make sure to not overwhelm InfluxDB
+during migration.
+
+The flag `--influx-chunk-size` controls the max amount of datapoints to return in single chunk from fetch requests.
+Please see more details [here](https://docs.influxdata.com/influxdb/v1.7/guides/querying_data/#chunking).
+The chunk size is used to control InfluxDB memory usage, so it won't OOM on processing large timeseries with
+billions of datapoints.
+
+See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+
+See `./vmctl influx --help` for details and full list of flags:
+```shellhelp
+   --influx-addr value              InfluxDB server addr (default: "http://localhost:8086")
+   --influx-user value              InfluxDB user [$INFLUX_USERNAME]
+   --influx-password value          InfluxDB user password [$INFLUX_PASSWORD]
+   --influx-database value          InfluxDB database
+   --influx-retention-policy value  InfluxDB retention policy (default: "autogen")
+   --influx-chunk-size value        The chunkSize defines max amount of series to be returned in one chunk (default: 10000)
+   --influx-concurrency value       Number of concurrently running fetch queries to InfluxDB (default: 1)
+   --influx-filter-series value     InfluxDB filter expression to select series. E.g. "from cpu where arch='x86' AND hostname='host_2753'".
+      See for details https://docs.influxdata.com/influxdb/v1.7/query_language/schema_exploration#show-series
+   --influx-filter-time-start value            The time filter to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+   --influx-filter-time-end value              The time filter to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+   --influx-measurement-field-separator value  The {separator} symbol used to concatenate {measurement} and {field} names into series name {measurement}{separator}{field}. (default: "_")
+   --influx-skip-database-label                Whether to skip adding the label 'db' to timeseries. (default: false)
+   --influx-prometheus-mode                    Whether to restore the original timeseries name previously written from Prometheus to InfluxDB v1 via remote_write. (default: false)
+   --influx-cert-file value                    Optional path to client-side TLS certificate file to use when connecting to -influx-addr
+   --influx-key-file value                     Optional path to client-side TLS key to use when connecting to -influx-addr
+   --influx-CA-file value                      Optional path to TLS CA file to use for verifying connections to -influx-addr. By default, system CA is used
+   --influx-server-name value                  Optional TLS server name to use for connections to -influx-addr. By default, the server name from -influx-addr is used
+   --influx-insecure-skip-verify               Whether to skip tls verification when connecting to -influx-addr (default: false)
+```

docs/victoriametrics/vmctl/mimir.md

@@ -0,0 +1,72 @@
+---
+title: Mimir
+weight: 8
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-mimir" 
+    weight: 8
+---
+
+GrafanaLabs Mimir supports [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+`vmctl` in [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) can be used
+for historical data migration from Mimir.
+
+By default, Mimir uses the `prometheus` path prefix so specifying the source
+should be as simple as `--remote-read-src-addr=http://<mimir>:9009/prometheus`.
+But if prefix was overridden via `prometheus_http_prefix`, then source address should be updated
+to `--remote-read-src-addr=http://<mimir>:9009/{prometheus_http_prefix}`.
+
+When you run Mimir, it exposes a port to serve HTTP on `8080 by default`.
+
+Next example of the local installation was in multi-tenant mode (3 instances of Mimir) with nginx as load balancer.
+Load balancer expose single port `:9090`. As you can see in the example we call `:9009` instead of `:8080` because of proxy.
+
+The importing process example for the local installation of Mimir and single-node VictoriaMetrics(`http://localhost:8428`):
+
+```
+./vmctl remote-read 
+--remote-read-src-addr=http://<mimir>:9009/prometheus \
+--remote-read-filter-time-start=2021-10-18T00:00:00Z \
+--remote-read-step-interval=hour \
+--remote-read-headers=X-Scope-OrgID:demo \
+--remote-read-use-stream=true \
+--vm-addr=http://<victoria-metrics>:8428 \
+```
+
+> Mimir supports [streamed remote read API](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/),
+so it is recommended setting `--remote-read-use-stream=true` flag for better performance and resource usage.
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+And when the process finishes, you will see the following:
+```sh
+Split defined times into 8847 ranges to import. Continue? [Y/n]
+VM worker 0:→ 12176 samples/s
+VM worker 1:→ 11918 samples/s
+VM worker 2:→ 11261 samples/s
+VM worker 3:→ 12861 samples/s
+VM worker 4:→ 11096 samples/s
+VM worker 5:→ 11575 samples/s
+Processing ranges: 8847 / 8847 [█████████████████████████████████████████████████████████████████████████████] 100.00%
+2022/10/21 17:22:23 Import finished!
+2022/10/21 17:22:23 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 15.379614356s;
+  total samples: 81243;
+  samples/s: 5282.51;
+  total bytes: 6.1 MB;
+  bytes/s: 397.8 kB;
+  import requests: 6;
+  import requests retries: 0;
+2022/10/21 17:22:23 Total time: 16.287405248s
+```
+
+## Configuration
+
+If you run Mimir installation in multi-tenant mode, remote read protocol requires an Authentication header like `X-Scope-OrgID`. Y
+ou can define it via the flag `--remote-read-headers=X-Scope-OrgID:demo`.
+
+See [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) for more details.
+
+See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).

docs/victoriametrics/vmctl/opentsdb.md

@@ -0,0 +1,163 @@
+---
+title: OpenTSDB
+weight: 3
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-opentsdb"
+    weight: 3
+---
+`vmctl` can migrate historical data from OpenTSDB to VictoriaMetrics.
+See `./vmctl opentsdb --help` for details and full list of flags.
+
+> **Important:** OpenTSDB migration is not possible without a functioning [meta](http://opentsdb.net/docs/build/html/user_guide/metadata.html)
+table to search for metrics/series. Check in OpenTSDB config that appropriate options are [activated]( https://github.com/OpenTSDB/opentsdb/issues/681#issuecomment-177359563) 
+and HBase meta tables are present. W/o them migration won't work.
+
+OpenTSDB migration works like so:
+
+1. Find metrics based on selected filters (or the default filter set `['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']`):
+
+   `curl -Ss "http://opentsdb:4242/api/suggest?type=metrics&q=sys"`
+
+1. Find series associated with each returned metric:
+
+   `curl -Ss "http://opentsdb:4242/api/search/lookup?m=system.load5&limit=1000000"`
+
+   Here `results` return field should not be empty. Otherwise, it means that meta tables are absent and needs to be turned on previously.
+
+1. Download data for each series in chunks defined in the CLI switches:
+
+   `-retention=sum-1m-avg:1h:90d` means:
+    - `curl -Ss "http://opentsdb:4242/api/query?start=1h-ago&end=now&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+    - `curl -Ss "http://opentsdb:4242/api/query?start=2h-ago&end=1h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+    - `curl -Ss "http://opentsdb:4242/api/query?start=3h-ago&end=2h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+    - ...
+    - `curl -Ss "http://opentsdb:4242/api/query?start=2160h-ago&end=2159h-ago&m=sum:1m-avg-none:system.load5\{host=host1\}"`
+
+This means that we must stream data from OpenTSDB to VictoriaMetrics in chunks. This is where concurrency for OpenTSDB comes in.
+We can query multiple chunks at once, but we shouldn't perform too many chunks at a time to avoid overloading the OpenTSDB cluster.
+
+```sh
+$ ./vmctl opentsdb --otsdb-addr http://opentsdb:4242/ --otsdb-retentions sum-1m-avg:1h:1d \
+  --otsdb-filters system \ 
+  --otsdb-normalize \
+  --vm-addr <victoriametrics-addr>:8428 \
+OpenTSDB import mode
+2021/04/09 11:52:50 Will collect data starting at TS 1617990770
+2021/04/09 11:52:50 Loading all metrics from OpenTSDB for filters:  [system]
+Found 9 metrics to import. Continue? [Y/n]
+2021/04/09 11:52:51 Starting work on system.load1
+23 / 402200 [>____________________________________________________________________________________________] 0.01% 2 p/s
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+## Retention strings
+
+Starting with a relatively simple retention string (`sum-1m-avg:1h:30d`), let's describe how this is converted into actual queries.
+
+There are two essential parts of a retention string:
+
+1. [aggregation](#aggregation)
+1. [windows/time ranges](#windows)
+
+## Aggregation
+
+Retention strings essentially define the two levels of aggregation for our collected series.
+
+`sum-1m-avg` would become:
+
+- First order: `sum`
+- Second order: `1m-avg-none`
+
+### First Order Aggregations
+
+First-order aggregation addresses how to aggregate any un-mentioned tags.
+
+This is, conceptually, directly opposite to how PromQL deals with tags. In OpenTSDB, if a tag isn't explicitly mentioned,
+all values associated with that tag will be aggregated.
+
+It is recommended to use `sum` for the first aggregation because it is relatively quick and should not cause any changes 
+to the incoming data (because we collect each individual series).
+
+### Second Order Aggregations
+
+Second-order aggregation (`1m-avg` in our example) defines any windowing that should occur before returning the data.
+It is recommended to match the stat collection interval, so we again avoid transforming incoming data.
+
+We do not allow for defining the "null value" portion of the rollup window (e.g. in the aggregation, `1m-avg-none`, 
+the user cannot change `none`), as the goal of this tool is to avoid modifying incoming data.
+
+## Windows
+
+There are two important windows we define in a retention string:
+
+1. the "chunk" range of each query
+1. The time range we will be querying on with that "chunk"
+
+From our example, our windows are `1h:30d`.
+
+### Window "chunks"
+
+The window `1h` means that each individual query to OpenTSDB should only span 1 hour of time (e.g. `start=2h-ago&end=1h-ago`).
+
+It is important to ensure this window somewhat matches the row size in HBase to help improve query times.
+
+For example, if the query is hitting a rollup table with a 4-hour row size, we should set a chunk size of a multiple of 
+4 hours (e.g. `4h`, `8h`, etc.) to avoid requesting data across row boundaries. Landing on row boundaries allows for more
+consistent request times to HBase.
+
+The default table created in HBase for OpenTSDB has a 1-hour row size, so if you aren't sure on a correct row size to use,
+`1h` is a reasonable choice.
+
+### Time range
+
+The time range `30d` simply means we are asking for the last 30 days of data. This time range can be written using `h`, `d`, `w`, or `y`. (We can't use `m` for month because it already means `minute` in time parsing).
+
+## Results of retention string
+
+The resultant queries that will be created, based on our example retention string of `sum-1m-avg:1h:30d` look like this:
+```sh
+http://opentsdb:4242/api/query?start=1h-ago&end=now&m=sum:1m-avg-none:<series>
+http://opentsdb:4242/api/query?start=2h-ago&end=1h-ago&m=sum:1m-avg-none:<series>
+http://opentsdb:4242/api/query?start=3h-ago&end=2h-ago&m=sum:1m-avg-none:<series>
+...
+http://opentsdb:4242/api/query?start=721h-ago&end=720h-ago&m=sum:1m-avg-none:<series>
+```
+
+Chunking the data like this means each individual query returns faster, so we can start populating data into VictoriaMetrics quicker.
+
+## Restarting OpenTSDB migrations
+
+One important note for OpenTSDB migration: Queries/HBase scans can "get stuck" within OpenTSDB itself. 
+This can cause instability and performance issues within an OpenTSDB cluster, so stopping the migrator to deal with it may be necessary.
+Because of this, we provide the timestamp we started collecting data from at the beginning of the run. 
+You can stop and restart the importer using this "hard timestamp" to ensure you collect data from the same time range over multiple runs.
+
+## Configuration
+
+In OpenTSDB mode, vmctl fetches data from the OpenTSDB by executing read queries. The speed of migration is mostly limited
+by capabilities of OpenTSDB to respond to these queries fast enough. vmctl executes one read request at a time by default.
+Increase `--otsdb-concurrency` to execute more read requests concurrently. But make sure to not overwhelm OpenTSDB
+during migration.
+
+See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+
+See `./vmctl opentsdb --help` for details and full list of flags:
+```shellhelp
+   --otsdb-addr value                                     OpenTSDB server addr (default: "http://localhost:4242")
+   --otsdb-concurrency value                              Number of concurrently running fetch queries to OpenTSDB per metric (default: 1)
+   --otsdb-retentions value [ --otsdb-retentions value ]  Retentions patterns to collect on. Each pattern should describe the aggregation performed for the query, the row size (in HBase) that will define how long each individual query is, and the time range to query for. e.g. sum-1m-avg:1h:3d. The first time range defined should be a multiple of the row size in HBase. e.g. if the row size is 2 hours, 4h is good, 5h less so. We want each query to land on unique rows.
+   --otsdb-filters value [ --otsdb-filters value ]        Filters to process for discovering metrics in OpenTSDB (default: "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z")
+   --otsdb-offset-days value                              Days to offset our 'starting' point for collecting data from OpenTSDB (default: 0)
+   --otsdb-hard-ts-start value                            A specific timestamp to start from, will override using an offset (default: 0)
+   --otsdb-query-limit value                              Result limit on meta queries to OpenTSDB (affects both metric name and tag value queries, recommended to use a value exceeding your largest series) (default: 100000000)
+   --otsdb-msecstime                                      Whether OpenTSDB is writing values in milliseconds or seconds (default: false)
+   --otsdb-normalize                                      Whether to normalize all data received to lower case before forwarding to VictoriaMetrics (default: false)
+   --otsdb-cert-file value                                Optional path to client-side TLS certificate file to use when connecting to -otsdb-addr
+   --otsdb-key-file value                                 Optional path to client-side TLS key to use when connecting to -otsdb-addr
+   --otsdb-CA-file value                                  Optional path to TLS CA file to use for verifying connections to -otsdb-addr. By default, system CA is used
+   --otsdb-server-name value                              Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from -otsdb-addr is used
+   --otsdb-insecure-skip-verify                           Whether to skip tls verification when connecting to -otsdb-addr (default: false)
+```

docs/victoriametrics/vmctl/prometheus.md

@@ -0,0 +1,120 @@
+---
+title: Prometheus
+weight: 1
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-prometheus"
+    weight: 1
+---
+`vmctl` can migrate historical data from Prometheus to VictoriaMetrics by reading [Prometheus snapshot](https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot).
+See `./vmctl prometheus --help` for details and full list of flags. Also see Prometheus [related articles](https://docs.victoriametrics.com/victoriametrics/vmctl/#articles).
+
+To start migration, take [a snapshot of Prometheus data](https://www.robustperception.io/taking-snapshots-of-prometheus-data)
+and made it accessible for vmctl in the same filesystem:
+```sh
+./vmctl prometheus \
+  --vm-addr=<victoriametrics-addr>:8428 \
+  --prom-snapshot=/path/to/snapshot
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+As soon as required flags are provided and healthchecks are done, `vmctl` will start exploring Prometheus snapshot.
+It fetches all available blocks from the snapshot, reads the metadata and prints stats for discovered data:
+```sh
+./vmctl prometheus --prom-snapshot=/path/to/snapshot \
+  --vm-addr=http://localhost:8428
+Prometheus import mode
+Prometheus snapshot stats:
+  blocks found: 14;
+  blocks skipped: 0;
+  min time: 1581288163058 (2020-02-09T22:42:43Z);
+  max time: 1582409128139 (2020-02-22T22:05:28Z);
+  samples: 32549106;
+  series: 27289.
+Found 14 blocks to import. Continue? [Y/n] y
+14 / 14 [-------------------------------------------------------------------------------------------] 100.00% 0 p/s
+2020/02/23 15:50:03 Import finished!
+2020/02/23 15:50:03 VictoriaMetrics importer stats:
+  idle duration: 6.152953029s;
+  time spent while importing: 44.908522491s;
+  total samples: 32549106;
+  samples/s: 724786.84;
+  total bytes: 669.1 MB;
+  bytes/s: 14.9 MB;
+  import requests: 323;
+  import requests retries: 0;
+2020/02/23 15:50:03 Total time: 51.077451066s
+```
+
+## Filtering
+
+Filtering by time is configured via flags `--prom-filter-time-start` and `--prom-filter-time-end` in RFC3339 format.
+The filter is applied twice: to drop blocks on exploration phase and to filter timeseries within the blocks:
+```sh
+./vmctl prometheus --prom-snapshot=/path/to/snapshot \
+  --prom-filter-time-start=2020-02-07T00:07:01Z \
+  --prom-filter-time-end=2020-02-11T00:07:01Z
+Prometheus import mode
+Prometheus snapshot stats:
+  blocks found: 2;
+  blocks skipped: 12;
+  min time: 1581288163058 (2020-02-09T22:42:43Z);
+  max time: 1581328800000 (2020-02-10T10:00:00Z);
+  samples: 1657698;
+  series: 3930.
+Found 2 blocks to import. Continue? [Y/n] y
+```
+
+Filtering by labels is configured with following flags:
+- `--prom-filter-label` - the label name, e.g. `__name__` or `instance`;
+- `--prom-filter-label-value` - the regular expression to filter the label value. By default, matches all `.*`
+
+For example:
+```sh
+./vmctl prometheus --prom-snapshot=/path/to/snapshot \
+  --prom-filter-label="__name__" \
+  --prom-filter-label-value="promhttp.*" \
+  --prom-filter-time-start=2020-02-07T00:07:01Z \
+  --prom-filter-time-end=2020-02-11T00:07:01Z
+Prometheus import mode
+Prometheus snapshot stats:
+  blocks found: 2;
+  blocks skipped: 12;
+  min time: 1581288163058 (2020-02-09T22:42:43Z);
+  max time: 1581328800000 (2020-02-10T10:00:00Z);
+  samples: 1657698;
+  series: 3930.
+Found 2 blocks to import. Continue? [Y/n] y
+14 / 14 [-----------------------------------------------------------------------------------------------] 100.00% ? p/s
+2020/02/23 15:51:07 Import finished!
+2020/02/23 15:51:07 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 37.415461ms;
+  total samples: 10128;
+  samples/s: 270690.24;
+  total bytes: 195.2 kB;
+  bytes/s: 5.2 MB;
+  import requests: 2;
+  import requests retries: 0;
+2020/02/23 15:51:07 Total time: 7.153158218s
+```
+
+## Configuration
+
+In Prometheus mode, vmctl reads data from the given snapshot using Prometheus library. Its performance for is limited
+by the library performance, by the disk IO, and by `--prom-concurrency` - the number of concurrent readers. Prefer
+setting `--prom-concurrency` to the number of available to vmctl CPU cores.
+
+See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+
+See `./vmctl prometheus --help` for details and full list of flags:
+```shellhelp
+   --prom-snapshot value            Path to Prometheus snapshot. Pls see for details https://www.robustperception.io/taking-snapshots-of-prometheus-data
+   --prom-concurrency value         Number of concurrently running snapshot readers (default: 1)
+   --prom-filter-time-start value   The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+   --prom-filter-time-end value     The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+   --prom-filter-label value        Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
+   --prom-filter-label-value value  Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
+```

docs/victoriametrics/vmctl/promscale.md

@@ -0,0 +1,54 @@
+---
+title: Promscale
+weight: 7
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-promscale" 
+    weight: 7
+---
+
+[Promscale](https://github.com/timescale/promscale) supports [Prometheus Remote Read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+To migrate historical data from Promscale to VictoriaMetrics use `vmctl` in [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/).
+
+See the example of migration command below:
+```sh
+./vmctl remote-read \
+  --remote-read-src-addr=http://<promscale>:9201/read \
+  --remote-read-step-interval=day \
+  --vm-addr=http://<victoriametrics>:8428 \
+  --remote-read-filter-time-start=2023-08-21T00:00:00Z \
+  --remote-read-disable-path-append=true # promscale has custom remote read API HTTP path
+Selected time range "2023-08-21 00:00:00 +0000 UTC" - "2023-08-21 14:11:41.561979 +0000 UTC" will be split into 1 ranges according to "day" step. Continue? [Y/n] y
+VM worker 0:↙ 82831 samples/s                                                                                                                                                                        
+VM worker 1:↙ 54378 samples/s                                                                                                                                                                        
+VM worker 2:↙ 121616 samples/s                                                                                                                                                                       
+VM worker 3:↙ 59164 samples/s                                                                                                                                                                        
+VM worker 4:↙ 59220 samples/s                                                                                                                                                                        
+VM worker 5:↙ 102072 samples/s                                                                                                                                                                       
+Processing ranges: 1 / 1 [████████████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/08/21 16:11:55 Import finished!
+2023/08/21 16:11:55 VictoriaMetrics importer stats:
+  idle duration: 0s;
+  time spent while importing: 14.047045459s;
+  total samples: 262111;
+  samples/s: 18659.51;
+  total bytes: 5.3 MB;
+  bytes/s: 376.4 kB;
+  import requests: 6;
+  import requests retries: 0;
+2023/08/21 16:11:55 Total time: 14.063458792s
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+Here we specify the full path to Promscale's Remote Read API via `--remote-read-src-addr`, and disable auto-path
+appending via `--remote-read-disable-path-append` cmd-line flags. This is necessary, as Promscale has a different to
+Prometheus API path. Promscale doesn't support stream mode for Remote Read API,
+so we disable it via `--remote-read-use-stream=false`.
+
+## Configuration
+
+See [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) for more details.
+
+See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).

docs/victoriametrics/vmctl/remoteread.md

@@ -0,0 +1,88 @@
+---
+title: Remote Read
+weight: 4
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-remote-read"
+    weight: 4
+---
+
+`vmctl` supports `remote-read` mode for migrating data from remote databases that support
+[Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/).
+
+`vmctl remote-read` allows migrating data from the following systems:
+- [Cortex](https://docs.victoriametrics.com/victoriametrics/vmctl/cortex/)
+- [Mimir](https://docs.victoriametrics.com/victoriametrics/vmctl/mimir/)
+- [Promscale](https://docs.victoriametrics.com/victoriametrics/vmctl/promscale/)
+- [Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos#remote-read-protocol)
+
+Remote read API has two implementations of remote read API: default (`SAMPLES`) and
+[streamed](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/) (`STREAMED_XOR_CHUNKS`).
+Streamed version is more efficient but has lower adoption.
+
+See `./vmctl remote-read --help` for details and the full list of flags.
+
+> Migration via remote read protocol allows to fetch data via API. This is usually a resource intensive operation
+for Thanos and may be slow or expensive in terms of resources.
+
+The importing process example for local installation of Prometheus:
+
+```sh
+./vmctl remote-read \
+--remote-read-src-addr=http://<prometheus>:9091 \
+--remote-read-filter-time-start=2021-10-18T00:00:00Z \
+--remote-read-step-interval=hour \
+--vm-addr=http://<victoria-metrics>:8428 \
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+## Filtering
+
+Filtering by time can be configured via flags `--remote-read-filter-time-start` and `--remote-read-filter-time-end`
+in RFC3339 format.
+
+Filtering by labels can be configured via flags `--remote-read-filter-label` and `--remote-read-filter-label-value`.
+For example, `--remote-read-filter-label=tenant` and `--remote-read-filter-label-value="team-eu"` will select only series
+with `tenant="team-eu"` label-value pair.
+
+## Configuration 
+
+Migrating big volumes of data may result in remote read client reaching the timeout. Increase the value of 
+`--remote-read-http-timeout` (default `5m`) command-line flag when seeing timeouts or `context canceled` errors.
+
+Flag `--remote-read-step-interval` allows splitting export data into chunks to reduce pressure on the source `--remote-read-src-addr`.
+Valid values are `month, day, hour, minute`.
+
+Flag `--remote-read-use-stream` defines whether to use `SAMPLES` or `STREAMED_XOR_CHUNKS` mode. 
+Mode `STREAMED_XOR_CHUNKS` is much less resource intensive for the source, but not many databases support it. 
+By default, is uses `SAMPLES` mode.
+
+See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+
+See `./vmctl remote-read --help` for details and full list of flags:
+```shellhelp
+   --remote-read-concurrency value         Number of concurrently running remote read readers (default: 1)
+   --remote-read-filter-time-start value   The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'
+   --remote-read-filter-time-end value     The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
+   --remote-read-filter-label value        Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name. (default: "__name__")
+   --remote-read-filter-label-value value  Prometheus regular expression to filter label from "remote-read-filter-label-value" flag. (default: ".*")
+   --remote-read                           Use Prometheus remote read protocol (default: false)
+   --remote-read-use-stream                Defines whether to use SAMPLES or STREAMED_XOR_CHUNKS mode. By default, is uses SAMPLES mode. See https://prometheus.io/docs/prometheus/latest/querying/remote_read_api/#streamed-chunks (default: false)
+   --remote-read-step-interval value       The time interval to split the migration into steps. For example, to migrate 1y of data with '--remote-read-step-interval=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--remote-read-filter-time-reverse'. Requires setting '--remote-read-filter-time-start'. Valid values are 'month','week','day','hour','minute'.
+   --remote-read-filter-time-reverse       Whether to reverse the order of time intervals split by '--remote-read-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest data. (default: false)
+   --remote-read-src-addr value            Remote read address to perform read from.
+   --remote-read-user value                Remote read username for basic auth [$REMOTE_READ_USERNAME]
+   --remote-read-password value            Remote read password for basic auth [$REMOTE_READ_PASSWORD]
+   --remote-read-http-timeout value        Timeout defines timeout for HTTP requests made by remote read client (default: 0s)
+   --remote-read-headers value             Optional HTTP headers to send with each request to the corresponding remote source storage 
+      For example, --remote-read-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding remote source storage. 
+      Multiple headers must be delimited by '^^': --remote-read-headers='header1:value1^^header2:value2'
+   --remote-read-cert-file value       Optional path to client-side TLS certificate file to use when connecting to -remote-read-src-addr
+   --remote-read-key-file value        Optional path to client-side TLS key to use when connecting to -remote-read-src-addr
+   --remote-read-CA-file value         Optional path to TLS CA file to use for verifying connections to -remote-read-src-addr. By default, system CA is used
+   --remote-read-server-name value     Optional TLS server name to use for connections to remoteReadSrcAddr. By default, the server name from -remote-read-src-addr is used
+   --remote-read-insecure-skip-verify  Whether to skip TLS certificate verification when connecting to the remote read address (default: false)
+   --remote-read-disable-path-append   Whether to disable automatic appending of the /api/v1/read suffix to --remote-read-src-addr (default: false)
+```

docs/victoriametrics/vmctl/thanos.md

@@ -0,0 +1,145 @@
+---
+title: Thanos
+weight: 5
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-thanos"
+    weight: 5
+---
+
+Thanos uses the same storage engine as Prometheus and its data layout on-disk should be the same. That means
+`vmctl` in mode [Prometheus](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/) may be used for migrating historical data from Thanos.
+
+## Migration via Prometheus mode
+
+We assume you're using **Thanos Sidecar** on your Prometheus pods, and that you have a separate Thanos Store installation.
+To migrate the data we need to start writing fresh (current) data to VictoriaMetrics, and migrate historical data in background.
+
+### Current data
+
+1. For now, keep your Thanos Sidecar and Thanos-related Prometheus configuration, but make it to stream metrics to VictoriaMetrics:
+    ```yaml
+    remote_write:
+    - url: http://<victoriametrics-addr>:8428/api/v1/write
+    ```
+   
+_Replace `<victoriametrics-addr>` with the VictoriaMetrics hostname or IP address._
+
+For cluster version use vminsert address:
+```
+http://<vminsert-addr>:8480/insert/<tenant>/prometheus
+```
+_Replace `<vminsert-addr>` with the hostname or IP address of vminsert service._
+
+If you have more than 1 vminsert, configure [load-balancing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup).
+Replace `<tenant>` based on your [multitenancy settings](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy).
+
+2. Check the logs to make sure that Prometheus is sending and VM is receiving.
+   In Prometheus, make sure there are no errors. On the VM side, you should see messages like this:
+    ```sh
+    2020-04-27T18:38:46.474Z info VictoriaMetrics/lib/storage/partition.go:207 creating a partition "2025_04" with smallPartsPath="/victoria-metrics-data/data/small/2025_04", bigPartsPath="/victoria-metrics-data/data/big/2025_04"
+    2020-04-27T18:38:46.506Z info VictoriaMetrics/lib/storage/partition.go:222 partition "2025_04" has been created
+    ```
+
+3. Within two hours, Prometheus should finish its current data file and hand it off to Thanos Store for long term
+   storage.
+
+### Historical data
+
+Let's assume your data is stored on S3 served by minio. You first need to copy that out to a local filesystem,
+then import it into VM using `vmctl` in [Prometheus mode](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/).
+
+1. Copy data from minio.
+    1. Run the `minio/mc` Docker container.
+    1. `mc config host add minio http://minio:9000 accessKey secretKey`, substituting appropriate values for the last 3 items.
+    1. `mc cp -r minio/prometheus thanos-data`
+    1. When you copy the data from the minio be sure to copy the entire `thanos-data` directory, which contains all the blocks.
+       The directory structure should look like this:
+
+    ```sh
+      thanos-data
+      ├── 01JWS713P2E4MQW7T643GYGD69
+      │    ├── chunks
+      │    │    └── 000001
+      │    ├── index
+      │    ├── meta.json
+      │    └── tombstones
+   ```
+   If you have multiple blocks, they will be in the same directory, e.g.:
+
+    ```sh
+      thanos-data
+      ├── 01JWS713P2E4MQW7T643GYGD69
+      ├── 01JWS713P2E4MQW7T643GYGD70
+      ├── 01JWS713P2E4MQW7T643GYGD71
+      └── ...
+    ```
+    1. Import data into VictoriaMetrics with `vmctl`.
+
+2. Import using `vmctl`.
+    1. Follow the [Quick Start instructions](https://docs.victoriametrics.com/victoriametrics/vmctl#quick-start) to get `vmctl` on your machine.
+    1. Use [Prometheus](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/) mode to import data:
+    ```sh
+    vmctl prometheus --prom-snapshot thanos-data --vm-addr http://victoria-metrics:8428
+    ```
+
+## Remote read protocol
+
+> Migration via remote read protocol allows to fetch data via API. This is usually a resource intensive operation
+for Thanos and may be slow or expensive in terms of resources. 
+
+Currently, Thanos doesn't support streaming remote read protocol. It is [recommended](https://thanos.io/tip/thanos/integrations.md/#storeapi-as-prometheus-remote-read)
+to use [thanos-remote-read](https://github.com/G-Research/thanos-remote-read). It is a proxy, that allows exposing any Thanos
+service (or anything that exposes gRPC StoreAPI e.g. Querier) via Prometheus remote read protocol.
+
+Run the proxy and define the Thanos store address `./thanos-remote-read -store 127.0.0.1:19194`.
+It is important to know that `store` flag is Thanos Store API gRPC endpoint.
+The proxy exposes port to serve HTTP on `10080 by default`.
+
+The importing process example for local installation of Thanos and single-node VictoriaMetrics(`http://localhost:8428`):
+```sh
+./vmctl remote-read \
+--remote-read-src-addr=http://127.0.0.1:10080 \
+--remote-read-filter-time-start=2021-10-18T00:00:00Z \
+--remote-read-step-interval=hour \
+--vm-addr=http://127.0.0.1:8428 \
+```
+
+_See how to configure [--vm-addr](https://docs.victoriametrics.com/victoriametrics/vmctl#configuring-victoriametrics)._
+
+On the [thanos-remote-read](https://github.com/G-Research/thanos-remote-read) proxy side you will see logs like:
+```sh
+ts=2022-10-19T15:05:04.193916Z caller=main.go:278 level=info traceID=00000000000000000000000000000000 msg="thanos request" request="min_time:1666180800000 max_time:1666184399999 matchers:<type:RE value:\".*\" > aggregates:RAW "
+ts=2022-10-19T15:05:04.468852Z caller=main.go:278 level=info traceID=00000000000000000000000000000000 msg="thanos request" request="min_time:1666184400000 max_time:1666187999999 matchers:<type:RE value:\".*\" > aggregates:RAW "
+ts=2022-10-19T15:05:04.553914Z caller=main.go:278 level=info traceID=00000000000000000000000000000000 msg="thanos request" request="min_time:1666188000000 max_time:1666191364863 matchers:<type:RE value:\".*\" > aggregates:RAW "
+```
+
+And when process will finish you will see:
+```sh
+Split defined times into 8799 ranges to import. Continue? [Y/n]
+VM worker 0:↓ 98183 samples/s
+VM worker 1:↓ 114640 samples/s
+VM worker 2:↓ 131710 samples/s
+VM worker 3:↓ 114256 samples/s
+VM worker 4:↓ 105671 samples/s
+VM worker 5:↓ 124000 samples/s
+Processing ranges: 8799 / 8799 [██████████████████████████████████████████████████████████████████████████████] 100.00%
+2022/10/19 18:05:07 Import finished!
+2022/10/19 18:05:07 VictoriaMetrics importer stats:
+  idle duration: 52m13.987637229s;
+  time spent while importing: 9m1.728983776s;
+  total samples: 70836111;
+  samples/s: 130759.32;
+  total bytes: 2.2 GB;
+  bytes/s: 4.0 MB;
+  import requests: 356;
+  import requests retries: 0;
+2022/10/19 18:05:07 Total time: 9m2.607521618s
+```
+
+## Configuration
+
+See [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) for more details.
+
+See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).

docs/victoriametrics/vmctl/victoriametrics.md

@@ -0,0 +1,230 @@
+---
+title: VictoriaMetrics
+weight: 9
+menu:
+  docs:
+    parent: "vmctl"
+    identifier: "vmctl-victoriametrics" 
+    weight: 9
+---
+
+The simplest way to migrate data between VictoriaMetrics installations is [to copy data between instances](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#data-migration).
+But when simple copy isn't possible (migration between single-node and cluster, or re-sharding) or when data need to be
+modified - use `vmctl vm-native` to migrate data.
+
+See `./vmctl vm-native --help` for details and full list of flags.
+
+vmctl uses [native binary protocol](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format)
+to migrate data between VictoriaMetrics installations:
+* single-node to single-node
+* cluster to cluster
+* single to cluster and vice versa.
+
+Migration in `vm-native` mode takes two steps:
+1. Explore the list of the metrics to migrate via `api/v1/label/__name__/values` API;
+1. Migrate explored metrics one-by-one with specified `--vm-concurrency`.
+
+```sh
+./vmctl vm-native \
+    --vm-native-src-addr=http://127.0.0.1:8481/select/0/prometheus \ # migrate from
+    --vm-native-dst-addr=http://localhost:8428 \                     # migrate to
+    --vm-native-filter-time-start='2022-11-20T00:00:00Z' \           # starting from
+    --vm-native-filter-match='{__name__!~"vm_.*"}'                   # match only metrics without `vm_` prefix
+VictoriaMetrics Native import mode
+
+2023/03/02 09:22:02 Initing import process from "http://127.0.0.1:8481/select/0/prometheus/api/v1/export/native" 
+                    to "http://localhost:8428/api/v1/import/native" with filter 
+        filter: match[]={__name__!~"vm_.*"}
+        start: 2022-11-20T00:00:00Z
+2023/03/02 09:22:02 Exploring metrics...
+Found 9 metrics to import. Continue? [Y/n] 
+2023/03/02 09:22:04 Requests to make: 9
+Requests to make: 9 / 9 [█████████████████████████████████████████████████████████████████████████████] 100.00%
+2023/03/02 09:22:06 Import finished!
+2023/03/02 09:22:06 VictoriaMetrics importer stats:
+  time spent while importing: 3.632638875s;
+  total bytes: 7.8 MB;
+  bytes/s: 2.1 MB;
+  requests: 9;
+  requests retries: 0;
+2023/03/02 09:22:06 Total time: 3.633127625s
+```
+
+## Time intervals
+
+It is possible to split the migration process into steps based on time via `--vm-native-step-interval` cmd-line flag.
+It allows to reduce amount of matched series per each request and significantly reduced the load on installations 
+with [high churn rate](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-high-churn-rate).
+
+Supported values for `--vm-native-step-interval` are: `month`, `week`, `day`, `hour`, `minute`. 
+For example, when migrating 1 year of data with `--vm-native-step-interval=month` vmctl will execute it in 12 separate 
+requests from the beginning of the interval to its end. 
+
+> To reverse the order set `--vm-native-filter-time-reverse` and migration will start from the newest to the
+oldest data. `--vm-native-filter-time-start` is required to be set when using `--vm-native-step-interval`.
+
+It is recommended using default `month` step when migrating the data over the long time intervals. If you hit query
+limits on `--vm-native-src-addr` and can't or don't want to change them, try lowering the step interval to `week`, `day` or `hour`.
+
+Usage example:
+```sh
+./vmctl vm-native \
+    --vm-native-src-addr=http://127.0.0.1:8481/select/0/prometheus \ 
+    --vm-native-dst-addr=http://localhost:8428 \
+    --vm-native-filter-time-start='2022-11-20T00:00:00Z' \
+    --vm-native-step-interval=month \
+    --vm-native-filter-match='{__name__!~"vm_.*"}'    
+VictoriaMetrics Native import mode
+
+2023/03/02 09:18:05 Initing import process from "http://127.0.0.1:8481/select/0/prometheus/api/v1/export/native" to "http://localhost:8428/api/v1/import/native" with filter 
+        filter: match[]={__name__!~"vm_.*"}
+        start: 2022-11-20T00:00:00Z
+2023/03/02 09:18:05 Exploring metrics...
+Found 9 metrics to import. Continue? [Y/n] 
+2023/03/02 09:18:07 Selected time range will be split into 5 ranges according to "month" step. Requests to make: 45.
+```
+
+## Cluster to cluster
+
+Using cluster-to-cluster migration mode helps to migrate all tenants data in a single `vmctl` run.
+
+Cluster-to-cluster uses `/admin/tenants` endpoint {{% available_from "v1.84.0" %}} to discover list of tenants from source cluster.
+
+To use this mode set `--vm-intercluster` flag to `true`, `--vm-native-src-addr` flag to 'http://vmselect:8481/' 
+and `--vm-native-dst-addr` value to http://vminsert:8480/:
+```sh
+  ./vmctl vm-native --vm-native-src-addr=http://127.0.0.1:8481/ \
+  --vm-native-dst-addr=http://127.0.0.1:8480/ \
+  --vm-native-filter-match='{__name__="vm_app_uptime_seconds"}' \
+  --vm-native-filter-time-start='2023-02-01T00:00:00Z' \
+  --vm-native-step-interval=day \  
+  --vm-intercluster
+  
+VictoriaMetrics Native import mode
+2023/02/28 10:41:42 Discovering tenants...
+2023/02/28 10:41:42 The following tenants were discovered: [0:0 1:0 2:0 3:0 4:0]
+2023/02/28 10:41:42 Initing import process from "http://127.0.0.1:8481/select/0:0/prometheus/api/v1/export/native" to "http://127.0.0.1:8480/insert/0:0/prometheus/api/v1/import/native" with filter 
+        filter: match[]={__name__="vm_app_uptime_seconds"}
+        start: 2023-02-01T00:00:00Z for tenant 0:0 
+2023/02/28 10:41:42 Exploring metrics...
+2023/02/28 10:41:42 Found 1 metrics to import 
+2023/02/28 10:41:42 Selected time range will be split into 28 ranges according to "day" step. 
+Requests to make for tenant 0:0: 28 / 28 [████████████████████████████████████████████████████████████████████] 100.00%
+
+2023/02/28 10:41:45 Initing import process from "http://127.0.0.1:8481/select/1:0/prometheus/api/v1/export/native" to "http://127.0.0.1:8480/insert/1:0/prometheus/api/v1/import/native" with filter 
+        filter: match[]={__name__="vm_app_uptime_seconds"}
+        start: 2023-02-01T00:00:00Z for tenant 1:0 
+2023/02/28 10:41:45 Exploring metrics...
+2023/02/28 10:41:45 Found 1 metrics to import 
+2023/02/28 10:41:45 Selected time range will be split into 28 ranges according to "day" step. Requests to make: 28 
+Requests to make for tenant 1:0: 28 / 28 [████████████████████████████████████████████████████████████████████] 100.00%
+
+...
+
+2023/02/28 10:42:49 Import finished!
+2023/02/28 10:42:49 VictoriaMetrics importer stats:
+  time spent while importing: 1m6.714210417s;
+  total bytes: 39.7 MB;
+  bytes/s: 594.4 kB;
+  requests: 140;
+  requests retries: 0;
+2023/02/28 10:42:49 Total time: 1m7.147971417s
+```
+
+## Configuration
+
+Migrating big volumes of data may result in reaching the safety limits on `src` side.
+Please verify that `-search.maxExportDuration` and `-search.maxExportSeries` were set with proper values for `src`. 
+If hitting the limits, follow the recommendations [here](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format).
+If hitting `the number of matching timeseries exceeds...` error, adjust filters to match less time series or
+update `-search.maxSeries` command-line flag on vmselect/vmsingle;
+
+Migrating all the metrics from one VM to another may collide with existing application metrics
+(prefixed with `vm_`) at destination and lead to confusion when using
+[official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards).
+To avoid such situation try to filter out VM process metrics via `--vm-native-filter-match='{__name__!~"vm_.*"}'` flag.
+
+Migrating data with overlapping time range or via unstable network can produce duplicates series at destination.
+To avoid duplicates set `-dedup.minScrapeInterval=1ms` for `vmselect`/`vmstorage` at the destination.
+This will instruct `vmselect`/`vmstorage` to ignore duplicates with identical timestamps. Ignore this recommendation
+if you already have `-dedup.minScrapeInterval` set to 1ms or higher values at destination.
+ 
+When migrating data from one VM cluster to another, consider using [cluster-to-cluster mode](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/#cluster-to-cluster).
+Or manually specify addresses according to [URL format](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format):
+   ```sh
+   # Migrating from cluster specific tenantID to single
+   --vm-native-src-addr=http://<src-vmselect>:8481/select/0/prometheus
+   --vm-native-dst-addr=http://<dst-vmsingle>:8428
+    
+   # Migrating from single to cluster specific tenantID
+   --vm-native-src-addr=http://<src-vmsingle>:8428
+   --vm-native-dst-addr=http://<dst-vminsert>:8480/insert/0/prometheus
+    
+   # Migrating single to single
+   --vm-native-src-addr=http://<src-vmsingle>:8428
+   --vm-native-dst-addr=http://<dst-vmsingle>:8428
+    
+   # Migrating cluster to cluster for specific tenant ID
+   --vm-native-src-addr=http://<src-vmselect>:8481/select/0/prometheus
+   --vm-native-dst-addr=http://<dst-vminsert>:8480/insert/0/prometheus
+   ```
+
+When migrating data from VM cluster to Single-node VictoriaMetrics, vmctl will use the `/api/v1/export/native` API of the VM cluster,
+which attaches `vm_account_id` and `vm_project_id` labels to each time series. If you don't need to distinguish between tenants
+or simply want to remove these labels, try setting the `--vm-native-disable-binary-protocol` flag, which will use the `/api/v1/export` API,
+exporting and importing data in JSON format. Deduplication should be enabled at `-vm-native-src-addr` side if needed.
+
+Migrating data from VM cluster which had replication (`-replicationFactor` > 1) enabled won't produce the same amount
+of data copies for the destination database, and will result only in creating duplicates. To remove duplicates,
+destination database need to be configured with `-dedup.minScrapeInterval=1ms`. To restore the replication factor
+the destination `vminsert` component need to be configured with the according `-replicationFactor` value.
+See more about replication [here](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#replication-and-data-safety).
+
+`vmctl` supports `--vm-native-src-headers` and `--vm-native-dst-headers` to define headers sent with each request
+to the corresponding source address.
+
+`vmctl` supports `--vm-native-disable-http-keep-alive` to allow `vmctl` to use non-persistent HTTP connections to avoid
+error `use of closed network connection` when running a heavy export requests.
+
+See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
+
+See `./vmctl vm-native --help` for details and full list of flags:
+```shellhelp
+   --vm-native-filter-match value  Time series selector to match series for export. For example, select {instance!="localhost"} will match all series with "instance" label different to "localhost".
+   --vm-native-filter-time-start value  The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats
+   --vm-native-filter-time-end value    The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats
+   --vm-native-step-interval value      The time interval to split the migration into steps. For example, to migrate 1y of data with '--vm-native-step-interval=month' vmctl will execute it in 12 separate requests from the beginning of the time range to its end. To reverse the order use '--vm-native-filter-time-reverse'. Requires setting '--vm-native-filter-time-start'. Valid values are 'month','week','day','hour','minute'. (default: "month")
+   --vm-native-filter-time-reverse      Whether to reverse the order of time intervals split by '--vm-native-step-interval' cmd-line flag. When set, the migration will start from the newest to the oldest data. (default: false)
+   --vm-native-disable-http-keep-alive  Disable HTTP persistent connections for requests made to VictoriaMetrics components during export (default: false)
+   --vm-native-src-addr value           VictoriaMetrics address to perform export from. 
+   --vm-native-src-user value      VictoriaMetrics username for basic auth [$VM_NATIVE_SRC_USERNAME]
+   --vm-native-src-password value  VictoriaMetrics password for basic auth [$VM_NATIVE_SRC_PASSWORD]
+   --vm-native-src-headers value   Optional HTTP headers to send with each request to the corresponding source address. 
+2025/06/24 11:50:41 Total time: 1.551792ms
+      For example, --vm-native-src-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding source address. 
+      Multiple headers must be delimited by '^^': --vm-native-src-headers='header1:value1^^header2:value2'
+   --vm-native-src-bearer-token --vm-native-src-addr          Optional bearer auth token to use for the corresponding --vm-native-src-addr
+   --vm-native-src-cert-file --vm-native-src-addr             Optional path to client-side TLS certificate file to use when connecting to --vm-native-src-addr
+   --vm-native-src-key-file --vm-native-src-addr              Optional path to client-side TLS key to use when connecting to --vm-native-src-addr
+   --vm-native-src-ca-file --vm-native-src-addr               Optional path to TLS CA file to use for verifying connections to --vm-native-src-addr. By default, system CA is used
+   --vm-native-src-server-name --vm-native-src-addr           Optional TLS server name to use for connections to --vm-native-src-addr. By default, the server name from `--vm-native-src-addr` is used
+   --vm-native-src-insecure-skip-verify --vm-native-src-addr  Whether to skip TLS certificate verification when connecting to --vm-native-src-addr (default: false)
+   --vm-native-dst-addr value                                 VictoriaMetrics address to perform import to. 
+   --vm-native-dst-user value      VictoriaMetrics username for basic auth [$VM_NATIVE_DST_USERNAME]
+   --vm-native-dst-password value  VictoriaMetrics password for basic auth [$VM_NATIVE_DST_PASSWORD]
+   --vm-native-dst-headers value   Optional HTTP headers to send with each request to the corresponding destination address. 
+      For example, --vm-native-dst-headers='My-Auth:foobar' would send 'My-Auth: foobar' HTTP header with every request to the corresponding destination address. 
+      Multiple headers must be delimited by '^^': --vm-native-dst-headers='header1:value1^^header2:value2'
+   --vm-native-dst-bearer-token --vm-native-dst-addr          Optional bearer auth token to use for the corresponding --vm-native-dst-addr
+   --vm-native-dst-cert-file --vm-native-dst-addr             Optional path to client-side TLS certificate file to use when connecting to --vm-native-dst-addr
+   --vm-native-dst-key-file --vm-native-dst-addr              Optional path to client-side TLS key to use when connecting to --vm-native-dst-addr
+   --vm-native-dst-ca-file --vm-native-dst-addr               Optional path to TLS CA file to use for verifying connections to --vm-native-dst-addr. By default, system CA is used
+   --vm-native-dst-server-name --vm-native-dst-addr           Optional TLS server name to use for connections to --vm-native-dst-addr. By default, the server name from `--vm-native-dst-addr` is used
+   --vm-native-dst-insecure-skip-verify --vm-native-dst-addr  Whether to skip TLS certificate verification when connecting to --vm-native-dst-addr (default: false)
+       In this mode --vm-native-src-addr flag format is: 'http://vmselect:8481/'. --vm-native-dst-addr flag format is: http://vminsert:8480/. 
+   --vm-native-disable-per-metric-migration  Defines whether to disable per-metric migration and migrate all data via one connection. In this mode, vmctl makes less export/import requests, but can't provide a progress bar or retry failed requests. (default: false)
+   --vm-native-disable-binary-protocol       Whether to use https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format instead of https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format API.Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks.Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side. (default: false)
+   --vm-native-backoff-retries value         How many export/import retries to perform before giving up. (default: 10)
+   --vm-native-backoff-factor value          Factor to multiply the base duration after each failed export/import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-native-backoff-min-duration value    Minimum duration to wait before the first export/import retry. Each subsequent export/import retry will be multiplied by the '--vm-native-backoff-factor'. (default: 2s)
+```

docs/victoriametrics/vmctl/vmctl.md

@@ -0,0 +1,355 @@
+VictoriaMetrics command-line tool (**vmctl**) provides the following migration modes:
+- [Prometheus](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/) to VictoriaMetrics via [snapshot](https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot)
+- [InfluxDB](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/) to VictoriaMetrics
+- [OpenTSDB](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/) to VictoriaMetrics
+- [Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos/) to VictoriaMetrics via [snapshot](https://prometheus.io/docs/prometheus/latest/querying/api/#snapshot)
+- migrate data between [VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/) single or cluster version.
+- migrate data via [Prometheus remote read protocol](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) to VictoriaMetrics:
+    - [Thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos/#remote-read-protocol)
+    - [Cortex](https://docs.victoriametrics.com/victoriametrics/vmctl/cortex/)
+    - [Mimir](https://docs.victoriametrics.com/victoriametrics/vmctl/mimir/)
+    - [Promscale](https://docs.victoriametrics.com/victoriametrics/vmctl/promscale/)
+
+Additionally, vmctl supports [verify](#verifying-exported-blocks-from-victoriametrics) mode for exported blocks from
+VictoriaMetrics single or cluster version.
+
+## Articles 
+
+- [How to migrate data from Prometheus](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-d44a6728f043)
+- [How to migrate data from Prometheus. Filtering and modifying time series](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
+
+## Quick Start
+
+vmctl command-line tool is available as:
+* docker images at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmctl/) and [Quay](https://quay.io/repository/victoriametrics/vmctl?tab=tags)
+* [Binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) as part of `vmutils` package
+
+Download and unpack vmctl:
+```sh
+wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/vmutils-darwin-arm64-v1.120.0.tar.gz
+
+tar xzf vmutils-darwin-arm64-v1.120.0.tar.gz
+```
+
+Once binary is unpacked, see the full list of supported modes by running the following command:
+```sh
+$ ./vmctl-prod --help
+NAME:
+   vmctl - VictoriaMetrics command-line tool
+
+USAGE:
+   vmctl [global options] command [command options] [arguments...]
+
+COMMANDS:
+   opentsdb      Migrate time series from OpenTSDB
+   influx        Migrate time series from InfluxDB
+   remote-read   Migrate time series via Prometheus remote-read protocol
+   prometheus    Migrate time series from Prometheus
+   vm-native     Migrate time series between VictoriaMetrics installations
+   verify-block  Verifies exported block with VictoriaMetrics Native format
+```
+
+vmctl acts as a proxy between **source** (where to fetch data from) and **destination** (where to migrate data to).
+
+As a **source**, user needs to specify one of the migration modes (`prometheus`, `influx`, etc.). Each command has its
+own unique set of flags (e.g. prefixed with `influx-` for influx) and a common list of flags for destination (prefixed with `vm-` for VictoriaMetrics):
+
+```sh
+$ ./vmctl-prod influx --help
+OPTIONS:
+   --influx-addr value              InfluxDB server addr (default: "http://localhost:8086")
+   --influx-user value              InfluxDB user [$INFLUX_USERNAME]
+...
+   --vm-addr value        VictoriaMetrics address to perform import requests.
+   --vm-user value        VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value    VictoriaMetrics password for basic auth [$VM_PASSWORD]
+```
+
+See detailed documentation about each migration mode in the main menu under `vmctl` section.
+
+See how to [add extra labels](#adding-extra-labels) or [improve compression](#significant-figures).
+
+## Configuring VictoriaMetrics
+
+For every migration mode, user needs to specify **destination** - Victoriametrics `--vm-addr` address.
+For example:
+```sh
+./vmctl prometheus \
+  --vm-addr=<victoriametrics-addr>:8428 \
+  --prom-snapshot=/path/to/snapshot
+```
+_Replace `<victoriametrics-addr>` with the VictoriaMetrics hostname or IP address._
+
+For cluster version you need to additionally specify the `--vm-account-id` flag and use vminsert address:
+```
+http://<vminsert-addr>:8480
+```
+_Replace `<vminsert-addr>` with the hostname or IP address of vminsert service._
+
+If you have more than 1 vminsert, configure [load-balancing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup).
+
+## Migration tips
+
+Migration speed heavily depends on the following factors:
+1. Network bandwidth. Since vmctl acts as a proxy, it receives data from **source** and forward it to **destination**
+   via the network.
+2. **source** performance, or how quick source can return requested data
+3. **destination** performance, or how quick destination can accept and store data.
+
+> See the expected migration speed [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5366#issuecomment-1854251938).
+
+Importing speed can be adjusted via `--vm-concurrency` cmd-line flag, which controls the number of concurrent
+workers busy with processing. Please note, that each worker can load up to a single vCPU core on VictoriaMetrics.
+So try to set it according to allocated CPU resources of your VictoriaMetrics destination installation.
+
+Migration is a backfilling process, so it is recommended to read [Backfilling tips](https://github.com/VictoriaMetrics/VictoriaMetrics#backfilling) section.
+
+`vmctl` doesn't provide relabeling or other types of labels management. Instead, use [relabeling in VictoriaMetrics](https://github.com/VictoriaMetrics/vmctl/issues/4#issuecomment-683424375).
+
+### Importer stats
+
+After successful import `vmctl` prints some statistics for details.
+The important numbers to watch are following:
+
+- `idle duration` - shows time that importer spent while waiting for data from InfluxDB/Prometheus
+  to fill up `--vm-batch-size` batch size. Value shows total duration across all workers configured
+  via `--vm-concurrency`. High value may be a sign of too slow InfluxDB/Prometheus fetches or too
+  high `--vm-concurrency` value. Try to improve it by increasing `--<mode>-concurrency` value or
+  decreasing `--vm-concurrency` value.
+- `import requests` - shows how many import requests were issued to VM server.
+  The import request is issued once the batch size(`--vm-batch-size`) is full and ready to be sent.
+  Please prefer big batch sizes (50k-500k) to improve performance.
+- `import requests retries` - shows number of unsuccessful import requests. Non-zero value may be
+  a sign of network issues or VM being overloaded. See the logs during import for error messages.
+
+### Silent mode
+
+By default, `vmctl` waits confirmation from user before starting the import. If this is unwanted
+behavior and no user interaction required - pass `-s` flag to enable "silence" mode.
+
+Setting `-disable-progress-bar` cmd-line flag disables the progress bar during import.
+
+### Significant figures
+
+`vmctl` allows to limit the number of [significant figures](https://en.wikipedia.org/wiki/Significant_figures)
+before importing to improve data compression:
+
+- `--vm-round-digits` flag for rounding processed values to the given number of decimal digits after the point.
+  For example, `--vm-round-digits=2` would round `1.2345` to `1.23`. By default, the rounding is disabled.
+
+- `--vm-significant-figures` flag for limiting the number of significant figures in processed values. It takes no effect if set
+  to 0 (by default), but set `--vm-significant-figures=5` and `102.342305` will be rounded to `102.34`.
+
+The most common case for using these flags is to improve data compression for time series storing aggregation
+results such as `average`, `rate`, etc.
+
+See [How to migrate data from Prometheus. Filtering and modifying time series](https://medium.com/@romanhavronenko/victoriametrics-how-to-migrate-data-from-prometheus-filtering-and-modifying-time-series-6d40cea4bf21)
+for examples of compression improvement.
+
+### Adding extra labels
+
+`vmctl` allows to add extra labels to all imported series. It can be achieved with flag `--vm-extra-label label=value`.
+If multiple labels needs to be added, set flag for each label, for example, `--vm-extra-label label1=value1 --vm-extra-label label2=value2`.
+If timeseries already have label, that must be added with `--vm-extra-label` flag, flag has priority and will override label value from timeseries.
+
+### Rate limiting
+
+Limiting the rate of data transfer could help to reduce pressure on disk or on destination database.
+The rate limit may be set in bytes-per-second via `--vm-rate-limit` flag. Note that the rate limit is applied per worker,
+see `--vm-concurrency` flag.
+
+Please note, you can also use [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/)
+as a proxy between `vmctl` and destination with `-remoteWrite.rateLimit` flag enabled.
+
+## Verifying exported blocks from VictoriaMetrics
+
+In this mode, `vmctl` allows verifying correctness and integrity of data exported via
+[native format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-native-format)
+from VictoriaMetrics.
+You can verify exported data at disk before uploading it by `vmctl verify-block` command:
+
+```sh
+# export blocks from VictoriaMetrics
+curl localhost:8428/api/v1/export/native -g -d 'match[]={__name__!=""}' -o exported_data_block
+# verify block content
+./vmctl verify-block exported_data_block
+2022/03/30 18:04:50 verifying block at path="exported_data_block"
+2022/03/30 18:04:50 successfully verified block at path="exported_data_block", blockCount=123786
+2022/03/30 18:04:50 Total time: 100.108ms
+```
+
+## How to build
+
+It is recommended using [binary releases](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) - `vmctl` is located in `vmutils-*` archives there.
+
+### Development build
+
+1. [Install Go](https://golang.org/doc/install).
+1. Run `make vmctl` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl` binary and puts it into the `bin` folder.
+
+### Production build
+
+1. [Install docker](https://docs.docker.com/install/).
+1. Run `make vmctl-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl-prod` binary and puts it into the `bin` folder.
+
+### Building docker images
+
+Run `make package-vmctl`. It builds `victoriametrics/vmctl:<PKG_TAG>` docker image locally.
+`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
+The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-vmctl`.
+
+The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
+by setting it via `<ROOT_IMAGE>` environment variable. For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
+
+```sh
+ROOT_IMAGE=scratch make package-vmctl
+```
+
+### ARM build
+
+ARM build may run on Raspberry Pi or on [energy-efficient ARM servers](https://blog.cloudflare.com/arm-takes-wing/).
+
+#### Development ARM build
+
+1. [Install Go](https://golang.org/doc/install).
+1. Run `make vmctl-linux-arm` or `make vmctl-linux-arm64` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl-linux-arm` or `vmctl-linux-arm64` binary respectively and puts it into the `bin` folder.
+
+#### Production ARM build
+
+1. [Install docker](https://docs.docker.com/install/).
+1. Run `make vmctl-linux-arm-prod` or `make vmctl-linux-arm64-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaMetrics).
+   It builds `vmctl-linux-arm-prod` or `vmctl-linux-arm64-prod` binary respectively and puts it into the `bin` folder.
+
+## Command-line flags
+
+Run `vmctl -help` in order to see all the available options.
+
+Commands:
+```shellhelp
+  influx
+     Migrate time series from InfluxDB
+  opentsdb
+     Migrate time series from OpenTSDB.
+  prometheus
+     Migrate time series from Prometheus.
+  remote-read
+     Migrate time series via Prometheus remote-read protocol.
+  verify-block
+     Verifies exported block with VictoriaMetrics Native format.
+  vm-native
+     Migrate time series between VictoriaMetrics installations.
+```
+
+Flags available for all commands:
+
+```shellhelp
+  -s
+     Whether to run in silent mode. If set to true no confirmation prompts will appear. (default false)
+  -verbose
+     Whether to enable verbosity in logs output. (default false)
+  -disable-progress-bar
+     Whether to disable progress bar during the import. (default false)
+     
+   --vm-addr vmctl
+     VictoriaMetrics address to perform import requests. 
+     Should be the same as --httpListenAddr value for single-node version or vminsert component. 
+     When importing into the clustered version do not forget to set additionally --vm-account-id flag. 
+     Please note, that vmctl performs initial readiness check for the given address by checking `/health` endpoint. (default: "http://localhost:8428")
+   --vm-user value
+     VictoriaMetrics username for basic auth [$VM_USERNAME]
+   --vm-password value
+     VictoriaMetrics password for basic auth [$VM_PASSWORD]
+   --vm-account-id value
+     AccountID is an arbitrary 32-bit integer identifying namespace for data ingestion (aka tenant). 
+     AccountID is required when importing into the clustered version of VictoriaMetrics. 
+     It is possible to set it as accountID:projectID, where projectID is also arbitrary 32-bit integer. 
+     If projectID isn't set, then it equals to 0
+   --vm-concurrency value
+     Number of workers concurrently performing import requests to VM (default: 2)
+   --vm-compress
+     Whether to apply gzip compression to import requests (default: true)
+   --vm-batch-size value
+     How many samples importer collects before sending the import request to VM (default: 200000)
+   --vm-significant-figures value
+     The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures.
+     Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics.
+     See also --vm-round-digits option (default: 0)
+   --vm-round-digits value
+     Round metric values to the given number of decimal digits after the point. This option may be used for increasing 
+     on-disk compression level for the stored metrics (default: 100)
+   --vm-extra-label value [ --vm-extra-label value ]
+     Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flagwill have priority.
+     Flag can be set multiple times, to add few additional labels.
+   --vm-rate-limit value
+     Optional data transfer rate limit in bytes per second.
+     By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vmAddr' destination. (default: 0)
+   --vm-cert-file value
+     Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'
+   --vm-key-file value
+     Optional path to client-side TLS key to use when connecting to '--vmAddr'
+   --vm-CA-file value
+     Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used
+   --vm-server-name value
+     Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used
+   --vm-insecure-skip-verify
+     Whether to skip tls verification when connecting to '--vmAddr' (default: false)
+   --vm-backoff-retries value
+     How many import retries to perform before giving up. (default: 10)
+   --vm-backoff-factor value
+     Factor to multiply the base duration after each failed import retry. Must be greater than 1.0 (default: 1.8)
+   --vm-backoff-min-duration value
+     Minimum duration to wait before the first import retry. Each subsequent import retry will be multiplied by the '--vm-backoff-factor'. (default: 2s)
+```
+
+See list of available cmd-line flags for each command in the corresponding section.
+
+---
+
+Section below contains backward-compatible anchors for links that were moved or renamed.
+
+###### Migrating data from Prometheus
+
+Moved to [vmctl/prometheus](https://docs.victoriametrics.com/victoriametrics/vmctl/prometheus/).
+
+###### Migrating data from InfluxDB (1.x)
+
+Moved to [vmctl/influx](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/).
+
+###### Migrating data from InfluxDB (2.x)
+
+Moved to [vmctl/influx](https://docs.victoriametrics.com/victoriametrics/vmctl/influxdb/#influxdb-v2).
+
+###### Migrating data from OpenTSDB
+
+Moved to [vmctl/opentsdb](https://docs.victoriametrics.com/victoriametrics/vmctl/opentsdb/).
+
+###### Migrating data from Thanos
+
+Moved to [vmctl/thanos](https://docs.victoriametrics.com/victoriametrics/vmctl/thanos/).
+
+###### Migrating data by remote read protocol
+
+Moved to [vmctl/remoteread](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/).
+
+###### Migrating data from Cortex
+
+Moved to [vmctl/cortex](https://docs.victoriametrics.com/victoriametrics/vmctl/cortex/).
+
+###### Migrating data from Promscale
+
+Moved to [vmctl/promscale](https://docs.victoriametrics.com/victoriametrics/vmctl/promscale/).
+
+###### Migrating data from Mimir
+
+Moved to [vmctl/mimir](https://docs.victoriametrics.com/victoriametrics/vmctl/mimir/).
+
+###### Migrating data from VictoriaMetrics
+
+Moved to [vmctl/victoriametrics](https://docs.victoriametrics.com/victoriametrics/vmctl/victoriametrics/).
+
+###### Tuning
+
+Moved to [vmctl#migration-tips](https://docs.victoriametrics.com/victoriametrics/vmctl#migration-tips).

lib/logstorage/block_search.go

@@ -2,6 +2,7 @@ package logstorage
 
 import (
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -141,6 +142,9 @@ type blockSearch struct {
 	// It is initialized lazily by calling getColumnsHeader().
 	cshCache *columnsHeader
 
+	// bytesReadFromDisk tracks the total bytes read from disk files for this block search
+	bytesReadFromDisk atomic.Uint64
+
 	// seenStreams contains seen streamIDs for the recent searches.
 	//
 	// It is used for speeding up fetching _stream column.
@@ -197,6 +201,9 @@ func (bs *blockSearch) reset() {
 		bs.cshCache = nil
 	}
 
+	// Reset bytes read from disk counter
+	bs.bytesReadFromDisk.Store(0)
+
 	// Do not reset seenStreams, since its' lifetime is managed by blockResult.addStreamColumn() code.
 }
 
@@ -344,6 +351,9 @@ func (bs *blockSearch) getColumnsHeaderIndex() *columnsHeaderIndex {
 	if bs.cshIndexCache == nil {
 		bs.cshIndexBlockCache = readColumnsHeaderIndexBlock(bs.cshIndexBlockCache[:0], bs.bsw.p, &bs.bsw.bh)
 
+		// Track bytes read from disk
+		bs.bytesReadFromDisk.Add(uint64(len(bs.cshIndexBlockCache)))
+
 		bs.cshIndexCache = getColumnsHeaderIndex()
 		if err := bs.cshIndexCache.unmarshalInplace(bs.cshIndexBlockCache); err != nil {
 			logger.Panicf("FATAL: %s: cannot unmarshal columns header index: %s", bs.bsw.p.path, err)
@@ -376,6 +386,10 @@ func (bs *blockSearch) getColumnsHeader() *columnsHeader {
 func (bs *blockSearch) getColumnsHeaderBlock() []byte {
 	if !bs.cshBlockInitialized {
 		bs.cshBlockCache = readColumnsHeaderBlock(bs.cshBlockCache[:0], bs.bsw.p, &bs.bsw.bh)
+
+		// Track bytes read from disk
+		bs.bytesReadFromDisk.Add(uint64(len(bs.cshBlockCache)))
+
 		bs.cshBlockInitialized = true
 	}
 	return bs.cshBlockCache
@@ -425,6 +439,10 @@ func (bs *blockSearch) getBloomFilterForColumn(ch *columnHeader) *bloomFilter {
 	bb.B = bytesutil.ResizeNoCopyMayOverallocate(bb.B, int(bloomFilterSize))
 
 	bloomValuesFile.bloom.MustReadAt(bb.B, int64(ch.bloomFilterOffset))
+
+	// Track bytes read from disk
+	bs.bytesReadFromDisk.Add(uint64(len(bb.B)))
+
 	bf = getBloomFilter()
 	if err := bf.unmarshal(bb.B); err != nil {
 		logger.Panicf("FATAL: %s: cannot unmarshal bloom filter: %s", bs.partPath(), err)
@@ -458,6 +476,9 @@ func (bs *blockSearch) getValuesForColumn(ch *columnHeader) []string {
 	bb.B = bytesutil.ResizeNoCopyMayOverallocate(bb.B, int(valuesSize))
 	bloomValuesFile.values.MustReadAt(bb.B, int64(ch.valuesOffset))
 
+	// Track bytes read from disk
+	bs.bytesReadFromDisk.Add(uint64(len(bb.B)))
+
 	values = getStringBucket()
 	var err error
 	values.a, err = bs.sbu.unmarshal(values.a[:0], bb.B, bs.bsw.bh.rowsCount)
@@ -493,6 +514,9 @@ func (bs *blockSearch) getTimestamps() []int64 {
 	bb.B = bytesutil.ResizeNoCopyMayOverallocate(bb.B, int(blockSize))
 	p.timestampsFile.MustReadAt(bb.B, int64(th.blockOffset))
 
+	// Track bytes read from disk
+	bs.bytesReadFromDisk.Add(uint64(len(bb.B)))
+
 	rowsCount := int(bs.bsw.bh.rowsCount)
 	timestamps = encoding.GetInt64s(rowsCount)
 	var err error

lib/logstorage/datadb.go

@@ -16,6 +16,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 // The maximum size of big part.
@@ -54,14 +55,25 @@ type datadb struct {
 	// mergeIdx is used for generating unique directory names for parts
 	mergeIdx atomic.Uint64
 
-	inmemoryMergesTotal  atomic.Uint64
-	inmemoryActiveMerges atomic.Int64
+	inmemoryMergesTotal    atomic.Uint64
+	inmemoryActiveMerges   atomic.Int64
+	inmemoryMergeRowsTotal atomic.Uint64
 
-	smallPartMergesTotal  atomic.Uint64
-	smallPartActiveMerges atomic.Int64
+	smallPartMergesTotal    atomic.Uint64
+	smallPartActiveMerges   atomic.Int64
+	smallPartMergeRowsTotal atomic.Uint64
 
-	bigPartMergesTotal  atomic.Uint64
-	bigPartActiveMerges atomic.Int64
+	bigPartMergesTotal    atomic.Uint64
+	bigPartActiveMerges   atomic.Int64
+	bigPartMergeRowsTotal atomic.Uint64
+
+	// metrics that need to be updated directly
+	inmemoryPartMergeDuration *metrics.Summary
+	inmemoryPartMergeBytes    *metrics.Summary
+	smallPartMergeDuration    *metrics.Summary
+	smallPartMergeBytes       *metrics.Summary
+	bigPartMergeDuration      *metrics.Summary
+	bigPartMergeBytes         *metrics.Summary
 
 	// pt is the partition the datadb belongs to
 	pt *partition
@@ -188,10 +200,18 @@ func mustOpenDatadb(pt *partition, path string, flushInterval time.Duration) *da
 	ddb := &datadb{
 		pt:            pt,
 		flushInterval: flushInterval,
-		path:          path,
-		smallParts:    smallParts,
-		bigParts:      bigParts,
-		stopCh:        make(chan struct{}),
+
+		inmemoryPartMergeDuration: metrics.GetOrCreateSummary(`vl_merge_duration_seconds{type="storage/inmemory"}`),
+		inmemoryPartMergeBytes:    metrics.GetOrCreateSummary(`vl_merge_bytes{type="storage/inmemory"}`),
+		smallPartMergeDuration:    metrics.GetOrCreateSummary(`vl_merge_duration_seconds{type="storage/small"}`),
+		smallPartMergeBytes:       metrics.GetOrCreateSummary(`vl_merge_bytes{type="storage/small"}`),
+		bigPartMergeDuration:      metrics.GetOrCreateSummary(`vl_merge_duration_seconds{type="storage/big"}`),
+		bigPartMergeBytes:         metrics.GetOrCreateSummary(`vl_merge_bytes{type="storage/big"}`),
+
+		path:       path,
+		smallParts: smallParts,
+		bigParts:   bigParts,
+		stopCh:     make(chan struct{}),
 	}
 	ddb.rb.init(&ddb.wg, ddb.mustFlushLogRows)
 	ddb.mergeIdx.Store(uint64(time.Now().UnixNano()))
@@ -593,6 +613,21 @@ func (ddb *datadb) mustMergeParts(pws []*partWrapper, isFinal bool) {
 
 	ddb.swapSrcWithDstParts(pws, pwNew, dstPartType)
 
+	switch dstPartType {
+	case partInmemory:
+		ddb.inmemoryMergeRowsTotal.Add(srcRowsCount)
+		ddb.inmemoryPartMergeDuration.UpdateDuration(startTime)
+		ddb.inmemoryPartMergeBytes.Update(float64(dstSize))
+	case partSmall:
+		ddb.smallPartMergeRowsTotal.Add(srcRowsCount)
+		ddb.smallPartMergeDuration.UpdateDuration(startTime)
+		ddb.smallPartMergeBytes.Update(float64(dstSize))
+	case partBig:
+		ddb.bigPartMergeRowsTotal.Add(srcRowsCount)
+		ddb.bigPartMergeDuration.UpdateDuration(startTime)
+		ddb.bigPartMergeBytes.Update(float64(dstSize))
+	}
+
 	d := time.Since(startTime)
 	if d <= time.Minute {
 		return
@@ -673,6 +708,21 @@ type rowsBuffer struct {
 	nextIdx atomic.Uint64
 }
 
+func (rb *rowsBuffer) Len() uint64 {
+	shards := rb.shards
+	n := uint64(0)
+	for i := range shards {
+		shard := &shards[i]
+		shard.mu.Lock()
+		if shard.lr != nil {
+			n += uint64(shard.lr.Len())
+		}
+		shard.mu.Unlock()
+	}
+
+	return n
+}
+
 func (rb *rowsBuffer) init(wg *sync.WaitGroup, flushFunc func(lr *logRows)) {
 	shards := make([]rowsBufferShard, cgroup.AvailableCPUs())
 	for i := range shards {
@@ -684,7 +734,7 @@ func (rb *rowsBuffer) init(wg *sync.WaitGroup, flushFunc func(lr *logRows)) {
 }
 
 type rowsBufferShard struct {
-	wg        *sync.WaitGroup
+	wg        *sync.WaitGroup // wg is shared with datadb.
 	flushFunc func(lr *logRows)
 
 	mu         sync.Mutex
@@ -774,18 +824,30 @@ type DatadbStats struct {
 	// InmemoryActiveMerges is the number of currently active inmemory merges performed by the given datadb.
 	InmemoryActiveMerges uint64
 
+	// InmemoryMergeRowsTotal is the number of rows merged to inmemory parts.
+	InmemoryMergeRowsTotal uint64
+
 	// SmallPartMergesTotal is the number of small file merges performed in the given datadb.
 	SmallPartMergesTotal uint64
 
 	// SmallPartActiveMerges is the number of currently active small file merges performed by the given datadb.
 	SmallPartActiveMerges uint64
 
+	// SmallPartMergeRowsTotal is the number of rows merged to small parts.
+	SmallPartMergeRowsTotal uint64
+
 	// BigPartMergesTotal is the number of big file merges performed in the given datadb.
 	BigPartMergesTotal uint64
 
 	// BigPartActiveMerges is the number of currently active big file merges performed by the given datadb.
 	BigPartActiveMerges uint64
 
+	// BigPartMergeRowsTotal is the number of rows merged to big parts.
+	BigPartMergeRowsTotal uint64
+
+	// PendingRows is the number of rows, which weren't flushed to searchable part yet.
+	PendingRowsCount uint64
+
 	// InmemoryRowsCount is the number of rows, which weren't flushed to disk yet.
 	InmemoryRowsCount uint64
 
@@ -845,10 +907,15 @@ func (s *DatadbStats) RowsCount() uint64 {
 func (ddb *datadb) updateStats(s *DatadbStats) {
 	s.InmemoryMergesTotal += ddb.inmemoryMergesTotal.Load()
 	s.InmemoryActiveMerges += uint64(ddb.inmemoryActiveMerges.Load())
+	s.InmemoryMergeRowsTotal += ddb.inmemoryMergeRowsTotal.Load()
 	s.SmallPartMergesTotal += ddb.smallPartMergesTotal.Load()
 	s.SmallPartActiveMerges += uint64(ddb.smallPartActiveMerges.Load())
+	s.SmallPartMergeRowsTotal += ddb.smallPartMergeRowsTotal.Load()
 	s.BigPartMergesTotal += ddb.bigPartMergesTotal.Load()
 	s.BigPartActiveMerges += uint64(ddb.bigPartActiveMerges.Load())
+	s.BigPartMergeRowsTotal += ddb.bigPartMergeRowsTotal.Load()
+
+	s.PendingRowsCount = ddb.rb.Len()
 
 	ddb.partsLock.Lock()
 

lib/logstorage/filter_test.go

@@ -206,7 +206,7 @@ func testFilterMatchForStorage(t *testing.T, s *Storage, tenantID TenantID, f fi
 	var results []result
 
 	const workersCount = 3
-	s.search(workersCount, so, nil, func(_ uint, br *blockResult) {
+	s.search(workersCount, &searchStats{}, so, nil, func(_ uint, br *blockResult) {
 		// Verify columns
 		cs := br.getColumns()
 		if len(cs) != 2 {

lib/logstorage/indexdb.go

@@ -46,6 +46,27 @@ type IndexdbStats struct {
 
 	// IndexdbPartsCount is the number of parts in indexdb.
 	IndexdbPartsCount uint64
+
+	// IndexdbPendingItems is the number of pending items in IndexedDB before they are merged into the part.
+	IndexdbPendingItems uint64
+
+	// IndexdbActiveFileMerges is the number of active merges in indexdb.
+	IndexdbActiveFileMerges uint64
+
+	// IndexdbActiveInmemoryMerges is the number of active merges in indexdb.
+	IndexdbActiveInmemoryMerges uint64
+
+	// IndexdbFileMergesTotal is the number of merges in indexdb.
+	IndexdbFileMergesTotal uint64
+
+	// IndexdbInmemoryMergesTotal is the number of merges in indexdb.
+	IndexdbInmemoryMergesTotal uint64
+
+	// IndexdbFileItemsMerged is the number of items merged in indexdb.
+	IndexdbFileItemsMerged uint64
+
+	// IndexdbInmemoryItemsMerged is the number of items merged in indexdb.
+	IndexdbInmemoryItemsMerged uint64
 }
 
 type indexdb struct {
@@ -107,8 +128,15 @@ func (idb *indexdb) updateStats(d *IndexdbStats) {
 
 	d.IndexdbSizeBytes += tm.InmemorySizeBytes + tm.FileSizeBytes
 	d.IndexdbItemsCount += tm.InmemoryItemsCount + tm.FileItemsCount
+	d.IndexdbPendingItems += tm.PendingItems
 	d.IndexdbPartsCount += tm.InmemoryPartsCount + tm.FilePartsCount
 	d.IndexdbBlocksCount += tm.InmemoryBlocksCount + tm.FileBlocksCount
+	d.IndexdbActiveFileMerges = tm.ActiveFileMerges
+	d.IndexdbActiveInmemoryMerges = tm.ActiveInmemoryMerges
+	d.IndexdbFileMergesTotal += tm.FileMergesCount
+	d.IndexdbInmemoryMergesTotal += tm.InmemoryMergesCount
+	d.IndexdbFileItemsMerged += tm.FileItemsMerged
+	d.IndexdbInmemoryItemsMerged += tm.InmemoryItemsMerged
 }
 
 func (idb *indexdb) appendStreamTagsByStreamID(dst []byte, sid *streamID) []byte {

lib/logstorage/parser.go

@@ -255,6 +255,8 @@ type Query struct {
 
 	// timestamp is the timestamp context used for parsing the query.
 	timestamp int64
+
+	searchStats *searchStats
 }
 
 type queryOptions struct {
@@ -1281,9 +1283,10 @@ func parseQuery(lex *lexer) (*Query, error) {
 		return nil, fmt.Errorf("%w; context: [%s]", err, lex.context())
 	}
 	q := &Query{
-		opts:      opts,
-		f:         f,
-		timestamp: lex.currentTimestamp,
+		opts:        opts,
+		f:           f,
+		timestamp:   lex.currentTimestamp,
+		searchStats: &searchStats{},
 	}
 
 	if lex.isKeyword("|") {

lib/logstorage/partition.go

@@ -8,6 +8,11 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	streamIDCacheHitRate = metrics.NewSummary(`vl_cache_hit_rate{type="stream_id"}`)
 )
 
 // PartitionStats contains stats for the partition.
@@ -118,18 +123,23 @@ func mustClosePartition(pt *partition) {
 }
 
 func (pt *partition) mustAddRows(lr *LogRows) {
+	var cacheMisses uint64
+	var cacheHits uint64
+
 	// Register rows in indexdb
 	var pendingRows []int
 	streamIDs := lr.streamIDs
 	for i := range lr.timestamps {
 		streamID := &streamIDs[i]
 		if pt.hasStreamIDInCache(streamID) {
+			cacheHits++
 			continue
 		}
 		if len(pendingRows) == 0 || !streamIDs[pendingRows[len(pendingRows)-1]].equal(streamID) {
 			pendingRows = append(pendingRows, i)
 		}
 	}
+
 	if len(pendingRows) > 0 {
 		logNewStreams := pt.s.logNewStreams
 		streamTagsCanonicals := lr.streamTagsCanonicals
@@ -142,11 +152,14 @@ func (pt *partition) mustAddRows(lr *LogRows) {
 				continue
 			}
 			if pt.hasStreamIDInCache(streamID) {
+				cacheHits++
 				continue
 			}
+			cacheMisses++
 			if !pt.idb.hasStreamID(streamID) {
 				streamTagsCanonical := streamTagsCanonicals[rowIdx]
 				pt.idb.mustRegisterStream(streamID, streamTagsCanonical)
+
 				if logNewStreams {
 					pt.logNewStream(streamTagsCanonical, lr.rows[rowIdx])
 				}
@@ -155,6 +168,8 @@ func (pt *partition) mustAddRows(lr *LogRows) {
 		}
 	}
 
+	streamIDCacheHitRate.Update(float64(cacheHits) / float64(cacheHits+cacheMisses))
+
 	// Add rows to datadb
 	pt.ddb.mustAddRows(lr)
 	if pt.s.logIngestedRows {

lib/logstorage/storage.go

@@ -17,13 +17,13 @@ import (
 
 // StorageStats represents stats for the storage. It may be obtained by calling Storage.UpdateStats().
 type StorageStats struct {
-	// RowsDroppedTooBigTimestamp is the number of rows dropped during data ingestion because their timestamp is smaller than the minimum allowed
+	// RowsDroppedTooBigTimestamp is the number of rows dropped during data ingestion because their timestamp is smaller than the minimum allowed.
 	RowsDroppedTooBigTimestamp uint64
 
-	// RowsDroppedTooSmallTimestamp is the number of rows dropped during data ingestion because their timestamp is bigger than the maximum allowed
+	// RowsDroppedTooSmallTimestamp is the number of rows dropped during data ingestion because their timestamp is bigger than the maximum allowed.
 	RowsDroppedTooSmallTimestamp uint64
 
-	// PartitionsCount is the number of partitions in the storage
+	// PartitionsCount is the number of partitions in the storage.
 	PartitionsCount uint64
 
 	// IsReadOnly indicates whether the storage is read-only.

lib/logstorage/storage_search.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -16,6 +17,14 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prefixfilter"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+var (
+	rowsPerQuery        = metrics.NewHistogram(`vl_storage_rows_read_per_query`)
+	bytesPerQuery       = metrics.NewHistogram(`vl_storage_bytes_read_per_query`)
+	blocksPerQuery      = metrics.NewHistogram(`vl_storage_blocks_read_per_query`)
+	streamsUsedPerQuery = metrics.NewHistogram(`vl_storage_streams_used_per_query`)
 )
 
 // genericSearchOptions contain options used for search.
@@ -100,7 +109,16 @@ func (f writeBlockResultFunc) newDataBlockWriter() WriteDataBlockFunc {
 // RunQuery runs the given q and calls writeBlock for results.
 func (s *Storage) RunQuery(ctx context.Context, tenantIDs []TenantID, q *Query, writeBlock WriteDataBlockFunc) error {
 	writeBlockResult := writeBlock.newBlockResultWriter()
-	return s.runQuery(ctx, tenantIDs, q, writeBlockResult)
+
+	err := s.runQuery(ctx, tenantIDs, q, writeBlockResult)
+
+	// Update metrics regardless of the error
+	bytesPerQuery.Update(float64(q.searchStats.totalBytesFromDisk.Load()))
+	rowsPerQuery.Update(float64(q.searchStats.totalRows.Load()))
+	blocksPerQuery.Update(float64(q.searchStats.totalBlocks.Load()))
+	streamsUsedPerQuery.Update(float64(q.searchStats.fetchStreams.Load()))
+
+	return err
 }
 
 // runQueryFunc must run the given q and pass query results to writeBlock
@@ -133,7 +151,7 @@ func (s *Storage) runQuery(ctx context.Context, tenantIDs []TenantID, q *Query,
 	workersCount := q.GetConcurrency()
 
 	search := func(stopCh <-chan struct{}, writeBlockToPipes writeBlockResultFunc) error {
-		s.search(workersCount, so, stopCh, writeBlockToPipes)
+		s.search(workersCount, q.searchStats, so, stopCh, writeBlockToPipes)
 		return nil
 	}
 
@@ -1031,13 +1049,17 @@ func (db *DataBlock) initFromBlockResult(br *blockResult) {
 // search searches for the matching rows according to so.
 //
 // It calls writeBlock for each matching block.
-func (s *Storage) search(workersCount int, so *genericSearchOptions, stopCh <-chan struct{}, writeBlock writeBlockResultFunc) {
+func (s *Storage) search(workersCount int, ss *searchStats, so *genericSearchOptions, stopCh <-chan struct{}, writeBlock writeBlockResultFunc) {
 	// Spin up workers
 	var wgWorkers sync.WaitGroup
 	workCh := make(chan *blockSearchWorkBatch, workersCount)
 	wgWorkers.Add(workersCount)
+
 	for i := 0; i < workersCount; i++ {
 		go func(workerID uint) {
+			var totalBytesFromDisk uint64
+			var totalRows, totalBlocks int
+
 			bs := getBlockSearch()
 			bm := getBitmap(0)
 			for bswb := range workCh {
@@ -1051,6 +1073,10 @@ func (s *Storage) search(workersCount int, so *genericSearchOptions, stopCh <-ch
 					}
 
 					bs.search(bsw, bm)
+					totalBytesFromDisk += bs.bytesReadFromDisk.Load()
+					totalRows += bs.br.rowsLen
+					totalBlocks++
+
 					if bs.br.rowsLen > 0 {
 						writeBlock(workerID, &bs.br)
 					}
@@ -1059,6 +1085,11 @@ func (s *Storage) search(workersCount int, so *genericSearchOptions, stopCh <-ch
 				bswb.bsws = bswb.bsws[:0]
 				putBlockSearchWorkBatch(bswb)
 			}
+
+			ss.totalBytesFromDisk.Add(totalBytesFromDisk)
+			ss.totalRows.Add(uint64(totalRows))
+			ss.totalBlocks.Add(uint64(totalBlocks))
+
 			putBlockSearch(bs)
 			putBitmap(bm)
 			wgWorkers.Done()
@@ -1097,7 +1128,7 @@ func (s *Storage) search(workersCount int, so *genericSearchOptions, stopCh <-ch
 		partitionSearchConcurrencyLimitCh <- struct{}{}
 		wgSearchers.Add(1)
 		go func(idx int, pt *partition) {
-			psfs[idx] = pt.search(sf, f, so, workCh, stopCh)
+			psfs[idx] = pt.search(sf, ss, f, so, workCh, stopCh)
 			wgSearchers.Done()
 			<-partitionSearchConcurrencyLimitCh
 		}(i, ptw.pt)
@@ -1126,7 +1157,14 @@ var partitionSearchConcurrencyLimitCh = make(chan struct{}, cgroup.AvailableCPUs
 
 type partitionSearchFinalizer func()
 
-func (pt *partition) search(sf *StreamFilter, f filter, so *genericSearchOptions, workCh chan<- *blockSearchWorkBatch, stopCh <-chan struct{}) partitionSearchFinalizer {
+type searchStats struct {
+	fetchStreams       atomic.Uint64
+	totalBytesFromDisk atomic.Uint64
+	totalRows          atomic.Uint64
+	totalBlocks        atomic.Uint64
+}
+
+func (pt *partition) search(sf *StreamFilter, ss *searchStats, f filter, so *genericSearchOptions, workCh chan<- *blockSearchWorkBatch, stopCh <-chan struct{}) partitionSearchFinalizer {
 	if needStop(stopCh) {
 		// Do not spend CPU time on search, since it is already stopped.
 		return func() {}
@@ -1144,6 +1182,8 @@ func (pt *partition) search(sf *StreamFilter, f filter, so *genericSearchOptions
 		streamIDs = getStreamIDsForTenantIDs(so.streamIDs, tenantIDs)
 		tenantIDs = nil
 	}
+	ss.fetchStreams.Add(uint64(len(streamIDs)))
+
 	if hasStreamFilters(f) {
 		f = initStreamFilters(so.tenantIDs, pt.idb, f)
 	}

lib/logstorage/storage_search_test.go

@@ -937,7 +937,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, _ *blockResult) {
 			panic(fmt.Errorf("unexpected match"))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 	})
 	t.Run("missing-tenant-bigger-than-existing", func(_ *testing.T) {
 		tenantID := TenantID{
@@ -951,7 +951,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, _ *blockResult) {
 			panic(fmt.Errorf("unexpected match"))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 	})
 	t.Run("missing-tenant-middle", func(_ *testing.T) {
 		tenantID := TenantID{
@@ -965,7 +965,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, _ *blockResult) {
 			panic(fmt.Errorf("unexpected match"))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 	})
 	t.Run("matching-tenant-id", func(t *testing.T) {
 		for i := 0; i < tenantsCount; i++ {
@@ -981,7 +981,7 @@ func TestStorageSearch(t *testing.T) {
 			processBlock := func(_ uint, br *blockResult) {
 				rowsCountTotal.Add(uint32(br.rowsLen))
 			}
-			s.search(workersCount, so, nil, processBlock)
+			s.search(workersCount, &searchStats{}, so, nil, processBlock)
 
 			expectedRowsCount := streamsPerTenant * blocksPerStream * rowsPerBlock
 			if n := rowsCountTotal.Load(); n != uint32(expectedRowsCount) {
@@ -998,7 +998,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, br *blockResult) {
 			rowsCountTotal.Add(uint32(br.rowsLen))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 
 		expectedRowsCount := tenantsCount * streamsPerTenant * blocksPerStream * rowsPerBlock
 		if n := rowsCountTotal.Load(); n != uint32(expectedRowsCount) {
@@ -1014,7 +1014,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, _ *blockResult) {
 			panic(fmt.Errorf("unexpected match"))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 	})
 	t.Run("matching-stream-id", func(t *testing.T) {
 		for i := 0; i < streamsPerTenant; i++ {
@@ -1031,7 +1031,7 @@ func TestStorageSearch(t *testing.T) {
 			processBlock := func(_ uint, br *blockResult) {
 				rowsCountTotal.Add(uint32(br.rowsLen))
 			}
-			s.search(workersCount, so, nil, processBlock)
+			s.search(workersCount, &searchStats{}, so, nil, processBlock)
 
 			expectedRowsCount := blocksPerStream * rowsPerBlock
 			if n := rowsCountTotal.Load(); n != uint32(expectedRowsCount) {
@@ -1053,7 +1053,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, br *blockResult) {
 			rowsCountTotal.Add(uint32(br.rowsLen))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 
 		expectedRowsCount := streamsPerTenant * blocksPerStream * rowsPerBlock
 		if n := rowsCountTotal.Load(); n != uint32(expectedRowsCount) {
@@ -1083,7 +1083,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, br *blockResult) {
 			rowsCountTotal.Add(uint32(br.rowsLen))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 
 		expectedRowsCount := streamsPerTenant * blocksPerStream * 2
 		if n := rowsCountTotal.Load(); n != uint32(expectedRowsCount) {
@@ -1104,7 +1104,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, br *blockResult) {
 			rowsCountTotal.Add(uint32(br.rowsLen))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 
 		expectedRowsCount := blocksPerStream
 		if n := rowsCountTotal.Load(); n != uint32(expectedRowsCount) {
@@ -1124,7 +1124,7 @@ func TestStorageSearch(t *testing.T) {
 		processBlock := func(_ uint, _ *blockResult) {
 			panic(fmt.Errorf("unexpected match"))
 		}
-		s.search(workersCount, so, nil, processBlock)
+		s.search(workersCount, &searchStats{}, so, nil, processBlock)
 	})
 
 	s.MustClose()

lib/prompb/prompb.go

@@ -11,6 +11,9 @@ type WriteRequest struct {
 	// Timeseries is a list of time series in the given WriteRequest
 	Timeseries []TimeSeries
 
+	// Metadata is a list of metadata info in the given WriteRequest
+	Metadata []MetricMetadata
+
 	labelsPool  []Label
 	samplesPool []Sample
 }
@@ -20,6 +23,9 @@ func (wr *WriteRequest) Reset() {
 	clear(wr.Timeseries)
 	wr.Timeseries = wr.Timeseries[:0]
 
+	clear(wr.Metadata)
+	wr.Metadata = wr.Metadata[:0]
+
 	clear(wr.labelsPool)
 	wr.labelsPool = wr.labelsPool[:0]
 
@@ -62,8 +68,11 @@ func (wr *WriteRequest) UnmarshalProtobuf(src []byte) (err error) {
 
 	// message WriteRequest {
 	//    repeated TimeSeries timeseries = 1;
+	//    reserved 2;
+	//    repeated Metadata metadata = 3;
 	// }
 	tss := wr.Timeseries
+	mds := wr.Metadata
 	labelsPool := wr.labelsPool
 	samplesPool := wr.samplesPool
 	var fc easyproto.FieldContext
@@ -88,9 +97,25 @@ func (wr *WriteRequest) UnmarshalProtobuf(src []byte) (err error) {
 			if err != nil {
 				return fmt.Errorf("cannot unmarshal timeseries: %w", err)
 			}
+		case 3:
+			data, ok := fc.MessageData()
+			if !ok {
+				return fmt.Errorf("cannot read metricMetadata data")
+			}
+			if len(mds) < cap(mds) {
+				mds = mds[:len(mds)+1]
+			} else {
+				mds = append(mds, MetricMetadata{})
+			}
+			md := &mds[len(mds)-1]
+			if err := md.unmarshalProtobuf(data); err != nil {
+				return fmt.Errorf("cannot unmarshal metricMetadata: %w", err)
+			}
+
 		}
 	}
 	wr.Timeseries = tss
+	wr.Metadata = mds
 	wr.labelsPool = labelsPool
 	wr.samplesPool = samplesPool
 	return nil
@@ -203,3 +228,68 @@ func (s *Sample) unmarshalProtobuf(src []byte) (err error) {
 	}
 	return nil
 }
+
+// MetricMetadata represents additional meta information for specific MetricFamilyName
+// Refer to https://github.com/prometheus/prometheus/blob/c5282933765ec322a0664d0a0268f8276e83b156/prompb/types.proto#L21
+type MetricMetadata struct {
+	// Represents the metric type, these match the set from Prometheus.
+	// Refer to https://github.com/prometheus/common/blob/95acce133ca2c07a966a71d475fb936fc282db18/model/metadata.go for details.
+	Type             uint32
+	MetricFamilyName string
+	Help             string
+	Unit             string
+}
+
+func (mm *MetricMetadata) unmarshalProtobuf(src []byte) (err error) {
+	// message MetricMetadata {
+	//   enum MetricType {
+	//     UNKNOWN = 0;
+	//     COUNTER = 1;
+	//     GAUGE = 2;
+	//     HISTOGRAM = 3;
+	//     GAUGEHISTOGRAM = 4;
+	//     SUMMARY = 5;
+	//     INFO = 6;
+	//     STATESET = 7;
+	//   }
+	//
+	//   MetricType type = 1;
+	//   string metric_family_name = 2;
+	//   string help = 4;
+	//   string unit = 5;
+	// }
+	var fc easyproto.FieldContext
+	for len(src) > 0 {
+		src, err = fc.NextField(src)
+		if err != nil {
+			return fmt.Errorf("cannot read the next field: %w", err)
+		}
+		switch fc.FieldNum {
+		case 1:
+			value, ok := fc.Uint32()
+			if !ok {
+				return fmt.Errorf("cannot read metric type")
+			}
+			mm.Type = value
+		case 2:
+			value, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot read metric family name")
+			}
+			mm.MetricFamilyName = value
+		case 4:
+			value, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot read help")
+			}
+			mm.Help = value
+		case 5:
+			value, ok := fc.String()
+			if !ok {
+				return fmt.Errorf("cannot read unit")
+			}
+			mm.Unit = value
+		}
+	}
+	return nil
+}

lib/prompb/prompb_test.go

@@ -41,6 +41,15 @@ func TestWriteRequestUnmarshalProtobuf(t *testing.T) {
 				Samples: samples,
 			})
 		}
+		for _, md := range wr.Metadata {
+			wrm.Metadata = append(wrm.Metadata, prompbmarshal.MetricMetadata{
+				Type:             md.Type,
+				MetricFamilyName: md.MetricFamilyName,
+				Help:             md.Help,
+				Unit:             md.Unit,
+			})
+		}
+
 		dataResult := wrm.MarshalProtobuf(nil)
 		if !bytes.Equal(dataResult, data) {
 			t.Fatalf("unexpected data obtained after marshaling\ngot\n%X\nwant\n%X", dataResult, data)
@@ -123,6 +132,15 @@ func TestWriteRequestUnmarshalProtobuf(t *testing.T) {
 			},
 		},
 	}
+	wrm.Metadata = []prompbmarshal.MetricMetadata{
+		{
+			// COUNTER = 1
+			Type:             1,
+			MetricFamilyName: "process_cpu_seconds_total",
+			Help:             "Total user and system CPU time spent in seconds",
+			Unit:             "seconds",
+		},
+	}
 	data = wrm.MarshalProtobuf(data[:0])
 	f(data)
 
@@ -168,6 +186,29 @@ func TestWriteRequestUnmarshalProtobuf(t *testing.T) {
 			},
 		},
 	}
+	wrm.Metadata = []prompbmarshal.MetricMetadata{
+		{
+			// COUNTER = 1
+			Type:             1,
+			MetricFamilyName: "process_cpu_seconds_total",
+			Help:             "Total user and system CPU time spent in seconds",
+			Unit:             "seconds",
+		},
+	}
+	data = wrm.MarshalProtobuf(data[:0])
+	f(data)
+
+	wrm.Reset()
+	// pass only MetricMetadata, no series
+	wrm.Metadata = []prompbmarshal.MetricMetadata{
+		{
+			// COUNTER = 1
+			Type:             1,
+			MetricFamilyName: "process_cpu_seconds_total",
+			Help:             "Total user and system CPU time spent in seconds",
+			Unit:             "seconds",
+		},
+	}
 	data = wrm.MarshalProtobuf(data[:0])
 	f(data)
 }

lib/prompb/prompb_timing_test.go

@@ -75,6 +75,15 @@ var benchWriteRequest = func() *prompbmarshal.WriteRequest {
 	}
 	wrm := &prompbmarshal.WriteRequest{
 		Timeseries: tss,
+		Metadata: []prompbmarshal.MetricMetadata{
+			{
+				// COUNTER = 1
+				Type:             1,
+				MetricFamilyName: "process_cpu_seconds_total",
+				Help:             "Total user and system CPU time spent in seconds",
+				Unit:             "seconds",
+			},
+		},
 	}
 	return wrm
 }()

lib/prompbmarshal/prompbmarshal_test.go

@@ -38,6 +38,15 @@ func TestWriteRequestMarshalProtobuf(t *testing.T) {
 				},
 			},
 		},
+		Metadata: []prompbmarshal.MetricMetadata{
+			{
+				// COUNTER = 1
+				Type:             1,
+				MetricFamilyName: "process_cpu_seconds_total",
+				Help:             "Total user and system CPU time spent in seconds",
+				Unit:             "seconds",
+			},
+		},
 	}
 	data := wrm.MarshalProtobuf(nil)
 
@@ -69,6 +78,15 @@ func TestWriteRequestMarshalProtobuf(t *testing.T) {
 			Samples: samples,
 		})
 	}
+	for _, md := range wr.Metadata {
+		wrm.Metadata = append(wrm.Metadata, prompbmarshal.MetricMetadata{
+			Type:             md.Type,
+			MetricFamilyName: md.MetricFamilyName,
+			Help:             md.Help,
+			Unit:             md.Unit,
+		})
+	}
+
 	dataResult := wrm.MarshalProtobuf(nil)
 
 	if !bytes.Equal(dataResult, data) {

lib/prompbmarshal/prompbmarshal_timing_test.go

@@ -69,6 +69,14 @@ var benchWriteRequest = func() *WriteRequest {
 	}
 	wr := &WriteRequest{
 		Timeseries: tss,
+		Metadata: []MetricMetadata{
+			{
+				Type:             1,
+				MetricFamilyName: "process_cpu_seconds_total",
+				Help:             "Total user and system CPU time spent in seconds",
+				Unit:             "seconds",
+			},
+		},
 	}
 	return wr
 }()

lib/prompbmarshal/remote.pb.go

@@ -1,18 +1,27 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: remote.proto
-
 package prompbmarshal
 
 import (
 	"math/bits"
 )
 
+// WriteRequest represents Remote Write request
 type WriteRequest struct {
 	Timeseries []TimeSeries
+	Metadata   []MetricMetadata
 }
 
 func (m *WriteRequest) marshalToSizedBuffer(dst []byte) (int, error) {
 	i := len(dst)
+	for j := len(m.Metadata) - 1; j >= 0; j-- {
+		size, err := m.Metadata[j].marshalToSizedBuffer(dst[:i])
+		if err != nil {
+			return 0, err
+		}
+		i -= size
+		i = encodeVarint(dst, i, uint64(size))
+		i--
+		dst[i] = 0x1a
+	}
 	for j := len(m.Timeseries) - 1; j >= 0; j-- {
 		size, err := m.Timeseries[j].marshalToSizedBuffer(dst[:i])
 		if err != nil {
@@ -45,6 +54,10 @@ func (m *WriteRequest) size() (n int) {
 		l := e.size()
 		n += 1 + l + sov(uint64(l))
 	}
+	for _, e := range m.Metadata {
+		l := e.size()
+		n += 1 + l + sov(uint64(l))
+	}
 	return n
 }
 

lib/prompbmarshal/types.pb.go

@@ -1,6 +1,3 @@
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: types.proto
-
 package prompbmarshal
 
 import (
@@ -10,6 +7,7 @@ import (
 	"strconv"
 )
 
+// Sample is a metric sample
 type Sample struct {
 	Value     float64
 	Timestamp int64
@@ -21,6 +19,7 @@ type TimeSeries struct {
 	Samples []Sample
 }
 
+// Label is a key-value label pair
 type Label struct {
 	Name  string
 	Value string
@@ -150,3 +149,64 @@ func LabelsToString(labels []Label) string {
 	b = append(b, '}')
 	return string(b)
 }
+
+// MetricMetadata represents additional meta information for specific MetricFamilyName
+// Refer to https://github.com/prometheus/prometheus/blob/c5282933765ec322a0664d0a0268f8276e83b156/prompb/types.proto#L21
+type MetricMetadata struct {
+	// Represents the metric type, these match the set from Prometheus.
+	// Refer to https://github.com/prometheus/common/blob/95acce133ca2c07a966a71d475fb936fc282db18/model/metadata.go for details.
+	Type             uint32
+	MetricFamilyName string
+	Help             string
+	Unit             string
+}
+
+func (m *MetricMetadata) marshalToSizedBuffer(dst []byte) (int, error) {
+	i := len(dst)
+	if len(m.Unit) > 0 {
+		i -= len(m.Unit)
+		copy(dst[i:], m.Unit)
+		i = encodeVarint(dst, i, uint64(len(m.Unit)))
+		i--
+		dst[i] = 0x2a
+	}
+	if len(m.Help) > 0 {
+		i -= len(m.Help)
+		copy(dst[i:], m.Help)
+		i = encodeVarint(dst, i, uint64(len(m.Help)))
+		i--
+		dst[i] = 0x22
+	}
+	if len(m.MetricFamilyName) > 0 {
+		i -= len(m.MetricFamilyName)
+		copy(dst[i:], m.MetricFamilyName)
+		i = encodeVarint(dst, i, uint64(len(m.MetricFamilyName)))
+		i--
+		dst[i] = 0x12
+	}
+	if m.Type != 0 {
+		i = encodeVarint(dst, i, uint64(m.Type))
+		i--
+		dst[i] = 0x8
+	}
+	return len(dst) - i, nil
+}
+
+func (m *MetricMetadata) size() (n int) {
+	if m == nil {
+		return 0
+	}
+	if m.Type != 0 {
+		n += 1 + sov(uint64(m.Type))
+	}
+	if l := len(m.MetricFamilyName); l > 0 {
+		n += 1 + l + sov(uint64(l))
+	}
+	if l := len(m.Help); l > 0 {
+		n += 1 + l + sov(uint64(l))
+	}
+	if l := len(m.Unit); l > 0 {
+		n += 1 + l + sov(uint64(l))
+	}
+	return n
+}

lib/prompbmarshal/util.go

@@ -21,6 +21,7 @@ func (wr *WriteRequest) MarshalProtobuf(dst []byte) []byte {
 // Reset resets wr.
 func (wr *WriteRequest) Reset() {
 	wr.Timeseries = ResetTimeSeries(wr.Timeseries)
+	wr.Metadata = ResetMetadata(wr.Metadata)
 }
 
 // ResetTimeSeries clears all the GC references from tss and returns an empty tss ready for further use.
@@ -28,3 +29,9 @@ func ResetTimeSeries(tss []TimeSeries) []TimeSeries {
 	clear(tss)
 	return tss[:0]
 }
+
+// ResetMetadata clears all the GC references from mms and returns an empty mms ready for further use.
+func ResetMetadata(mms []MetricMetadata) []MetricMetadata {
+	clear(mms)
+	return mms[:0]
+}

lib/promscrape/targetstatus.go

@@ -412,6 +412,12 @@ func (dt *droppedTargets) getTotalTargets() int {
 func labelsHash(labels *promutil.Labels) uint64 {
 	d := xxhashPool.Get().(*xxhash.Digest)
 	for _, label := range labels.GetLabels() {
+		// exclude annotations from hash generation
+		// annotations are mutable and should not be used for objects identification
+		// See this issue: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8626
+		if strings.HasPrefix(label.Name, "__meta_kubernetes_") && strings.Contains(label.Name, "_annotation_") {
+			continue
+		}
 		_, _ = d.WriteString(label.Name)
 		_, _ = d.WriteString(label.Value)
 	}

lib/promscrape/targetstatus_test.go

@@ -58,3 +58,44 @@ func TestWriteActiveTargetsJSON(t *testing.T) {
 	})
 	f("unknown", []activeTarget{})
 }
+
+func TestRegisterDroppedTargets(t *testing.T) {
+	type opts struct {
+		toRegister       []*promutil.Labels
+		wantTotalTargets int
+	}
+	f := func(opts opts) {
+		t.Helper()
+		dtm := &droppedTargets{
+			m: make(map[uint64]droppedTarget),
+		}
+
+		for _, originalLabels := range opts.toRegister {
+			dtm.Register(originalLabels, nil, targetDropReasonRelabeling, nil)
+		}
+		got := dtm.getTotalTargets()
+		if got != opts.wantTotalTargets {
+			t.Fatalf("unexpected total targets: (-%d;+%d)", opts.wantTotalTargets, got)
+		}
+	}
+
+	// duplicate annotations
+	f(opts{
+		toRegister: []*promutil.Labels{
+			promutil.MustNewLabelsFromString(`{up="1",__meta_kubernetes_endpoints_annotation_updated="123"}`),
+			promutil.MustNewLabelsFromString(`{up="1",__meta_kubernetes_endpoints_annotation_updated="125"}`),
+			promutil.MustNewLabelsFromString(`{up="1",__meta_docker_annotation_some="5"}`),
+		},
+		wantTotalTargets: 2,
+	})
+	// collision with missing annotation
+	f(opts{
+		toRegister: []*promutil.Labels{
+			promutil.MustNewLabelsFromString(`{up="1",pod="vmagent-0"}`),
+			promutil.MustNewLabelsFromString(`{up="1",pod="vmagent-0",__meta_kubernetes_endpoints_annotation_updated="125"}`),
+			promutil.MustNewLabelsFromString(`{up="1",__meta_docker_annotation_some="5"}`),
+		},
+		wantTotalTargets: 2,
+	})
+
+}