Update lib/promscrape/relabel_debug.go

Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Signed-off-by: Zhu Jiekun <jiekun@victoriametrics.com>

.golangci.yml

@@ -1,18 +1,9 @@
 version: "2"
 linters:
-  enable:
-    - errorlint
   settings:
     errcheck:
       exclude-functions:
         - (net/http.ResponseWriter).Write
-    errorlint:
-      errorf: true
-      # Do not enable `comparison` and `asserts`: they produce false positives,
-      # since many call sites intentionally compare sentinel errors directly (e.g. err == io.EOF)
-      # when the producer is documented to return them unwrapped. See https://github.com/VictoriaMetrics/VictoriaLogs/pull/1490
-      comparison: false
-      asserts: false
   exclusions:
     generated: lax
     presets:

app/vmagent/main.go

@@ -462,7 +462,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+		rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
+		promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
 		return true
 	case "/prometheus/target-relabel-debug", "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
 	p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(d)
 }
 
+// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
+func GetURLRelabelConfigString() []string {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
+	}
+	var ss []string
+	for i := range *remoteWriteURLs {
+		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
+		ss = append(ss, string(cfgDataBytes))
+	}
+	return ss
+}
+
 func reloadRelabelConfigs() {
 	rcs := allRelabelConfigs.Load()
 	if !rcs.isSet() {

app/vmctl/utils.go

@@ -74,9 +74,9 @@ func wrapErr(vmErr *vm.ImportError, verbose bool) error {
 		verboseMsg = "(enable `--verbose` output to get more details)"
 	}
 	if vmErr.Err == nil {
-		return fmt.Errorf("%w\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
+		return fmt.Errorf("%s\n\tLatest delivered batch for timestamps range %d - %d %s\n%s",
 			vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 	}
-	return fmt.Errorf("%w\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
+	return fmt.Errorf("%s\n\tImporting batch failed for timestamps range %d - %d %s\n%s",
 		vmErr.Err, minTS, maxTS, verboseMsg, errTS)
 }

app/vmselect/main.go

@@ -538,7 +538,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
 		return true
 	case "/metric-relabel-debug":
 		promscrapeMetricRelabelDebugRequests.Inc()
-		promscrape.WriteMetricRelabelDebug(w, r)
+		promscrape.WriteMetricRelabelDebug(w, r, "", nil)
 		return true
 	case "/target-relabel-debug":
 		promscrapeTargetRelabelDebugRequests.Inc()

app/vmstorage/main.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/metrics"
-
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -23,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/stringsutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vminsertapi"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -153,7 +152,7 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 		LogNewSeries:                *logNewSeries,
 	}
 	strg := storage.MustOpenStorage(*storageDataPath, opts)
-	vmStorage = newVMStorageSingleNode(strg, vmselectMaxConcurrentRequests, resetCacheIfNeeded)
+	vmStorage = newVMStorage(strg, vmselectMaxConcurrentRequests, resetCacheIfNeeded)
 
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)
@@ -175,15 +174,15 @@ func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []stora
 	GetSearch = vmStorage.GetSearch
 	PutSearch = vmStorage.PutSearch
 	RequestHandler = vmStorage.requestHandler
-	DebugFlush = vmStorage.vms.s.DebugFlush
+	DebugFlush = vmStorage.s.DebugFlush
 }
 
 var storageMetrics *metrics.Set
 
 var (
-	// vmStorageSingleNode is an instance of vmstorage used by vminsert and
+	// vmStorage is an instance of vmstorage used by vminsert and
 	// vmselect for writing and reading data.
-	vmStorage      *VMStorageSingleNode
+	vmStorage      *VMStorage
 	VMInsertAPI    vminsertapi.API
 	VMSelectAPI    vmselectapi.API
 	GetSearch      func(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (*storage.Search, int, error)
@@ -209,15 +208,12 @@ func Stop() {
 	logger.Infof("the vmstorage has been stopped")
 }
 
-func (vmssn *VMStorageSingleNode) requestHandler(w http.ResponseWriter, r *http.Request) bool {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.requestHandler(w, r)
-}
-
 // requestHandler is a storage request handler.
 // TODO(@rtm0): Move to a separate file, request_handler.go
 func (vms *VMStorage) requestHandler(w http.ResponseWriter, r *http.Request) bool {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
 		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey) {
@@ -361,15 +357,11 @@ var (
 	snapshotsDeleteAllErrorsTotal = metrics.NewCounter(`vm_http_request_errors_total{path="/snapshot/delete_all"}`)
 )
 
-// TODO(@rtm0): Move to metrics.go.
-func (vmssn *VMStorageSingleNode) writeStorageMetrics(w io.Writer) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	vmssn.vms.writeStorageMetrics(w)
-}
-
 // TODO(@rtm0): Move to metrics.go.
 func (vms *VMStorage) writeStorageMetrics(w io.Writer) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	strg := vms.s
 	var m storage.Metrics
 	strg.UpdateMetrics(&m)

app/vmstorage/vmstorage.go

@@ -1,6 +1,7 @@
 package vmstorage
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"sync"
@@ -14,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
 )
@@ -34,7 +36,7 @@ var (
 // newVMStorage creates a new instance of of VMStorage.
 //
 // The created VMStorage instance takes ownership of s.
-func newVMStorage(s *storage.Storage, vmselectMaxConcurrentRequests int) *VMStorage {
+func newVMStorage(s *storage.Storage, vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) *VMStorage {
 	if err := encoding.CheckPrecisionBits(uint8(*precisionBits)); err != nil {
 		logger.Fatalf("invalid -precisionBits=%d: %s", *precisionBits, err)
 	}
@@ -49,6 +51,8 @@ func newVMStorage(s *storage.Storage, vmselectMaxConcurrentRequests int) *VMStor
 		maxUniqueTimeseries:           *maxUniqueTimeseries,
 		maxUniqueTimeSeriesCalculated: maxUniqueTimeseriesCalculated,
 		staleSnapshotsRemoverCh:       make(chan struct{}),
+		wg:                            syncwg.WaitGroup{},
+		resetCacheIfNeeded:            resetCacheIfNeeded,
 	}
 	vms.initStaleSnapshotsRemover()
 	return vms
@@ -78,6 +82,17 @@ type VMStorage struct {
 	maxUniqueTimeSeriesCalculated int
 	staleSnapshotsRemoverCh       chan struct{}
 	staleSnapshotsRemoverWG       sync.WaitGroup
+
+	// wg is used to wrap every storage call into wg.Add(1) ... wg.Done()
+	// for proper graceful shutdown when Stop is called.
+	//
+	// Use syncwg instead of sync, since Add is called from concurrent
+	// goroutines.
+	wg syncwg.WaitGroup
+
+	// resetCacheIfNeeded is a callback for automatic resetting of response
+	// cache if needed.
+	resetCacheIfNeeded func(mrs []storage.MetricRow)
 }
 
 func (vms *VMStorage) initStaleSnapshotsRemover() {
@@ -103,6 +118,7 @@ func (vms *VMStorage) initStaleSnapshotsRemover() {
 func (vms *VMStorage) Stop() {
 	close(vms.staleSnapshotsRemoverCh)
 	vms.staleSnapshotsRemoverWG.Wait()
+	vms.wg.WaitAndBlock()
 	vms.s.MustClose()
 }
 
@@ -111,6 +127,14 @@ func (vms *VMStorage) Stop() {
 // The caller should limit the number of concurrent calls to WriteRows() in
 // order to limit memory usage.
 func (vms *VMStorage) WriteRows(rows []storage.MetricRow) error {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
+	if vms.s.IsReadOnly() {
+		return errReadOnly
+	}
+	vms.resetCacheIfNeeded(rows)
+
 	vms.s.AddRows(rows, uint8(*precisionBits))
 	return nil
 }
@@ -120,26 +144,41 @@ func (vms *VMStorage) WriteRows(rows []storage.MetricRow) error {
 // The caller should limit the number of concurrent calls to WriteMetadata() in
 // order to limit memory usage.
 func (vms *VMStorage) WriteMetadata(rows []metricsmetadata.Row) error {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
+	if vms.s.IsReadOnly() {
+		return errReadOnly
+	}
 	vms.s.AddMetadataRows(rows)
 	return nil
 }
 
+var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
+
 // IsReadOnly returns true is the storage is in read-only mode.
 func (vms *VMStorage) IsReadOnly() bool {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
 	return vms.s.IsReadOnly()
 }
 
 func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
+	vms.wg.Add(1)
+
 	tr := sq.GetTimeRange()
 	maxMetrics := vms.getMaxMetrics(sq.MaxMetrics)
 	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
 	if err != nil {
+		vms.wg.Done()
 		return nil, err
 	}
 	if len(tfss) == 0 {
+		vms.wg.Done()
 		return nil, fmt.Errorf("missing tag filters")
 	}
 	bi := getBlockIterator()
+	bi.wgDone = vms.wg.Done
 	bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
 	if err := bi.sr.Error(); err != nil {
 		bi.MustClose()
@@ -161,8 +200,9 @@ func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
 
 // blockIterator implements vmselectapi.BlockIterator
 type blockIterator struct {
-	sr storage.Search
-	mb storage.MetricBlock
+	sr     storage.Search
+	mb     storage.MetricBlock
+	wgDone func()
 }
 
 var blockIteratorsPool sync.Pool
@@ -171,6 +211,8 @@ func (bi *blockIterator) MustClose() {
 	bi.sr.MustClose()
 	bi.mb.MetricName = nil
 	bi.mb.Block.Reset()
+	bi.wgDone()
+	bi.wgDone = nil
 	blockIteratorsPool.Put(bi)
 }
 
@@ -197,8 +239,63 @@ func (bi *blockIterator) Error() error {
 	return bi.sr.Error()
 }
 
+// GetSearch sets up an instance of storage search and returns it to the caller
+// along with the max series count that the search can return.
+//
+// This method is not part of the vmselectapi.API and must only be used by
+// vmsingle HTTP handlers.
+//
+// Callers of this method must call PutSearch() once the search instance is not
+// needed anymore.
+func (vms *VMStorage) GetSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (*storage.Search, int, error) {
+	vms.wg.Add(1)
+
+	tr := sq.GetTimeRange()
+	maxMetrics := vms.getMaxMetrics(sq.MaxMetrics)
+	tfss, err := vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
+	if err != nil {
+		vms.wg.Done()
+		return nil, 0, err
+	}
+
+	sr := getSearch()
+	maxSeriesCount := sr.Init(qt, vms.s, tfss, tr, sq.MaxMetrics, deadline)
+	return sr, maxSeriesCount, nil
+}
+
+// PutSearch resets the search once it is not needed anymore and puts it aside
+// for future reuse.
+//
+// This method is not part of the vmselectapi.API and must only be used by
+// vmsingle HTTP handlers.
+//
+// The method must only be used on search instances that have been created with
+// GetSearch().
+func (vms *VMStorage) PutSearch(sr *storage.Search) {
+	putSearch(sr)
+	vms.wg.Done()
+}
+
+func getSearch() *storage.Search {
+	v := ssPool.Get()
+	if v == nil {
+		return &storage.Search{}
+	}
+	return v.(*storage.Search)
+}
+
+func putSearch(sr *storage.Search) {
+	sr.MustClose()
+	ssPool.Put(sr)
+}
+
+var ssPool sync.Pool
+
 // SearchMetricNames returns metric names for the given tfss on the given tr.
 func (vms *VMStorage) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	tr := sq.GetTimeRange()
 	maxMetrics := sq.MaxMetrics
 	if maxMetrics <= 0 {
@@ -219,6 +316,9 @@ func (vms *VMStorage) SearchMetricNames(qt *querytracer.Tracer, sq *storage.Sear
 // SearchLabelValues searches for label values for the given labelName, tfss and
 // tr.
 func (vms *VMStorage) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	tr := sq.GetTimeRange()
 	if maxLabelValues <= 0 || maxLabelValues > *maxTagValues {
 		maxLabelValues = *maxTagValues
@@ -244,6 +344,9 @@ func (vms *VMStorage) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuer
 // similar APIs.
 func (vms *VMStorage) TagValueSuffixes(qt *querytracer.Tracer, _, _ uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte,
 	maxSuffixes int, deadline uint64) ([]string, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	if maxSuffixes <= 0 || maxSuffixes > *maxTagValueSuffixesPerSearch {
 		maxSuffixes = *maxTagValueSuffixesPerSearch
 	}
@@ -260,6 +363,9 @@ func (vms *VMStorage) TagValueSuffixes(qt *querytracer.Tracer, _, _ uint32, tr s
 
 // SearchLabelNames searches for tag keys matching the given tfss on tr.
 func (vms *VMStorage) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	tr := sq.GetTimeRange()
 	if maxLabelNames <= 0 || maxLabelNames > *maxTagKeys {
 		maxLabelNames = *maxTagKeys
@@ -278,6 +384,8 @@ func (vms *VMStorage) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery
 }
 
 func (vms *VMStorage) SeriesCount(_ *querytracer.Tracer, _, _ uint32, deadline uint64) (uint64, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
 	return vms.s.GetSeriesCount(deadline)
 }
 
@@ -287,6 +395,9 @@ func (vms *VMStorage) Tenants(_ *querytracer.Tracer, _ storage.TimeRange, _ uint
 
 // GetTSDBStatus returns TSDB status for given filters on the given date.
 func (vms *VMStorage) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	tr := sq.GetTimeRange()
 	maxMetrics := sq.MaxMetrics
 	if maxMetrics <= 0 {
@@ -306,6 +417,9 @@ func (vms *VMStorage) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery
 //
 // Returns the number of deleted series.
 func (vms *VMStorage) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	tr := sq.GetTimeRange()
 	maxMetrics := sq.MaxMetrics
 	if maxMetrics <= 0 {
@@ -324,17 +438,26 @@ func (vms *VMStorage) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQue
 }
 
 func (vms *VMStorage) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, _ uint64) error {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	vms.s.RegisterMetricNames(qt, mrs)
 	return nil
 }
 
 // GetMetricNamesUsageStats returns metric name usage stats.
 func (vms *VMStorage) GetMetricNamesUsageStats(qt *querytracer.Tracer, _ *storage.TenantToken, limit, le int, matchPattern string, _ uint64) (metricnamestats.StatsResult, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	return vms.s.GetMetricNamesStats(qt, limit, le, matchPattern), nil
 }
 
 // ResetMetricNamesStats resets state for metric names usage tracker
 func (vms *VMStorage) ResetMetricNamesUsageStats(qt *querytracer.Tracer, _ uint64) error {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
+
 	vms.s.ResetMetricNamesStats(qt)
 	return nil
 }
@@ -371,6 +494,8 @@ func (vms *VMStorage) setupTfss(qt *querytracer.Tracer, sq *storage.SearchQuery,
 }
 
 func (vms *VMStorage) GetMetadataRecords(qt *querytracer.Tracer, _ *storage.TenantToken, limit int, metricName string, _ uint64) ([]*metricsmetadata.Row, error) {
+	vms.wg.Add(1)
+	defer vms.wg.Done()
 	return vms.s.GetMetadataRows(qt, limit, metricName), nil
 }
 

app/vmstorage/vmstorage_single_node.go

@@ -1,213 +0,0 @@
-package vmstorage
-
-import (
-	"errors"
-	"fmt"
-	"sync"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
-)
-
-// newVMStorageSingleNode creates a new instance of of VMStorage for vmsingle.
-func newVMStorageSingleNode(s *storage.Storage, maxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) *VMStorageSingleNode {
-	vms := newVMStorage(s, maxConcurrentRequests)
-	return &VMStorageSingleNode{
-		vms:                vms,
-		wg:                 syncwg.WaitGroup{},
-		resetCacheIfNeeded: resetCacheIfNeeded,
-	}
-}
-
-type VMStorageSingleNode struct {
-	vms *VMStorage
-
-	// wg is used to wrap every storage call into wg.Add(1) ... wg.Done()
-	// for proper graceful shutdown when Stop is called.
-	//
-	// Use syncwg instead of sync, since Add is called from concurrent
-	// goroutines.
-	wg syncwg.WaitGroup
-
-	// resetCacheIfNeeded is a callback for automatic resetting of response
-	// cache if needed.
-	resetCacheIfNeeded func(mrs []storage.MetricRow)
-}
-
-func (vmssn *VMStorageSingleNode) Stop() {
-	vmssn.wg.WaitAndBlock()
-	vmssn.vms.Stop()
-}
-
-// WriteRows writes metric rows to the storage.
-//
-// Returns an error if the storage is in read-only mode.
-//
-// The caller should limit the number of concurrent calls to WriteRows() in
-// order to limit memory usage.
-func (vmssn *VMStorageSingleNode) WriteRows(rows []storage.MetricRow) error {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-
-	if vmssn.vms.IsReadOnly() {
-		return errReadOnly
-	}
-	vmssn.resetCacheIfNeeded(rows)
-	return vmssn.vms.WriteRows(rows)
-}
-
-// WriteMetadata writes metrics metadata to storage.
-//
-// Returns an error if the storage is in read-only mode.
-//
-// The caller should limit the number of concurrent calls to WriteMetadata() in
-// order to limit memory usage.
-func (vmssn *VMStorageSingleNode) WriteMetadata(rows []metricsmetadata.Row) error {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-
-	if vmssn.vms.IsReadOnly() {
-		return errReadOnly
-	}
-	return vmssn.vms.WriteMetadata(rows)
-}
-
-var errReadOnly = errors.New("the storage is in read-only mode; check -storage.minFreeDiskSpaceBytes command-line flag value")
-
-func (vmssn *VMStorageSingleNode) IsReadOnly() bool {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.IsReadOnly()
-}
-
-func (vmssn *VMStorageSingleNode) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
-	return nil, fmt.Errorf("not implemented in vmsingle")
-}
-
-// GetSearch sets up an instance of storage search and returns it to the caller
-// along with the max series count that the search can return.
-//
-// This method is not part of the vmselectapi.API and must only be used by
-// vmsingle HTTP handlers.
-//
-// Callers of this method must call PutSearch() once the search instance is not
-// needed anymore.
-func (vmssn *VMStorageSingleNode) GetSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (*storage.Search, int, error) {
-	vmssn.wg.Add(1)
-
-	tr := sq.GetTimeRange()
-	maxMetrics := vmssn.vms.getMaxMetrics(sq.MaxMetrics)
-	tfss, err := vmssn.vms.setupTfss(qt, sq, tr, maxMetrics, deadline)
-	if err != nil {
-		vmssn.wg.Done()
-		return nil, 0, err
-	}
-
-	sr := getSearch()
-	maxSeriesCount := sr.Init(qt, vmssn.vms.s, tfss, tr, sq.MaxMetrics, deadline)
-	return sr, maxSeriesCount, nil
-}
-
-// PutSearch resets the search once it is not needed anymore and puts it aside
-// for future reuse.
-//
-// This method is not part of the vmselectapi.API and must only be used by
-// vmsingle HTTP handlers.
-//
-// The method must only be used on search instances that have been created with
-// GetSearch().
-func (vmssn *VMStorageSingleNode) PutSearch(sr *storage.Search) {
-	putSearch(sr)
-	vmssn.wg.Done()
-}
-
-func getSearch() *storage.Search {
-	v := ssPool.Get()
-	if v == nil {
-		return &storage.Search{}
-	}
-	return v.(*storage.Search)
-}
-
-func putSearch(sr *storage.Search) {
-	sr.MustClose()
-	ssPool.Put(sr)
-}
-
-var ssPool sync.Pool
-
-func (vmssn *VMStorageSingleNode) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.SearchMetricNames(qt, sq, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.LabelNames(qt, sq, maxLabelNames, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.SeriesCount(qt, accountID, projectID, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.Tenants(qt, tr, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.DeleteSeries(qt, sq, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.RegisterMetricNames(qt, mrs, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.ResetMetricNamesUsageStats(qt, deadline)
-}
-
-func (vmssn *VMStorageSingleNode) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
-	vmssn.wg.Add(1)
-	defer vmssn.wg.Done()
-	return vmssn.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
-}

app/vmstorage/vmstorage_test.go

@@ -48,7 +48,7 @@ func TestGetMaxMetrics(t *testing.T) {
 		t.Helper()
 		*maxUniqueTimeseries = storageMaxUniqueTimeseries
 		s := storage.MustOpenStorage(t.Name(), storage.OpenOptions{})
-		vms := newVMStorage(s, maxConcurrentRequests)
+		vms := newVMStorage(s, maxConcurrentRequests, func(mrs []storage.MetricRow) {})
 		defer vms.Stop()
 		maxMetrics := vms.getMaxMetrics(searchQueryLimit)
 		if maxMetrics != expect {

apptest/client.go

@@ -156,14 +156,14 @@ func readAllAndClose(t *testing.T, responseBody io.ReadCloser) string {
 //
 // This type is expected to be embedded by the apps that serve metrics.
 type metricsClient struct {
-	metricsCli *Client
-	url        string
+	cli *Client
+	url string
 }
 
 func newMetricsClient(cli *Client, addr string) *metricsClient {
 	return &metricsClient{
-		metricsCli: cli,
-		url:        fmt.Sprintf("http://%s/metrics", addr),
+		cli: cli,
+		url: fmt.Sprintf("http://%s/metrics", addr),
 	}
 }
 
@@ -179,7 +179,7 @@ func (c *metricsClient) GetIntMetric(t *testing.T, metricName string) int {
 func (c *metricsClient) GetMetric(t *testing.T, metricName string) float64 {
 	t.Helper()
 
-	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
+	metrics, statusCode := c.cli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -205,7 +205,7 @@ func (c *metricsClient) GetMetricsByPrefix(t *testing.T, prefix string) []float6
 
 	values := []float64{}
 
-	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
+	metrics, statusCode := c.cli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -234,7 +234,7 @@ func (c *metricsClient) GetMetricsByRegexp(t *testing.T, re *regexp.Regexp) []fl
 
 	values := []float64{}
 
-	metrics, statusCode := c.metricsCli.Get(t, c.url, nil)
+	metrics, statusCode := c.cli.Get(t, c.url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -270,7 +270,7 @@ func (c *metricsClient) rpcRowsSentTotal(t *testing.T) int {
 }
 
 type vmselectClient struct {
-	vmselectCli              *Client
+	cli                      *Client
 	url                      func(op, path string, opts QueryOpts) string
 	metricNamesStatsResetURL string
 	tenantsURL               string
@@ -287,7 +287,7 @@ func (c *vmselectClient) PrometheusAPIV1Export(t *testing.T, query string, opts
 	values := opts.asURLValues()
 	values.Add("match[]", query)
 	values.Add("format", "promapi")
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1QueryResponse(t, res)
 }
 
@@ -302,7 +302,7 @@ func (c *vmselectClient) PrometheusAPIV1ExportNative(t *testing.T, query string,
 	values := opts.asURLValues()
 	values.Add("match[]", query)
 	values.Add("format", "promapi")
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return []byte(res)
 }
 
@@ -315,7 +315,7 @@ func (c *vmselectClient) PrometheusAPIV1Query(t *testing.T, query string, opts Q
 	url := c.url("select", "prometheus/api/v1/query", opts)
 	values := opts.asURLValues()
 	values.Add("query", query)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1QueryResponse(t, res)
 }
 
@@ -329,7 +329,7 @@ func (c *vmselectClient) PrometheusAPIV1QueryRange(t *testing.T, query string, o
 	url := c.url("select", "prometheus/api/v1/query_range", opts)
 	values := opts.asURLValues()
 	values.Add("query", query)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1QueryResponse(t, res)
 }
 
@@ -342,7 +342,7 @@ func (c *vmselectClient) PrometheusAPIV1Series(t *testing.T, matchQuery string,
 	url := c.url("select", "prometheus/api/v1/series", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1SeriesResponse(t, res)
 }
 
@@ -354,7 +354,7 @@ func (c *vmselectClient) PrometheusAPIV1SeriesCount(t *testing.T, opts QueryOpts
 	t.Helper()
 	url := c.url("select", "prometheus/api/v1/series/count", opts)
 	values := opts.asURLValues()
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1SeriesCountResponse(t, res)
 }
 
@@ -367,7 +367,7 @@ func (c *vmselectClient) PrometheusAPIV1Labels(t *testing.T, matchQuery string,
 	url := c.url("select", "prometheus/api/v1/labels", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1LabelsResponse(t, res)
 }
 
@@ -382,7 +382,7 @@ func (c *vmselectClient) PrometheusAPIV1LabelValues(t *testing.T, labelName, mat
 	url := c.url("select", path, opts)
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1LabelValuesResponse(t, res)
 }
 
@@ -394,7 +394,7 @@ func (c *vmselectClient) PrometheusAPIV1Metadata(t *testing.T, metric string, li
 	values := opts.asURLValues()
 	values.Add("metric", metric)
 	values.Add("limit", strconv.Itoa(limit))
-	res, _ := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, _ := c.cli.PostForm(t, url, values, opts.Headers)
 	return NewPrometheusAPIV1Metadata(t, res)
 }
 
@@ -408,7 +408,7 @@ func (c *vmselectClient) PrometheusAPIV1AdminTSDBDeleteSeries(t *testing.T, matc
 	url := c.url("delete", "prometheus/api/v1/admin/tsdb/delete_series", opts)
 	values := opts.asURLValues()
 	values.Add("match[]", matchQuery)
-	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusNoContent {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
 	}
@@ -426,7 +426,7 @@ func (c *vmselectClient) PrometheusAPIV1StatusMetricNamesStats(t *testing.T, lim
 	values.Add("limit", limit)
 	values.Add("le", le)
 	values.Add("match_pattern", matchPattern)
-	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
 	}
@@ -455,7 +455,7 @@ func (c *vmselectClient) PrometheusAPIV1StatusTSDB(t *testing.T, matchQuery stri
 	addNonEmpty("match[]", matchQuery)
 	addNonEmpty("topN", topN)
 	addNonEmpty("date", date)
-	res, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	res, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
 	}
@@ -476,7 +476,7 @@ func (c *vmselectClient) GraphiteMetricsIndex(t *testing.T, opts QueryOpts) Grap
 	t.Helper()
 
 	url := c.url("select", "graphite/metrics/index.json", opts)
-	res, statusCode := c.vmselectCli.Get(t, url, opts.Headers)
+	res, statusCode := c.cli.Get(t, url, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
 	}
@@ -499,7 +499,7 @@ func (c *vmselectClient) GraphiteMetricsFind(t *testing.T, query string, opts Qu
 	url := c.url("select", "graphite/metrics/find", opts)
 	values := opts.asURLValues()
 	values.Add("query", query)
-	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	resText, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
 	}
@@ -522,7 +522,7 @@ func (c *vmselectClient) GraphiteMetricsExpand(t *testing.T, query string, opts
 	url := c.url("select", "graphite/metrics/expand", opts)
 	values := opts.asURLValues()
 	values.Add("query", query)
-	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	resText, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
 	}
@@ -546,7 +546,7 @@ func (c *vmselectClient) GraphiteRender(t *testing.T, target string, opts QueryO
 	values := opts.asURLValues()
 	values.Add("format", "json")
 	values.Add("target", target)
-	resText, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	resText, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, resText)
 	}
@@ -567,7 +567,7 @@ func (c *vmselectClient) GraphiteTagsTagSeries(t *testing.T, record string, opts
 	url := c.url("select", "graphite/tags/tagSeries", opts)
 	values := opts.asURLValues()
 	values.Add("path", record)
-	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	_, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if got, want := statusCode, http.StatusNotImplemented; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d", got, want)
 	}
@@ -584,7 +584,7 @@ func (c *vmselectClient) GraphiteTagsTagMultiSeries(t *testing.T, records []stri
 	for _, rec := range records {
 		values.Add("path", rec)
 	}
-	_, statusCode := c.vmselectCli.PostForm(t, url, values, opts.Headers)
+	_, statusCode := c.cli.PostForm(t, url, values, opts.Headers)
 	if got, want := statusCode, http.StatusNotImplemented; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d", got, want)
 	}
@@ -598,7 +598,7 @@ func (c *vmselectClient) GraphiteTagsTagMultiSeries(t *testing.T, records []stri
 func (c *vmselectClient) PrometheusAPIV1AdminStatusMetricNamesStatsReset(t *testing.T, opts QueryOpts) {
 	t.Helper()
 	values := opts.asURLValues()
-	res, statusCode := c.vmselectCli.PostForm(t, c.metricNamesStatsResetURL, values, opts.Headers)
+	res, statusCode := c.cli.PostForm(t, c.metricNamesStatsResetURL, values, opts.Headers)
 	if statusCode != http.StatusNoContent {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusNoContent, res)
 	}
@@ -608,7 +608,7 @@ func (c *vmselectClient) PrometheusAPIV1AdminStatusMetricNamesStatsReset(t *test
 // /admin/tenants endpoint.
 func (c *vmselectClient) APIV1AdminTenants(t *testing.T, opts QueryOpts) *AdminTenantsResponse {
 	t.Helper()
-	res, statusCode := c.vmselectCli.Get(t, c.tenantsURL, opts.Headers)
+	res, statusCode := c.cli.Get(t, c.tenantsURL, opts.Headers)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", statusCode, http.StatusOK, res)
 	}
@@ -622,7 +622,7 @@ func (c *vmselectClient) APIV1AdminTenants(t *testing.T, opts QueryOpts) *AdminT
 }
 
 type vminsertClient struct {
-	vminsertCli        *Client
+	cli                *Client
 	url                func(op, path string, opts QueryOpts) string
 	openTSDBURL        func(op, path string, opts QueryOpts) string
 	graphiteListenAddr string
@@ -647,7 +647,7 @@ func (c *vminsertClient) PrometheusAPIV1ImportCSV(t *testing.T, records []string
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "text/plain")
 	c.sendBlocking(t, len(records), func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusNoContent {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 		}
@@ -671,7 +671,7 @@ func (c *vminsertClient) PrometheusAPIV1ImportNative(t *testing.T, data []byte,
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "text/plain")
 	c.sendBlocking(t, 1, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusNoContent {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 		}
@@ -693,7 +693,7 @@ func (c *vminsertClient) PrometheusAPIV1Write(t *testing.T, wr prompb.WriteReque
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "application/x-protobuf")
 	c.sendBlocking(t, recordsCount, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusNoContent {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 		}
@@ -745,7 +745,7 @@ func (c *vminsertClient) PrometheusAPIV1ImportPrometheus(t *testing.T, records [
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "text/plain")
 	c.sendBlocking(t, recordsCount, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusNoContent {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 		}
@@ -771,7 +771,7 @@ func (c *vminsertClient) InfluxWrite(t *testing.T, records []string, opts QueryO
 	headers.Set("Content-Type", "text/plain")
 	c.sendBlocking(t, len(records), func() {
 		t.Helper()
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusNoContent {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 		}
@@ -805,7 +805,7 @@ func (c *vminsertClient) OpentelemetryV1Metrics(t *testing.T, md otlppb.MetricsD
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "application/x-protobuf")
 	c.sendBlocking(t, recordsCount, func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusOK {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 		}
@@ -830,7 +830,7 @@ func (c *vminsertClient) OpenTSDBAPIPut(t *testing.T, records []string, opts Que
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "application/json")
 	c.sendBlocking(t, len(records), func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusNoContent {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusNoContent)
 		}
@@ -853,7 +853,7 @@ func (c *vminsertClient) ZabbixConnectorHistory(t *testing.T, records []string,
 	headers := opts.getHeaders()
 	headers.Set("Content-Type", "application/json")
 	c.sendBlocking(t, len(records), func() {
-		_, statusCode := c.vminsertCli.Post(t, url, data, headers)
+		_, statusCode := c.cli.Post(t, url, data, headers)
 		if statusCode != http.StatusOK {
 			t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 		}
@@ -867,11 +867,11 @@ func (c *vminsertClient) ZabbixConnectorHistory(t *testing.T, records []string,
 // See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
 func (c *vminsertClient) GraphiteWrite(t *testing.T, records []string, _ QueryOpts) {
 	t.Helper()
-	c.vminsertCli.Write(t, c.graphiteListenAddr, records)
+	c.cli.Write(t, c.graphiteListenAddr, records)
 }
 
 type vmstorageClient struct {
-	vmstorageCli   *Client
+	cli            *Client
 	httpListenAddr string
 }
 
@@ -881,7 +881,7 @@ func (c *vmstorageClient) ForceFlush(t *testing.T) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/internal/force_flush", c.httpListenAddr)
-	_, statusCode := c.vmstorageCli.Get(t, url, nil)
+	_, statusCode := c.cli.Get(t, url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -892,7 +892,7 @@ func (c *vmstorageClient) ForceMerge(t *testing.T) {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/internal/force_merge", c.httpListenAddr)
-	_, statusCode := c.vmstorageCli.Get(t, url, nil)
+	_, statusCode := c.cli.Get(t, url, nil)
 	if statusCode != http.StatusOK {
 		t.Fatalf("unexpected status code: got %d, want %d", statusCode, http.StatusOK)
 	}
@@ -905,7 +905,7 @@ func (c *vmstorageClient) ForceMerge(t *testing.T) {
 func (c *vmstorageClient) SnapshotCreate(t *testing.T) *SnapshotCreateResponse {
 	t.Helper()
 
-	data, statusCode := c.vmstorageCli.Post(t, c.SnapshotCreateURL(), nil, nil)
+	data, statusCode := c.cli.Post(t, c.SnapshotCreateURL(), nil, nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -931,7 +931,7 @@ func (c *vmstorageClient) APIV1AdminTSDBSnapshot(t *testing.T) *APIV1AdminTSDBSn
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/api/v1/admin/tsdb/snapshot", c.httpListenAddr)
-	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
+	data, statusCode := c.cli.Post(t, url, nil, nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -952,7 +952,7 @@ func (c *vmstorageClient) SnapshotList(t *testing.T) *SnapshotListResponse {
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/snapshot/list", c.httpListenAddr)
-	data, statusCode := c.vmstorageCli.Get(t, url, nil)
+	data, statusCode := c.cli.Get(t, url, nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}
@@ -973,7 +973,7 @@ func (c *vmstorageClient) SnapshotDelete(t *testing.T, snapshotName string) *Sna
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/snapshot/delete?snapshot=%s", c.httpListenAddr, snapshotName)
-	data, statusCode := c.vmstorageCli.Delete(t, url)
+	data, statusCode := c.cli.Delete(t, url)
 	wantStatusCodes := map[int]bool{
 		http.StatusOK:                  true,
 		http.StatusInternalServerError: true,
@@ -998,7 +998,7 @@ func (c *vmstorageClient) SnapshotDeleteAll(t *testing.T) *SnapshotDeleteAllResp
 	t.Helper()
 
 	url := fmt.Sprintf("http://%s/snapshot/delete_all", c.httpListenAddr)
-	data, statusCode := c.vmstorageCli.Post(t, url, nil, nil)
+	data, statusCode := c.cli.Post(t, url, nil, nil)
 	if got, want := statusCode, http.StatusOK; got != want {
 		t.Fatalf("unexpected status code: got %d, want %d, resp text=%q", got, want, data)
 	}

apptest/vminsert.go

@@ -77,7 +77,7 @@ type vminsertRuntimeValues struct {
 func newVminsert(app *app, cli *Client, rt vminsertRuntimeValues) *Vminsert {
 	metricsClient := newMetricsClient(cli, rt.httpListenAddr)
 	vminsertClient := &vminsertClient{
-		vminsertCli: cli,
+		cli: cli,
 		url: func(op, path string, opts QueryOpts) string {
 			return getClusterPath(rt.httpListenAddr, op, path, opts)
 		},

apptest/vmselect.go

@@ -48,7 +48,7 @@ func newVmselect(app *app, cli *Client, rt vmselectRuntimeValues) *Vmselect {
 		app:           app,
 		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmselectClient: &vmselectClient{
-			vmselectCli: cli,
+			cli: cli,
 			url: func(op, path string, opts QueryOpts) string {
 				return getClusterPath(rt.httpListenAddr, op, path, opts)
 			},

apptest/vmsingle.go

@@ -58,11 +58,11 @@ func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 		app:           app,
 		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmstorageClient: &vmstorageClient{
-			vmstorageCli:   cli,
+			cli:            cli,
 			httpListenAddr: rt.httpListenAddr,
 		},
 		vmselectClient: &vmselectClient{
-			vmselectCli: cli,
+			cli: cli,
 			url: func(op, path string, opts QueryOpts) string {
 				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
 			},
@@ -70,7 +70,7 @@ func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
 			tenantsURL:               "vmsingle-does-not-serve-tenants",
 		},
 		vminsertClient: &vminsertClient{
-			vminsertCli: cli,
+			cli: cli,
 			url: func(_, path string, _ QueryOpts) string {
 				return fmt.Sprintf("http://%s/%s", rt.httpListenAddr, path)
 			},

apptest/vmstorage.go

@@ -63,7 +63,7 @@ func newVmstorage(app *app, cli *Client, rt vmstorageRuntimeValues) *Vmstorage {
 		app:           app,
 		metricsClient: newMetricsClient(cli, rt.httpListenAddr),
 		vmstorageClient: &vmstorageClient{
-			vmstorageCli:   cli,
+			cli:            cli,
 			httpListenAddr: rt.httpListenAddr,
 		},
 		storageDataPath: rt.storageDataPath,

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.5
     depends_on:
       - "victoriametrics"
     ports:

deployment/docker/vmanomaly/vmanomaly-integration/vmanomaly_guide_dashboard.json

@@ -43,7 +43,7 @@
         "content": "If you don't observe any data initially, please wait a few minutes for it to appear. \n\nUpon the first running the guide (if there is not enough node_exporter monitoring data collected in your system), you may notice a significant number of false positive anomalies found. The predictions will become more accurate with at least two weeks' (full `fit_window`) worth of data provided to vmanomaly.\n\nEach row displays information for a distinct mode. The query used for anomaly detection is `sum(rate(node_cpu_seconds_total[5m])) by (mode, instance, job)`.\n",
         "mode": "markdown"
       },
-      "pluginVersion": "10.2.1",
+      "pluginVersion": "12.2.0",
       "title": "Overview",
       "type": "text"
     },

docs/anomaly-detection/CHANGELOG.md

@@ -14,6 +14,21 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
+## v1.29.5
+Released: 2026-06-11
+
+- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v170) to [v1.7.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v171), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v171) for details.
+
+- IMPROVEMENT: Redesigned [hot reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) config change detection to content-based polling with configurable `-configCheckInterval`, improving reliability for Kubernetes ConfigMap symlink rotations and other filesystems where event delivery can be inconsistent.
+
+- IMPROVEMENT: Refined config validation errors for broken or invalid config sections, so startup and reload failures point to the affected section more clearly (e.g. YAML indentation typos).
+
+- IMPROVEMENT: Tightened config validation for [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) `infer_every` and [`IsolationForestModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate) `contamination`, including clearer handling of missing scheduler intervals, numeric contamination strings, and invalid non-finite values.
+
+- BUGFIX: Fixed a multiprocessing startup issue with `settings.n_workers > 1` that could leave scheduled data fetch or successive inference jobs stuck and repeatedly skipped by internal scheduler.
+
+- BUGFIX: Bounded [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [`VLogsReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader) data fetch and post-fetch processing waits so stalled datasource reads or multiprocessing dataframe creation no longer keep [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) `data_fetch` jobs running indefinitely. Previously, such stuck jobs could keep internal scheduler's `max_instances=1` slot per (scheduler, query) pair occupied, causing future data fetch, fit, or infer runs to be skipped until vmanomaly was restarted. The config validator now also warns when the configured reader timeout budget can exceed the connected scheduler interval.
+
 ## v1.29.4
 Released: 2026-05-15
 

docs/anomaly-detection/FAQ.md

@@ -423,7 +423,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.5
     # ...
     restart: always
     volumes:
@@ -641,7 +641,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.4 && docker image tag victoriametrics/vmanomaly:v1.29.4 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.29.5 && docker image tag victoriametrics/vmanomaly:v1.29.5 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/Migration.md

@@ -45,8 +45,7 @@ There are 2 types of compatibility to consider when migrating in stateful mode:
 
 | Group start | Group end | Compatibility | Notes |
 |---------|--------- |------------|-------|
-| [v1.29.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1293) | Latest* | Fully Compatible | Just a placeholder for new releases |
-| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1293) | Fully Compatible | - |
+| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.5](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1295) | Fully Compatible | - |
 | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Partially compatible* | Dumped models of class [prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) and [seasonal quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) have problems with loading to [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) due to dropped `pytz` library. **Upgrading directly from v1.28.7 to [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) with a fix is suggested** |
 | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
 | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |

docs/anomaly-detection/QuickStart.md

@@ -37,29 +37,39 @@ The `vmanomaly` service supports a set of command-line arguments to configure it
 > Single-dashed command-line argument {{% available_from "v1.23.3" anomaly %}} format can be used, e.g. `-license.forceOffline` in addition to `--license.forceOffline`. This aligns better with other VictoriaMetrics ecosystem components. Mixing the two styles is also supported, e.g. `-license.forceOffline --loggerLevel INFO`.
 
 ```shellhelp
-usage: vmanomaly.py [-h] [--license STRING | --licenseFile PATH] [--license.forceOffline] [--loggerLevel {DEBUG,WARNING,FATAL,ERROR,INFO}] [--watch] [--dryRun] [--outputSpec PATH] config [config ...]
+usage: vmanomaly.py [-h] [--license STRING | --licenseFile PATH] [--license.forceOffline] [--loggerLevel {DEBUG,INFO,WARNING,ERROR,FATAL}] [--watch] [-configCheckInterval DURATION] [--dryRun] [--outputSpec PATH] config [config ...]
 
 VictoriaMetrics Anomaly Detection Service
 
 positional arguments:
-  config                YAML config file(s) or directories containing YAML files. Multiple files will recursively merge each other values so multiple configs can be combined. If a directory is provided,
-                        all `.yaml` files inside will be merged, without recursion. Default: vmanomaly.yaml is expected in the current directory.
+  config                YAML config file(s) or directories containing YAML files. Multiple files will recursively merge each other
+                        values so multiple configs can be combined. If a directory is provided, all `.yaml` files inside will be
+                        merged, without recursion. Default: vmanomaly.yaml is expected in the current directory.
 
 options:
   -h                    Show this help message and exit
-  --license STRING      License key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/trial/ to obtain a trial license.
-  --licenseFile PATH    Path to file with license key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/trial/ to obtain a trial license.
+  --license STRING      License key for VictoriaMetrics Enterprise. See https://victoriametrics.com/products/enterprise/trial/ to
+                        obtain a trial license.
+  --licenseFile PATH    Path to file with license key for VictoriaMetrics Enterprise. See
+                        https://victoriametrics.com/products/enterprise/trial/ to obtain a trial license.
   --license.forceOffline
-                        Whether to force offline verification for VictoriaMetrics Enterprise license key, which has been passed either via -license or via -licenseFile command-line flag. The issued
-                        license key must support offline verification feature. Contact info@victoriametrics.com if you need offline license verification.
-  --loggerLevel {DEBUG,WARNING,FATAL,ERROR,INFO}
-                        Minimum level to log. Possible values: DEBUG, INFO, WARNING, ERROR, FATAL.
-  --watch               Watch config files for changes and trigger hot reloads. Watches the specified config file or directory for modifications, deletions, or additions. Upon detecting changes,
-                        triggers config reload. If new config validation fails, continues with previous valid config and state.
-  --dryRun              Validate only: parse + merge all YAML(s) and run schema checks, then exit. Does not require a license to run. Does not expose metrics, or launch vmanomaly service(s).
+                        Whether to force offline verification for VictoriaMetrics Enterprise license key, which has been passed either
+                        via -license or via -licenseFile command-line flag. The issued license key must support offline verification
+                        feature. Contact info@victoriametrics.com if you need offline license verification.
+  --loggerLevel {DEBUG,INFO,WARNING,ERROR,FATAL}
+                        Minimum level to log. Possible values: {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'FATAL'}.
+  --watch               Watch config files for changes and trigger hot reloads. Watches the specified config file or directory for
+                        modifications, deletions, or additions. Upon detecting changes, triggers config reload. If new config
+                        validation fails, continues with previous valid config and state.
+  -configCheckInterval DURATION
+                        Interval for checking watched config files for content changes. Default: 30s.
+  --dryRun              Validate only: parse + merge all YAML(s) and run schema checks, then exit. Does not require a license to run.
+                        Does not expose metrics, or launch vmanomaly service(s).
   --outputSpec PATH     Target location of .yaml output spec.
 ```
 
+{{% available_from "v1.29.5" anomaly %}} When `--watch` is enabled, config changes are detected by fixed-interval content polling instead of filesystem event delivery. The polling frequency is controlled by `-configCheckInterval` (default: `30s`). The same option can also be passed as `--configCheckInterval`, `--config.check.interval`, `--config-check-interval`, `--config_check_interval`, or in key-value form such as `configCheckInterval=30s`.
+
 You can specify these options when running `vmanomaly` to fine-tune logging levels or handle licensing configurations, as per your requirements.
 
 ### Licensing
@@ -122,7 +132,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.4
+docker pull victoriametrics/vmanomaly:v1.29.5
 ```
 
 2. Create the license file with your license key.
@@ -142,7 +152,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.4 \
+    victoriametrics/vmanomaly:v1.29.5 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -159,7 +169,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.4 \
+    victoriametrics/vmanomaly:v1.29.5 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -172,7 +182,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.5
     # ...
     restart: always
     volumes:

docs/anomaly-detection/UI.md

@@ -315,7 +315,7 @@ docker run -it --rm \
   -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
   -p 8080:8080 \
   -p 8490:8490 \
-  victoriametrics/vmanomaly:v1.29.4 \
+  victoriametrics/vmanomaly:v1.29.5 \
   vmanomaly_config.yaml
 ```
 
@@ -640,6 +640,17 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
+### v1.7.1
+Released: 2026-06-11
+
+vmanomaly version: [v1.29.5](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1295)
+
+- FEATURE: Added bulk Apply/Decline actions for [Copilot](#ai-assistance) chat suggestions.
+
+- BUGFIX: Fixed modal windows closing when the mouse is released outside the window during text selection.
+
+- BUGFIX: Fixed tooltip hover behavior so tooltips do not disappear while the cursor moves into the hover content.
+
 ### v1.7.0
 Released: 2026-05-15
 

docs/anomaly-detection/components/README.md

@@ -143,11 +143,14 @@ server:
 
 > This feature is better used in conjunction with [stateful service](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the state of the models and schedulers between restarts and reuse what can be reused, thus avoiding unnecessary re-training of models, re-initialization of schedulers and re-reading of data.
 
-{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change filesystem events without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_config_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
+{{% available_from "v1.25.0" anomaly %}} Service supports hot reload of configuration files, which allows for automatic reloading of configurations on config files change without the need of explicit service restart. This can be enabled via the `--watch` [CLI argument](https://docs.victoriametrics.com/anomaly-detection/quickstart/#command-line-arguments). `vmanomaly_config_reload_enabled` flag in [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) will be set to 1 (if enabled) or 0 (if disabled).
+
+> [!NOTE]
+> {{% deprecated_from "v1.29.5" anomaly %}} File system event-based hot reload has been deprecated in favor of content-based polling with configurable `-configCheckInterval` due to reliability issues with Kubernetes ConfigMap symlink rotations and other filesystems where event delivery can be inconsistent. If you were using file system event-based hot reload, please switch to content-based polling by enabling `--watch` flag and configuring `-configCheckInterval` as needed.
 
 ### How it works
 
-It works by watching for file system events, such as modifications, creations, or deletions of `.yml|.yaml` files in the specified directories. When a change is detected, the service will attempt to reload the configuration files, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration) and reinitialize the components. If the reload is successful, the `vmanomaly_config_reloads_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
+It works by checking watched `.yml|.yaml` file contents in the specified files or directories on the configured interval `-configCheckInterval` (default is `30s`) {{% available_from "v1.29.5" anomaly %}}. When a content change is detected, the service will attempt to reload the configuration files after the existing debounce window, rebuild the [global config](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#global-configuration) and reinitialize the components. If the reload is successful, the `vmanomaly_config_reloads_total` metric will be incremented for `status="success"` label, otherwise it will be incremented with `status="failure"` label and a respective error message on config validation failure(s) will be logged.
 
 > If the reload fails, the service will log an error message indicating the reason for the failure, and the **previous configuration will remain active until a successful reload occurs** to preserve the service's stability. This means that if there are errors in the new configuration, the service will continue to operate with the last valid configuration until the issues are resolved.
 

docs/anomaly-detection/components/models.md

@@ -449,9 +449,9 @@ models:
 
 > The `decay` argument works only in combination with [online models](#online-models) like [`ZScoreOnlineModel`](#online-z-score) or [`OnlineQuantileModel`](#online-seasonal-quantile).
 
-The `decay` {{% available_from "v1.23.0" anomaly %}} argument is used to control the (exponential) **decay factor** for online models, which determines how quickly the model adapts to new data. It is a float value between `0.0` and `1.0`, where:
-- `1.0` means no decay (the model treats all data equally, without giving more weight to recent data). This is the default value for backward compatibility.
-- Less than `1.0` means that the model will give more weight to recent data, effectively "forgetting" older data over time.
+The `decay` {{% available_from "v1.23.0" anomaly %}} argument is used to control the (exponential) **decay factor** for online models, which determines how quickly the model adapts to new data. It is a positive float value from `(0.0, 1.0]` interval, where:
+- Value `1.0` means no decay (the model treats all data points equally, without giving more weight to recent ones). This is the default value for backward compatibility.
+- Values less than `1.0` mean that the model will give more weight to recent data, effectively "forgetting" older data over time.
 
 Roughly speaking, for the recent N datapoints model processes `decay` = `d` means that these datapoints will contribute to the model as [1 - d^X] percent of total importance, for example decay of
 - `0.99` means that 100 recent datapoints will contribute as [1 - 0.99^100] = 63.23% of total importance
@@ -998,7 +998,7 @@ Here we use Isolation Forest implementation from `scikit-learn` [library](https:
 
 * `class` (string) - model class name `"model.isolation_forest.IsolationForestMultivariateModel"` (or `isolation_forest_multivariate` with class alias support {{% available_from "v1.13.0" anomaly %}})
 
-* `contamination` (float or string, optional) - The amount of contamination of the data set, i.e. the proportion of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. Default value - "auto". Should be either `"auto"` or be in the range (0.0, 0.5].
+* `contamination` (float or string, optional) - The amount of contamination of the data set, i.e. the proportion of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. Default value - "auto". Should be either `"auto"` or be in the range (0.0, 0.5]. {{% available_from "v1.29.5" anomaly %}} Numeric strings, such as `"0.01"`, are accepted, while invalid non-finite values, such as `nan`, `inf`, and `-inf`, are rejected during config validation.
 
 * `seasonal_features` (list of string) - List of seasonality to encode through [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca), i.e. `dow` (day of week). **Introduced in [1.12.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1120)**. 
   - Empty by default for backward compatibility.
@@ -1265,7 +1265,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.4
+docker pull victoriametrics/vmanomaly:v1.29.5
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1279,7 +1279,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.29.4 /config.yaml \
+victoriametrics/vmanomaly:v1.29.5 /config.yaml \
 --licenseFile=/license
 --watch
 ```

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -10,12 +10,12 @@ sitemap:
 
 - To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
 - In the tutorial, we'll be using the following VictoriaMetrics components:
-  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.137.0)
-  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.137.0)
-  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.137.0)
-- [Grafana](https://grafana.com/) (v.10.2.1)
+  -  [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.145.0)
+  -  [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.145.0)
+  -  [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.145.0)
+- [Grafana](https://grafana.com/) (v12.2.0)
 - [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
-- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
+- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
 
 ![typical setup diagram](guide-vmanomaly-vmalert_overview.webp)
 
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
 services:
   vmagent:
     container_name: vmagent
-    image: victoriametrics/vmagent:v1.137.0
+    image: victoriametrics/vmagent:v1.145.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -340,7 +340,7 @@ services:
 
   victoriametrics:
     container_name: victoriametrics
-    image: victoriametrics/victoria-metrics:v1.137.0
+    image: victoriametrics/victoria-metrics:v1.145.0
     ports:
       - 8428:8428
     volumes:
@@ -356,7 +356,7 @@ services:
 
   grafana:
     container_name: grafana
-    image: grafana/grafana-oss:10.2.1
+    image: grafana/grafana:12.2.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -373,7 +373,7 @@ services:
 
   vmalert:
     container_name: vmalert
-    image: victoriametrics/vmalert:v1.137.0
+    image: victoriametrics/vmalert:v1.145.0
     depends_on:
       - "victoriametrics"
     ports:
@@ -395,7 +395,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.5
     depends_on:
       - "victoriametrics"
     ports:
@@ -412,7 +412,7 @@ services:
       - "--licenseFile=/license"
   alertmanager:
     container_name: alertmanager
-    image: prom/alertmanager:v0.27.0
+    image: prom/alertmanager:v0.28.1
     volumes:
       - ./alertmanager.yml:/config/alertmanager.yml
     command:
@@ -424,7 +424,7 @@ services:
     restart: always
 
   node-exporter:
-    image: quay.io/prometheus/node-exporter:v1.7.0
+    image: quay.io/prometheus/node-exporter:v1.9.1
     container_name: node-exporter
     ports:
       - 9100:9100

docs/victoriametrics/changelog/CHANGELOG.md

@@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): enhance metrics relabel debug by adding remote write relabel configs to the relabel configs input. See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).
+
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See PR [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808) for details.
 
 ## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)

lib/mergeset/table_search_timing_test.go

@@ -94,7 +94,7 @@ func benchmarkTableSearchKeysExt(b *testing.B, tb *Table, keys [][]byte, stripSu
 				}
 				ts.Seek(searchKey)
 				if !ts.NextItem() {
-					panic(fmt.Errorf("BUG: NextItem must return true for searchKeys[%d]=%q; err=%w", i, searchKey, ts.Error()))
+					panic(fmt.Errorf("BUG: NextItem must return true for searchKeys[%d]=%q; err=%v", i, searchKey, ts.Error()))
 				}
 				if !bytes.HasPrefix(ts.Item, searchKey) {
 					panic(fmt.Errorf("BUG: unexpected item found for searchKey[%d]=%q; got %q; want %q", i, searchKey, ts.Item, key))

lib/promrelabel/debug.go

@@ -9,47 +9,50 @@ import (
 )
 
 // WriteMetricRelabelDebug writes /metric-relabel-debug page to w with the corresponding args.
-func WriteMetricRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, format string, err error) {
-	writeRelabelDebug(w, false, targetID, metric, relabelConfigs, format, err)
+func WriteMetricRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, format string, err error) {
+	writeRelabelDebug(w, false, targetID, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, format, err)
 }
 
 // WriteTargetRelabelDebug writes /target-relabel-debug page to w with the corresponding args.
 func WriteTargetRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, format string, err error) {
-	writeRelabelDebug(w, true, targetID, metric, relabelConfigs, format, err)
+	writeRelabelDebug(w, true, targetID, metric, relabelConfigs, "", 0, format, err)
 }
 
-func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, relabelConfigs, format string, err error) {
+func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, format string, err error) {
 	if metric == "" {
 		metric = "{}"
 	}
 	targetURL := ""
 	if err != nil {
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 		return
 	}
 
 	metric, err = normalizeInputLabels(metric)
 	if err != nil {
 		err = fmt.Errorf("cannot parse metric: %w", err)
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 		return
 	}
 
 	labels, err := promutil.NewLabelsFromString(metric)
 	if err != nil {
 		err = fmt.Errorf("cannot parse metric: %w", err)
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 		return
 	}
-	pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
+
+	// merge relabel configs
+	fullRelabelConfigs := relabelConfigs + "\n" + rwRelabelConfigs
+	pcs, err := ParseRelabelConfigsData([]byte(fullRelabelConfigs))
 	if err != nil {
 		err = fmt.Errorf("cannot parse relabel configs: %w", err)
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 		return
 	}
 
 	dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
-	WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, nil)
+	WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, nil)
 }
 
 func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutil.Labels, isTargetRelabel bool) ([]DebugStep, string) {

lib/promrelabel/debug.qtpl

@@ -6,15 +6,15 @@
 
 {% stripspace %}
 
-{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs  string, err error) %}
+{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) %}
   {% if format == "json" %}
-      {%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err) %}
   {% else %}
-      {%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err) %}
   {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -49,7 +49,7 @@ function submitRelabelDebugForm(e) {
 
         <div class="m-3">
         <form method="POST" onsubmit="submitRelabelDebugForm(event)">
-            {%= relabelDebugFormInputs(metric, relabelConfigs) %}
+            {%= relabelDebugFormInputs(metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel) %}
             {% if targetID != "" %}
               <input type="hidden" name="id" value="{%s targetID %}" />
             {% endif %}
@@ -70,12 +70,34 @@ function submitRelabelDebugForm(e) {
 </html>
 {% endfunc %}
 
-{% func relabelDebugFormInputs(metric, relabelConfigs string) %}
+{% func relabelDebugFormInputs(metric, relabelConfigs string, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool) %}
     <div>
         Relabel configs:<br/>
         <textarea name="relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">{%s relabelConfigs %}</textarea>
     </div>
 
+
+    {% if !isTargetRelabel %}
+    <div>
+        <div class="m-1">
+            Remote write relabel configs:
+            <div class="d-flex align-items-center gap-2 mt-1">
+                <select name="url_relabel_configs_index" class="form-select form-select-sm w-auto">
+                    <option value="">-- select remote write url --</option>
+                    {% if rwURLRelabelConfigLength > 0 %}
+                        {% for i := range rwURLRelabelConfigLength %}
+                        <option value="{%d i %}">remote-write-url-{%d i %}</option>
+                        {% endfor %}
+                    {% endif %}
+                </select>
+                <input type="submit" name="reload_url_relabel_configs" value="Reload" class="btn btn-secondary btn-sm" />
+            </div>
+        </div>
+        <textarea name="remote_write_relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">{%s rwRelabelConfigs %}</textarea>
+    </div>
+    {% endif %}
+
+
     <div>
         Labels:<br/>
         <textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">{%s metric %}</textarea>
@@ -151,7 +173,7 @@ function submitRelabelDebugForm(e) {
 {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) %}
 {
     {% if err != nil %}
         "status": "error",

lib/promrelabel/debug.qtpl.go

@@ -25,37 +25,37 @@ var (
 )
 
 //line lib/promrelabel/debug.qtpl:9
-func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) {
+func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) {
 //line lib/promrelabel/debug.qtpl:10
 	if format == "json" {
 //line lib/promrelabel/debug.qtpl:11
-		StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
+		StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 //line lib/promrelabel/debug.qtpl:12
 	} else {
 //line lib/promrelabel/debug.qtpl:13
-		StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
+		StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 //line lib/promrelabel/debug.qtpl:14
 	}
 //line lib/promrelabel/debug.qtpl:15
 }
 
 //line lib/promrelabel/debug.qtpl:15
-func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) {
+func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) {
 //line lib/promrelabel/debug.qtpl:15
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line lib/promrelabel/debug.qtpl:15
-	StreamRelabelDebugSteps(qw422016, targetURL, targetID, format, dss, metric, relabelConfigs, err)
+	StreamRelabelDebugSteps(qw422016, targetURL, targetID, format, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 //line lib/promrelabel/debug.qtpl:15
 	qt422016.ReleaseWriter(qw422016)
 //line lib/promrelabel/debug.qtpl:15
 }
 
 //line lib/promrelabel/debug.qtpl:15
-func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) string {
+func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) string {
 //line lib/promrelabel/debug.qtpl:15
 	qb422016 := qt422016.AcquireByteBuffer()
 //line lib/promrelabel/debug.qtpl:15
-	WriteRelabelDebugSteps(qb422016, targetURL, targetID, format, dss, metric, relabelConfigs, err)
+	WriteRelabelDebugSteps(qb422016, targetURL, targetID, format, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 //line lib/promrelabel/debug.qtpl:15
 	qs422016 := string(qb422016.B)
 //line lib/promrelabel/debug.qtpl:15
@@ -66,7 +66,7 @@ func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metr
 }
 
 //line lib/promrelabel/debug.qtpl:17
-func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
+func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) {
 //line lib/promrelabel/debug.qtpl:17
 	qw422016.N().S(`<!DOCTYPE html><html lang="en"><head>`)
 //line lib/promrelabel/debug.qtpl:21
@@ -120,7 +120,7 @@ func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID
 //line lib/promrelabel/debug.qtpl:48
 	qw422016.N().S(`<div class="m-3"><form method="POST" onsubmit="submitRelabelDebugForm(event)">`)
 //line lib/promrelabel/debug.qtpl:52
-	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
+	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel)
 //line lib/promrelabel/debug.qtpl:53
 	if targetID != "" {
 //line lib/promrelabel/debug.qtpl:53
@@ -153,22 +153,22 @@ func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID
 }
 
 //line lib/promrelabel/debug.qtpl:71
-func WriteRelabelDebugStepsHTML(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
+func WriteRelabelDebugStepsHTML(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) {
 //line lib/promrelabel/debug.qtpl:71
 	qw422016 := qt422016.AcquireWriter(qq422016)
 //line lib/promrelabel/debug.qtpl:71
-	StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
+	StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 //line lib/promrelabel/debug.qtpl:71
 	qt422016.ReleaseWriter(qw422016)
 //line lib/promrelabel/debug.qtpl:71
 }
 
 //line lib/promrelabel/debug.qtpl:71
-func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
+func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) string {
 //line lib/promrelabel/debug.qtpl:71
 	qb422016 := qt422016.AcquireByteBuffer()
 //line lib/promrelabel/debug.qtpl:71
-	WriteRelabelDebugStepsHTML(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
+	WriteRelabelDebugStepsHTML(qb422016, targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
 //line lib/promrelabel/debug.qtpl:71
 	qs422016 := string(qb422016.B)
 //line lib/promrelabel/debug.qtpl:71
@@ -179,326 +179,358 @@ func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric,
 }
 
 //line lib/promrelabel/debug.qtpl:73
-func streamrelabelDebugFormInputs(qw422016 *qt422016.Writer, metric, relabelConfigs string) {
+func streamrelabelDebugFormInputs(qw422016 *qt422016.Writer, metric, relabelConfigs string, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool) {
 //line lib/promrelabel/debug.qtpl:73
 	qw422016.N().S(`<div>Relabel configs:<br/><textarea name="relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">`)
 //line lib/promrelabel/debug.qtpl:76
 	qw422016.E().S(relabelConfigs)
 //line lib/promrelabel/debug.qtpl:76
-	qw422016.N().S(`</textarea></div><div>Labels:<br/><textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">`)
-//line lib/promrelabel/debug.qtpl:81
-	qw422016.E().S(metric)
-//line lib/promrelabel/debug.qtpl:81
 	qw422016.N().S(`</textarea></div>`)
-//line lib/promrelabel/debug.qtpl:83
-}
-
-//line lib/promrelabel/debug.qtpl:83
-func writerelabelDebugFormInputs(qq422016 qtio422016.Writer, metric, relabelConfigs string) {
-//line lib/promrelabel/debug.qtpl:83
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:83
-	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
-//line lib/promrelabel/debug.qtpl:83
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:83
-}
-
-//line lib/promrelabel/debug.qtpl:83
-func relabelDebugFormInputs(metric, relabelConfigs string) string {
-//line lib/promrelabel/debug.qtpl:83
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:83
-	writerelabelDebugFormInputs(qb422016, metric, relabelConfigs)
-//line lib/promrelabel/debug.qtpl:83
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:83
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:83
-	return qs422016
-//line lib/promrelabel/debug.qtpl:83
-}
-
-//line lib/promrelabel/debug.qtpl:85
-func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep, targetURL, targetID string) {
-//line lib/promrelabel/debug.qtpl:86
-	if len(dss) > 0 {
-//line lib/promrelabel/debug.qtpl:86
-		qw422016.N().S(`<div class="m-3"><b>Original labels:</b> <samp>`)
+//line lib/promrelabel/debug.qtpl:80
+	if !isTargetRelabel {
+//line lib/promrelabel/debug.qtpl:80
+		qw422016.N().S(`<div><div class="m-1">Remote write relabel configs:<div class="d-flex align-items-center gap-2 mt-1"><select name="url_relabel_configs_index" class="form-select form-select-sm w-auto"><option value="">-- select remote write url --</option>`)
+//line lib/promrelabel/debug.qtpl:87
+		if rwURLRelabelConfigLength > 0 {
 //line lib/promrelabel/debug.qtpl:88
-		streammustFormatLabels(qw422016, dss[0].In)
+			for i := range rwURLRelabelConfigLength {
 //line lib/promrelabel/debug.qtpl:88
-		qw422016.N().S(`</samp></div>`)
+				qw422016.N().S(`<option value="`)
+//line lib/promrelabel/debug.qtpl:89
+				qw422016.N().D(i)
+//line lib/promrelabel/debug.qtpl:89
+				qw422016.N().S(`">remote-write-url-`)
+//line lib/promrelabel/debug.qtpl:89
+				qw422016.N().D(i)
+//line lib/promrelabel/debug.qtpl:89
+				qw422016.N().S(`</option>`)
 //line lib/promrelabel/debug.qtpl:90
+			}
+//line lib/promrelabel/debug.qtpl:91
+		}
+//line lib/promrelabel/debug.qtpl:91
+		qw422016.N().S(`</select><input type="submit" name="reload_url_relabel_configs" value="Reload" class="btn btn-secondary btn-sm" /></div></div><textarea name="remote_write_relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">`)
+//line lib/promrelabel/debug.qtpl:96
+		qw422016.E().S(rwRelabelConfigs)
+//line lib/promrelabel/debug.qtpl:96
+		qw422016.N().S(`</textarea></div>`)
+//line lib/promrelabel/debug.qtpl:98
 	}
-//line lib/promrelabel/debug.qtpl:90
-	qw422016.N().S(`<table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Step</th><th scope="col" style="width: 25%">Relabeling Rule</th><th scope="col" style="width: 35%">Input Labels</th><th scope="col" stile="width: 35%">Output labels</a></tr></thead><tbody>`)
-//line lib/promrelabel/debug.qtpl:101
-	for i, ds := range dss {
+//line lib/promrelabel/debug.qtpl:98
+	qw422016.N().S(`<div>Labels:<br/><textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">`)
 //line lib/promrelabel/debug.qtpl:103
+	qw422016.E().S(metric)
+//line lib/promrelabel/debug.qtpl:103
+	qw422016.N().S(`</textarea></div>`)
+//line lib/promrelabel/debug.qtpl:105
+}
+
+//line lib/promrelabel/debug.qtpl:105
+func writerelabelDebugFormInputs(qq422016 qtio422016.Writer, metric, relabelConfigs string, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool) {
+//line lib/promrelabel/debug.qtpl:105
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:105
+	streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel)
+//line lib/promrelabel/debug.qtpl:105
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:105
+}
+
+//line lib/promrelabel/debug.qtpl:105
+func relabelDebugFormInputs(metric, relabelConfigs string, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool) string {
+//line lib/promrelabel/debug.qtpl:105
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:105
+	writerelabelDebugFormInputs(qb422016, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel)
+//line lib/promrelabel/debug.qtpl:105
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:105
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:105
+	return qs422016
+//line lib/promrelabel/debug.qtpl:105
+}
+
+//line lib/promrelabel/debug.qtpl:107
+func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep, targetURL, targetID string) {
+//line lib/promrelabel/debug.qtpl:108
+	if len(dss) > 0 {
+//line lib/promrelabel/debug.qtpl:108
+		qw422016.N().S(`<div class="m-3"><b>Original labels:</b> <samp>`)
+//line lib/promrelabel/debug.qtpl:110
+		streammustFormatLabels(qw422016, dss[0].In)
+//line lib/promrelabel/debug.qtpl:110
+		qw422016.N().S(`</samp></div>`)
+//line lib/promrelabel/debug.qtpl:112
+	}
+//line lib/promrelabel/debug.qtpl:112
+	qw422016.N().S(`<table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Step</th><th scope="col" style="width: 25%">Relabeling Rule</th><th scope="col" style="width: 35%">Input Labels</th><th scope="col" stile="width: 35%">Output labels</a></tr></thead><tbody>`)
+//line lib/promrelabel/debug.qtpl:123
+	for i, ds := range dss {
+//line lib/promrelabel/debug.qtpl:125
 		inLabels, inErr := promutil.NewLabelsFromString(ds.In)
 		outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
 		changedLabels := getChangedLabelNames(inLabels, outLabels)
 
-//line lib/promrelabel/debug.qtpl:106
+//line lib/promrelabel/debug.qtpl:128
 		qw422016.N().S(`<tr><td>`)
-//line lib/promrelabel/debug.qtpl:108
+//line lib/promrelabel/debug.qtpl:130
 		qw422016.N().D(i)
-//line lib/promrelabel/debug.qtpl:108
+//line lib/promrelabel/debug.qtpl:130
 		qw422016.N().S(`</td><td><b><pre class="m-2">`)
-//line lib/promrelabel/debug.qtpl:109
-		qw422016.E().S(ds.Rule)
-//line lib/promrelabel/debug.qtpl:109
-		qw422016.N().S(`</pre></b></td><td>`)
-//line lib/promrelabel/debug.qtpl:111
-		if inErr == nil {
-//line lib/promrelabel/debug.qtpl:111
-			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="deleted and updated labels highlighted in red">`)
-//line lib/promrelabel/debug.qtpl:113
-			streamlabelsWithHighlight(qw422016, inLabels, changedLabels, "#D15757")
-//line lib/promrelabel/debug.qtpl:113
-			qw422016.N().S(`</div>`)
-//line lib/promrelabel/debug.qtpl:115
-		} else {
-//line lib/promrelabel/debug.qtpl:115
-			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing input labels"><pre>`)
-//line lib/promrelabel/debug.qtpl:117
-			qw422016.E().S(inErr.Error())
-//line lib/promrelabel/debug.qtpl:117
-			qw422016.N().S(`</pre></div>`)
-//line lib/promrelabel/debug.qtpl:119
-			break
-//line lib/promrelabel/debug.qtpl:120
-		}
-//line lib/promrelabel/debug.qtpl:120
-		qw422016.N().S(`</td><td>`)
-//line lib/promrelabel/debug.qtpl:123
-		if outErr == nil {
-//line lib/promrelabel/debug.qtpl:123
-			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">`)
-//line lib/promrelabel/debug.qtpl:125
-			streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "#4495e0")
-//line lib/promrelabel/debug.qtpl:125
-			qw422016.N().S(`</div>`)
-//line lib/promrelabel/debug.qtpl:127
-		} else {
-//line lib/promrelabel/debug.qtpl:127
-			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing output labels"><pre>`)
-//line lib/promrelabel/debug.qtpl:129
-			qw422016.E().S(outErr.Error())
-//line lib/promrelabel/debug.qtpl:129
-			qw422016.N().S(`</pre></div>`)
 //line lib/promrelabel/debug.qtpl:131
+		qw422016.E().S(ds.Rule)
+//line lib/promrelabel/debug.qtpl:131
+		qw422016.N().S(`</pre></b></td><td>`)
+//line lib/promrelabel/debug.qtpl:133
+		if inErr == nil {
+//line lib/promrelabel/debug.qtpl:133
+			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="deleted and updated labels highlighted in red">`)
+//line lib/promrelabel/debug.qtpl:135
+			streamlabelsWithHighlight(qw422016, inLabels, changedLabels, "#D15757")
+//line lib/promrelabel/debug.qtpl:135
+			qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:137
+		} else {
+//line lib/promrelabel/debug.qtpl:137
+			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing input labels"><pre>`)
+//line lib/promrelabel/debug.qtpl:139
+			qw422016.E().S(inErr.Error())
+//line lib/promrelabel/debug.qtpl:139
+			qw422016.N().S(`</pre></div>`)
+//line lib/promrelabel/debug.qtpl:141
 			break
-//line lib/promrelabel/debug.qtpl:132
+//line lib/promrelabel/debug.qtpl:142
 		}
-//line lib/promrelabel/debug.qtpl:132
-		qw422016.N().S(`</td></tr>`)
-//line lib/promrelabel/debug.qtpl:135
-	}
-//line lib/promrelabel/debug.qtpl:135
-	qw422016.N().S(`</tbody></table>`)
-//line lib/promrelabel/debug.qtpl:138
-	if len(dss) > 0 {
-//line lib/promrelabel/debug.qtpl:138
-		qw422016.N().S(`<div class="m-3"><b>Resulting labels:</b> <samp>`)
-//line lib/promrelabel/debug.qtpl:140
-		streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
-//line lib/promrelabel/debug.qtpl:140
-		qw422016.N().S(`</samp>`)
-//line lib/promrelabel/debug.qtpl:141
-		if targetURL != "" {
-//line lib/promrelabel/debug.qtpl:141
-			qw422016.N().S(`<div><b>Target URL:</b>`)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(` `)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(`<a href="`)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.E().S(targetURL)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(`" target="_blank">`)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.E().S(targetURL)
-//line lib/promrelabel/debug.qtpl:143
-			qw422016.N().S(`</a>`)
-//line lib/promrelabel/debug.qtpl:144
-			if targetID != "" {
+//line lib/promrelabel/debug.qtpl:142
+		qw422016.N().S(`</td><td>`)
 //line lib/promrelabel/debug.qtpl:145
-				qw422016.N().S(` `)
+		if outErr == nil {
 //line lib/promrelabel/debug.qtpl:145
-				qw422016.N().S(`(<a href="target_response?id=`)
-//line lib/promrelabel/debug.qtpl:146
-				qw422016.E().S(targetID)
-//line lib/promrelabel/debug.qtpl:146
-				qw422016.N().S(`" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)`)
+			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">`)
 //line lib/promrelabel/debug.qtpl:147
-			}
+			streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "#4495e0")
 //line lib/promrelabel/debug.qtpl:147
 			qw422016.N().S(`</div>`)
 //line lib/promrelabel/debug.qtpl:149
-		}
+		} else {
 //line lib/promrelabel/debug.qtpl:149
-		qw422016.N().S(`</div>`)
+			qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing output labels"><pre>`)
 //line lib/promrelabel/debug.qtpl:151
+			qw422016.E().S(outErr.Error())
+//line lib/promrelabel/debug.qtpl:151
+			qw422016.N().S(`</pre></div>`)
+//line lib/promrelabel/debug.qtpl:153
+			break
+//line lib/promrelabel/debug.qtpl:154
+		}
+//line lib/promrelabel/debug.qtpl:154
+		qw422016.N().S(`</td></tr>`)
+//line lib/promrelabel/debug.qtpl:157
 	}
-//line lib/promrelabel/debug.qtpl:152
-}
-
-//line lib/promrelabel/debug.qtpl:152
-func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep, targetURL, targetID string) {
-//line lib/promrelabel/debug.qtpl:152
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:152
-	streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
-//line lib/promrelabel/debug.qtpl:152
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:152
-}
-
-//line lib/promrelabel/debug.qtpl:152
-func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) string {
-//line lib/promrelabel/debug.qtpl:152
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:152
-	writerelabelDebugSteps(qb422016, dss, targetURL, targetID)
-//line lib/promrelabel/debug.qtpl:152
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:152
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:152
-	return qs422016
-//line lib/promrelabel/debug.qtpl:152
-}
-
-//line lib/promrelabel/debug.qtpl:154
-func StreamRelabelDebugStepsJSON(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
-//line lib/promrelabel/debug.qtpl:154
-	qw422016.N().S(`{`)
-//line lib/promrelabel/debug.qtpl:156
-	if err != nil {
-//line lib/promrelabel/debug.qtpl:156
-		qw422016.N().S(`"status": "error","error":`)
-//line lib/promrelabel/debug.qtpl:158
-		qw422016.N().Q(fmt.Sprintf("Error: %s", err))
-//line lib/promrelabel/debug.qtpl:159
-	} else {
+//line lib/promrelabel/debug.qtpl:157
+	qw422016.N().S(`</tbody></table>`)
 //line lib/promrelabel/debug.qtpl:160
+	if len(dss) > 0 {
+//line lib/promrelabel/debug.qtpl:160
+		qw422016.N().S(`<div class="m-3"><b>Resulting labels:</b> <samp>`)
+//line lib/promrelabel/debug.qtpl:162
+		streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
+//line lib/promrelabel/debug.qtpl:162
+		qw422016.N().S(`</samp>`)
+//line lib/promrelabel/debug.qtpl:163
+		if targetURL != "" {
+//line lib/promrelabel/debug.qtpl:163
+			qw422016.N().S(`<div><b>Target URL:</b>`)
+//line lib/promrelabel/debug.qtpl:165
+			qw422016.N().S(` `)
+//line lib/promrelabel/debug.qtpl:165
+			qw422016.N().S(`<a href="`)
+//line lib/promrelabel/debug.qtpl:165
+			qw422016.E().S(targetURL)
+//line lib/promrelabel/debug.qtpl:165
+			qw422016.N().S(`" target="_blank">`)
+//line lib/promrelabel/debug.qtpl:165
+			qw422016.E().S(targetURL)
+//line lib/promrelabel/debug.qtpl:165
+			qw422016.N().S(`</a>`)
+//line lib/promrelabel/debug.qtpl:166
+			if targetID != "" {
+//line lib/promrelabel/debug.qtpl:167
+				qw422016.N().S(` `)
+//line lib/promrelabel/debug.qtpl:167
+				qw422016.N().S(`(<a href="target_response?id=`)
+//line lib/promrelabel/debug.qtpl:168
+				qw422016.E().S(targetID)
+//line lib/promrelabel/debug.qtpl:168
+				qw422016.N().S(`" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)`)
+//line lib/promrelabel/debug.qtpl:169
+			}
+//line lib/promrelabel/debug.qtpl:169
+			qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:171
+		}
+//line lib/promrelabel/debug.qtpl:171
+		qw422016.N().S(`</div>`)
+//line lib/promrelabel/debug.qtpl:173
+	}
+//line lib/promrelabel/debug.qtpl:174
+}
+
+//line lib/promrelabel/debug.qtpl:174
+func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep, targetURL, targetID string) {
+//line lib/promrelabel/debug.qtpl:174
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:174
+	streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
+//line lib/promrelabel/debug.qtpl:174
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:174
+}
+
+//line lib/promrelabel/debug.qtpl:174
+func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) string {
+//line lib/promrelabel/debug.qtpl:174
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:174
+	writerelabelDebugSteps(qb422016, dss, targetURL, targetID)
+//line lib/promrelabel/debug.qtpl:174
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:174
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:174
+	return qs422016
+//line lib/promrelabel/debug.qtpl:174
+}
+
+//line lib/promrelabel/debug.qtpl:176
+func StreamRelabelDebugStepsJSON(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) {
+//line lib/promrelabel/debug.qtpl:176
+	qw422016.N().S(`{`)
+//line lib/promrelabel/debug.qtpl:178
+	if err != nil {
+//line lib/promrelabel/debug.qtpl:178
+		qw422016.N().S(`"status": "error","error":`)
+//line lib/promrelabel/debug.qtpl:180
+		qw422016.N().Q(fmt.Sprintf("Error: %s", err))
+//line lib/promrelabel/debug.qtpl:181
+	} else {
+//line lib/promrelabel/debug.qtpl:182
 		var hasError bool
 
-//line lib/promrelabel/debug.qtpl:160
+//line lib/promrelabel/debug.qtpl:182
 		qw422016.N().S(`"status": "success","steps": [`)
-//line lib/promrelabel/debug.qtpl:163
+//line lib/promrelabel/debug.qtpl:185
 		for i, ds := range dss {
-//line lib/promrelabel/debug.qtpl:165
+//line lib/promrelabel/debug.qtpl:187
 			inLabels, inErr := promutil.NewLabelsFromString(ds.In)
 			outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
 			changedLabels := getChangedLabelNames(inLabels, outLabels)
 
-//line lib/promrelabel/debug.qtpl:168
+//line lib/promrelabel/debug.qtpl:190
 			qw422016.N().S(`{"inLabels":`)
-//line lib/promrelabel/debug.qtpl:170
-			qw422016.N().Q(labelsWithHighlight(inLabels, changedLabels, "#D15757"))
-//line lib/promrelabel/debug.qtpl:170
-			qw422016.N().S(`,"outLabels":`)
-//line lib/promrelabel/debug.qtpl:171
-			qw422016.N().Q(labelsWithHighlight(outLabels, changedLabels, "#4495e0"))
-//line lib/promrelabel/debug.qtpl:171
-			qw422016.N().S(`,"rule":`)
-//line lib/promrelabel/debug.qtpl:172
-			qw422016.N().Q(ds.Rule)
-//line lib/promrelabel/debug.qtpl:172
-			qw422016.N().S(`,"errors": {`)
-//line lib/promrelabel/debug.qtpl:174
-			if inErr != nil {
-//line lib/promrelabel/debug.qtpl:174
-				qw422016.N().S(`"inLabels":`)
-//line lib/promrelabel/debug.qtpl:175
-				qw422016.N().Q(`<span style="color: #D15757">` + inErr.Error() + `</span>`)
-//line lib/promrelabel/debug.qtpl:175
-				if outErr != nil {
-//line lib/promrelabel/debug.qtpl:175
-					qw422016.N().S(`,`)
-//line lib/promrelabel/debug.qtpl:175
-				}
-//line lib/promrelabel/debug.qtpl:176
-				hasError = true
-
-//line lib/promrelabel/debug.qtpl:177
-			} else {
-//line lib/promrelabel/debug.qtpl:178
-			}
-//line lib/promrelabel/debug.qtpl:179
-			if outErr != nil {
-//line lib/promrelabel/debug.qtpl:179
-				qw422016.N().S(`"outLabels":`)
-//line lib/promrelabel/debug.qtpl:180
-				qw422016.N().Q(`<span style="color: #D15757">` + outErr.Error() + `</span>`)
-//line lib/promrelabel/debug.qtpl:181
-				hasError = true
-
-//line lib/promrelabel/debug.qtpl:182
-			}
-//line lib/promrelabel/debug.qtpl:182
-			qw422016.N().S(`}}`)
-//line lib/promrelabel/debug.qtpl:185
-			if i != len(dss)-1 {
-//line lib/promrelabel/debug.qtpl:185
-				qw422016.N().S(`,`)
-//line lib/promrelabel/debug.qtpl:185
-			}
-//line lib/promrelabel/debug.qtpl:186
-		}
-//line lib/promrelabel/debug.qtpl:186
-		qw422016.N().S(`]`)
-//line lib/promrelabel/debug.qtpl:188
-		if len(dss) > 0 && !hasError {
-//line lib/promrelabel/debug.qtpl:188
-			qw422016.N().S(`,"originalLabels":`)
-//line lib/promrelabel/debug.qtpl:190
-			qw422016.N().Q(mustFormatLabels(dss[0].In))
-//line lib/promrelabel/debug.qtpl:190
-			qw422016.N().S(`,"resultingLabels":`)
-//line lib/promrelabel/debug.qtpl:191
-			qw422016.N().Q(mustFormatLabels(dss[len(dss)-1].Out))
 //line lib/promrelabel/debug.qtpl:192
-		}
+			qw422016.N().Q(labelsWithHighlight(inLabels, changedLabels, "#D15757"))
+//line lib/promrelabel/debug.qtpl:192
+			qw422016.N().S(`,"outLabels":`)
 //line lib/promrelabel/debug.qtpl:193
-	}
+			qw422016.N().Q(labelsWithHighlight(outLabels, changedLabels, "#4495e0"))
 //line lib/promrelabel/debug.qtpl:193
-	qw422016.N().S(`}`)
-//line lib/promrelabel/debug.qtpl:195
-}
-
-//line lib/promrelabel/debug.qtpl:195
-func WriteRelabelDebugStepsJSON(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
-//line lib/promrelabel/debug.qtpl:195
-	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:195
-	StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
-//line lib/promrelabel/debug.qtpl:195
-	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:195
-}
-
-//line lib/promrelabel/debug.qtpl:195
-func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
-//line lib/promrelabel/debug.qtpl:195
-	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:195
-	WriteRelabelDebugStepsJSON(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
-//line lib/promrelabel/debug.qtpl:195
-	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:195
-	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:195
-	return qs422016
-//line lib/promrelabel/debug.qtpl:195
-}
-
+			qw422016.N().S(`,"rule":`)
+//line lib/promrelabel/debug.qtpl:194
+			qw422016.N().Q(ds.Rule)
+//line lib/promrelabel/debug.qtpl:194
+			qw422016.N().S(`,"errors": {`)
+//line lib/promrelabel/debug.qtpl:196
+			if inErr != nil {
+//line lib/promrelabel/debug.qtpl:196
+				qw422016.N().S(`"inLabels":`)
 //line lib/promrelabel/debug.qtpl:197
-func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
+				qw422016.N().Q(`<span style="color: #D15757">` + inErr.Error() + `</span>`)
+//line lib/promrelabel/debug.qtpl:197
+				if outErr != nil {
+//line lib/promrelabel/debug.qtpl:197
+					qw422016.N().S(`,`)
+//line lib/promrelabel/debug.qtpl:197
+				}
+//line lib/promrelabel/debug.qtpl:198
+				hasError = true
+
 //line lib/promrelabel/debug.qtpl:199
+			} else {
+//line lib/promrelabel/debug.qtpl:200
+			}
+//line lib/promrelabel/debug.qtpl:201
+			if outErr != nil {
+//line lib/promrelabel/debug.qtpl:201
+				qw422016.N().S(`"outLabels":`)
+//line lib/promrelabel/debug.qtpl:202
+				qw422016.N().Q(`<span style="color: #D15757">` + outErr.Error() + `</span>`)
+//line lib/promrelabel/debug.qtpl:203
+				hasError = true
+
+//line lib/promrelabel/debug.qtpl:204
+			}
+//line lib/promrelabel/debug.qtpl:204
+			qw422016.N().S(`}}`)
+//line lib/promrelabel/debug.qtpl:207
+			if i != len(dss)-1 {
+//line lib/promrelabel/debug.qtpl:207
+				qw422016.N().S(`,`)
+//line lib/promrelabel/debug.qtpl:207
+			}
+//line lib/promrelabel/debug.qtpl:208
+		}
+//line lib/promrelabel/debug.qtpl:208
+		qw422016.N().S(`]`)
+//line lib/promrelabel/debug.qtpl:210
+		if len(dss) > 0 && !hasError {
+//line lib/promrelabel/debug.qtpl:210
+			qw422016.N().S(`,"originalLabels":`)
+//line lib/promrelabel/debug.qtpl:212
+			qw422016.N().Q(mustFormatLabels(dss[0].In))
+//line lib/promrelabel/debug.qtpl:212
+			qw422016.N().S(`,"resultingLabels":`)
+//line lib/promrelabel/debug.qtpl:213
+			qw422016.N().Q(mustFormatLabels(dss[len(dss)-1].Out))
+//line lib/promrelabel/debug.qtpl:214
+		}
+//line lib/promrelabel/debug.qtpl:215
+	}
+//line lib/promrelabel/debug.qtpl:215
+	qw422016.N().S(`}`)
+//line lib/promrelabel/debug.qtpl:217
+}
+
+//line lib/promrelabel/debug.qtpl:217
+func WriteRelabelDebugStepsJSON(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) {
+//line lib/promrelabel/debug.qtpl:217
+	qw422016 := qt422016.AcquireWriter(qq422016)
+//line lib/promrelabel/debug.qtpl:217
+	StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
+//line lib/promrelabel/debug.qtpl:217
+	qt422016.ReleaseWriter(qw422016)
+//line lib/promrelabel/debug.qtpl:217
+}
+
+//line lib/promrelabel/debug.qtpl:217
+func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs, rwRelabelConfigs string, rwURLRelabelConfigLength int, isTargetRelabel bool, err error) string {
+//line lib/promrelabel/debug.qtpl:217
+	qb422016 := qt422016.AcquireByteBuffer()
+//line lib/promrelabel/debug.qtpl:217
+	WriteRelabelDebugStepsJSON(qb422016, targetURL, targetID, dss, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigLength, isTargetRelabel, err)
+//line lib/promrelabel/debug.qtpl:217
+	qs422016 := string(qb422016.B)
+//line lib/promrelabel/debug.qtpl:217
+	qt422016.ReleaseByteBuffer(qb422016)
+//line lib/promrelabel/debug.qtpl:217
+	return qs422016
+//line lib/promrelabel/debug.qtpl:217
+}
+
+//line lib/promrelabel/debug.qtpl:219
+func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
+//line lib/promrelabel/debug.qtpl:221
 	labelsList := labels.GetLabels()
 	metricName := ""
 	for i, label := range labelsList {
@@ -509,153 +541,153 @@ func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Label
 		}
 	}
 
-//line lib/promrelabel/debug.qtpl:209
+//line lib/promrelabel/debug.qtpl:231
 	if metricName != "" {
-//line lib/promrelabel/debug.qtpl:210
+//line lib/promrelabel/debug.qtpl:232
 		if _, ok := highlight["__name__"]; ok {
-//line lib/promrelabel/debug.qtpl:210
+//line lib/promrelabel/debug.qtpl:232
 			qw422016.N().S(`<span style="font-weight:bold;color:`)
-//line lib/promrelabel/debug.qtpl:211
+//line lib/promrelabel/debug.qtpl:233
 			qw422016.E().S(color)
-//line lib/promrelabel/debug.qtpl:211
+//line lib/promrelabel/debug.qtpl:233
 			qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:211
+//line lib/promrelabel/debug.qtpl:233
 			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:211
+//line lib/promrelabel/debug.qtpl:233
 			qw422016.N().S(`</span>`)
-//line lib/promrelabel/debug.qtpl:212
+//line lib/promrelabel/debug.qtpl:234
 		} else {
-//line lib/promrelabel/debug.qtpl:213
+//line lib/promrelabel/debug.qtpl:235
 			qw422016.E().S(metricName)
-//line lib/promrelabel/debug.qtpl:214
+//line lib/promrelabel/debug.qtpl:236
 		}
-//line lib/promrelabel/debug.qtpl:215
+//line lib/promrelabel/debug.qtpl:237
 		if len(labelsList) == 0 {
-//line lib/promrelabel/debug.qtpl:215
+//line lib/promrelabel/debug.qtpl:237
 			return
-//line lib/promrelabel/debug.qtpl:215
+//line lib/promrelabel/debug.qtpl:237
 		}
-//line lib/promrelabel/debug.qtpl:216
+//line lib/promrelabel/debug.qtpl:238
 	}
-//line lib/promrelabel/debug.qtpl:216
+//line lib/promrelabel/debug.qtpl:238
 	qw422016.N().S(`{`)
-//line lib/promrelabel/debug.qtpl:218
+//line lib/promrelabel/debug.qtpl:240
 	for i, label := range labelsList {
-//line lib/promrelabel/debug.qtpl:219
+//line lib/promrelabel/debug.qtpl:241
 		if _, ok := highlight[label.Name]; ok {
-//line lib/promrelabel/debug.qtpl:219
+//line lib/promrelabel/debug.qtpl:241
 			qw422016.N().S(`<span style="font-weight:bold;color:`)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:242
 			qw422016.E().S(color)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:242
 			qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:242
 			qw422016.E().S(label.Name)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:242
 			qw422016.N().S(`=`)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:242
 			qw422016.E().Q(label.Value)
-//line lib/promrelabel/debug.qtpl:220
+//line lib/promrelabel/debug.qtpl:242
 			qw422016.N().S(`</span>`)
-//line lib/promrelabel/debug.qtpl:221
+//line lib/promrelabel/debug.qtpl:243
 		} else {
-//line lib/promrelabel/debug.qtpl:222
+//line lib/promrelabel/debug.qtpl:244
 			qw422016.E().S(label.Name)
-//line lib/promrelabel/debug.qtpl:222
+//line lib/promrelabel/debug.qtpl:244
 			qw422016.N().S(`=`)
-//line lib/promrelabel/debug.qtpl:222
+//line lib/promrelabel/debug.qtpl:244
 			qw422016.E().Q(label.Value)
-//line lib/promrelabel/debug.qtpl:223
+//line lib/promrelabel/debug.qtpl:245
 		}
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:246
 		if i < len(labelsList)-1 {
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:246
 			qw422016.N().S(`,`)
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:246
 			qw422016.N().S(` `)
-//line lib/promrelabel/debug.qtpl:224
+//line lib/promrelabel/debug.qtpl:246
 		}
-//line lib/promrelabel/debug.qtpl:225
+//line lib/promrelabel/debug.qtpl:247
 	}
-//line lib/promrelabel/debug.qtpl:225
+//line lib/promrelabel/debug.qtpl:247
 	qw422016.N().S(`}`)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 }
 
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	streamlabelsWithHighlight(qw422016, labels, highlight, color)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 }
 
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 func labelsWithHighlight(labels *promutil.Labels, highlight map[string]struct{}, color string) string {
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	writelabelsWithHighlight(qb422016, labels, highlight, color)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 	return qs422016
-//line lib/promrelabel/debug.qtpl:227
+//line lib/promrelabel/debug.qtpl:249
 }
 
-//line lib/promrelabel/debug.qtpl:229
+//line lib/promrelabel/debug.qtpl:251
 func streammustFormatLabels(qw422016 *qt422016.Writer, s string) {
-//line lib/promrelabel/debug.qtpl:230
+//line lib/promrelabel/debug.qtpl:252
 	labels, err := promutil.NewLabelsFromString(s)
 
-//line lib/promrelabel/debug.qtpl:231
+//line lib/promrelabel/debug.qtpl:253
 	if err != nil {
-//line lib/promrelabel/debug.qtpl:231
+//line lib/promrelabel/debug.qtpl:253
 		qw422016.N().S(`<span style="color: red" title="error parsing labels:`)
-//line lib/promrelabel/debug.qtpl:232
+//line lib/promrelabel/debug.qtpl:254
 		qw422016.E().S(err.Error())
-//line lib/promrelabel/debug.qtpl:232
+//line lib/promrelabel/debug.qtpl:254
 		qw422016.N().S(`">`)
-//line lib/promrelabel/debug.qtpl:232
+//line lib/promrelabel/debug.qtpl:254
 		qw422016.E().S("error parsing labels: " + err.Error())
-//line lib/promrelabel/debug.qtpl:232
+//line lib/promrelabel/debug.qtpl:254
 		qw422016.N().S(`</span>`)
-//line lib/promrelabel/debug.qtpl:233
+//line lib/promrelabel/debug.qtpl:255
 	} else {
-//line lib/promrelabel/debug.qtpl:234
+//line lib/promrelabel/debug.qtpl:256
 		streamlabelsWithHighlight(qw422016, labels, nil, "")
-//line lib/promrelabel/debug.qtpl:235
+//line lib/promrelabel/debug.qtpl:257
 	}
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 }
 
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 func writemustFormatLabels(qq422016 qtio422016.Writer, s string) {
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	streammustFormatLabels(qw422016, s)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	qt422016.ReleaseWriter(qw422016)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 }
 
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 func mustFormatLabels(s string) string {
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	qb422016 := qt422016.AcquireByteBuffer()
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	writemustFormatLabels(qb422016, s)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	qs422016 := string(qb422016.B)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	qt422016.ReleaseByteBuffer(qb422016)
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 	return qs422016
-//line lib/promrelabel/debug.qtpl:236
+//line lib/promrelabel/debug.qtpl:258
 }

lib/promrelabel/debug_test.go

@@ -13,7 +13,7 @@ func TestWriteRelabelDebugSupportFormats(t *testing.T) {
 	f := func(input, rule, expect string) {
 		// execute
 		outputWriter := bytes.NewBuffer(nil)
-		writeRelabelDebug(outputWriter, false, "", input, rule, "json", nil)
+		writeRelabelDebug(outputWriter, false, "", input, rule, "", 0, "json", nil)
 
 		// the response is in JSON with HTML content, extract the `resultingLabels` in JSON and unescape it.
 		resultingLabels := fastjson.GetString(outputWriter.Bytes(), `resultingLabels`)

lib/promscrape/relabel_debug.go

@@ -3,34 +3,84 @@ package promscrape
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page
-func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
+// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page.
+// remotewrite-related relabel configs could be empty as vmsingle doesn't provide remote write feature.
+func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request, rwGlobalRelabelConfigs string, rwURLRelabelConfigss []string) {
 	targetID := r.FormValue("id")
 	metric := r.FormValue("metric")
 	relabelConfigs := r.FormValue("relabel_configs")
+	rwRelabelConfigs := r.FormValue("remote_write_relabel_configs") // global + per-URL configs.
+
+	rwURLRelabelConfigsIdx := r.FormValue("url_relabel_configs_index")     // only for per-URL configs and has to be set with reload_url_relabel_configs.
+	reloadRWURLRelabelConfigs := r.FormValue("reload_url_relabel_configs") // if set, it will reset the whole remote_write_relabel_configs.
+
 	format := r.FormValue("format")
 	var err error
 
-	if metric == "" && relabelConfigs == "" && targetID != "" {
+	rwURLRelabelConfigsLength := len(rwURLRelabelConfigss)
+
+	// if everything is not set, we should load the initial data for user.
+	if metric == "" && relabelConfigs == "" && rwRelabelConfigs == "" && rwURLRelabelConfigsIdx == "" && reloadRWURLRelabelConfigs == "" && targetID != "" {
 		pcs, labels, ok := getMetricRelabelContextByTargetID(targetID)
 		if !ok {
 			err = fmt.Errorf("cannot find target for id=%s", targetID)
 			targetID = ""
 		} else {
 			metric = labels.String()
-			relabelConfigs = pcs.String()
+			relabelConfigs += pcs.String()
+
+			// by default use the first per-URL remote write relabel config, if exists.
+			rwURLRelabelConfigs := ""
+			if len(rwURLRelabelConfigss) > 0 {
+				rwURLRelabelConfigs = rwURLRelabelConfigss[0]
+			}
+
+			if rwGlobalRelabelConfigs != "" {
+				rwRelabelConfigs += "\n# -remoteWrite.relabelConfig"
+				rwRelabelConfigs += "\n" + rwGlobalRelabelConfigs
+			}
+			if rwURLRelabelConfigs != "" {
+				rwRelabelConfigs += "\n# -remoteWrite.urlRelabelConfig"
+				rwRelabelConfigs += "\n" + rwURLRelabelConfigs
+			}
 		}
 	}
+
+	// if reloadRWURLRelabelConfigs is set, it means user clicked the button and want to reload the rwRelabelConfigs by rwURLRelabelConfigsIdx
+	if reloadRWURLRelabelConfigs != "" {
+		// set the per-URL remote write relabel according to index, any error will fall back the index to 0.
+		rwURLRelabelConfigs := ""
+		if len(rwURLRelabelConfigss) > 0 {
+			// ignore the error if the input is invalid or exceed the length, and fallback to 0.
+			idx, _ := strconv.Atoi(rwURLRelabelConfigsIdx)
+			if idx < 0 || idx >= len(rwURLRelabelConfigss) {
+				idx = 0
+			}
+			rwURLRelabelConfigs = rwURLRelabelConfigss[idx]
+		}
+
+		// reload will remove the existing content
+		if rwGlobalRelabelConfigs != "" {
+			rwRelabelConfigs = "\n# -remoteWrite.relabelConfig"
+			rwRelabelConfigs += "\n" + rwGlobalRelabelConfigs
+		}
+		if rwURLRelabelConfigs != "" {
+			rwRelabelConfigs += "\n# -remoteWrite.urlRelabelConfig"
+			rwRelabelConfigs += "\n" + rwURLRelabelConfigs
+		}
+	}
+
 	if format == "json" {
 		httpserver.EnableCORS(w, r)
 		w.Header().Set("Content-Type", "application/json")
 	}
-	promrelabel.WriteMetricRelabelDebug(w, targetID, metric, relabelConfigs, format, err)
+	promrelabel.WriteMetricRelabelDebug(w, targetID, metric, relabelConfigs, rwRelabelConfigs, rwURLRelabelConfigsLength, format, err)
 }
 
 // WriteTargetRelabelDebug generates response for /target-relabel-debug page