app/{vmagent,vmsingle}: properly init relabel and stream aggr config reload related metrics

Previously  `*_last_reload_successful` metrics are set to 1 even when respective
configuration is not defined, besides stream aggregation and scraping.

 This commit initializes config reload related metrics only when respective
configuration is set.

Related issue:
https://github.com/VictoriaMetrics/helm-charts/issues/2119

app/vmagent/remotewrite/relabel.go

@@ -7,10 +7,13 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
+
+	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -31,12 +34,59 @@ var (
 
 var labelsGlobal []prompbmarshal.Label
 
+var (
+	relabelConfigReloads      *metrics.Counter
+	relabelConfigReloadErrors *metrics.Counter
+	relabelConfigSuccess      *metrics.Gauge
+	relabelConfigTimestamp    *metrics.Counter
+)
+
+func initRelabelMetrics() {
+	relabelConfigReloads = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
+	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
+	relabelConfigSuccess = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
+	relabelConfigTimestamp = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
+}
+
 // CheckRelabelConfigs checks -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig.
 func CheckRelabelConfigs() error {
 	_, err := loadRelabelConfigs()
 	return err
 }
 
+func initRelabelConfigs() {
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		logger.Fatalf("cannot initialize relabel configs: %s", err)
+	}
+	allRelabelConfigs.Store(rcs)
+	if rcs.isSet() {
+		initRelabelMetrics()
+		relabelConfigSuccess.Set(1)
+		relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	}
+}
+
+func reloadRelabelConfigs() {
+	rcs := allRelabelConfigs.Load()
+	if !rcs.isSet() {
+		return
+	}
+	relabelConfigReloads.Inc()
+	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
+	rcs, err := loadRelabelConfigs()
+	if err != nil {
+		relabelConfigReloadErrors.Inc()
+		relabelConfigSuccess.Set(0)
+		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
+		return
+	}
+	allRelabelConfigs.Store(rcs)
+	relabelConfigSuccess.Set(1)
+	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	logger.Infof("successfully reloaded relabel configs")
+}
+
 func loadRelabelConfigs() (*relabelConfigs, error) {
 	var rcs relabelConfigs
 	if *relabelConfigPathGlobal != "" {
@@ -70,6 +120,21 @@ type relabelConfigs struct {
 	perURL []*promrelabel.ParsedConfigs
 }
 
+func (rcs *relabelConfigs) isSet() bool {
+	if rcs != nil {
+		return false
+	}
+	if rcs.global.Len() > 0 {
+		return true
+	}
+	for _, pc := range rcs.perURL {
+		if pc.Len() > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 // initLabelsGlobal must be called after parsing command-line flags.
 func initLabelsGlobal() {
 	labelsGlobal = nil

app/vmagent/remotewrite/remotewrite.go

@@ -16,7 +16,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -197,13 +196,7 @@ func Init() {
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240
 	sighupCh := procutil.NewSighupChan()
 
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		logger.Fatalf("cannot load relabel configs: %s", err)
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
+	initRelabelConfigs()
 
 	initStreamAggrConfigGlobal()
 
@@ -269,29 +262,6 @@ func dropDanglingQueues() {
 	}
 }
 
-func reloadRelabelConfigs() {
-	relabelConfigReloads.Inc()
-	logger.Infof("reloading relabel configs pointed by -remoteWrite.relabelConfig and -remoteWrite.urlRelabelConfig")
-	rcs, err := loadRelabelConfigs()
-	if err != nil {
-		relabelConfigReloadErrors.Inc()
-		relabelConfigSuccess.Set(0)
-		logger.Errorf("cannot reload relabel configs; preserving the previous configs; error: %s", err)
-		return
-	}
-	allRelabelConfigs.Store(rcs)
-	relabelConfigSuccess.Set(1)
-	relabelConfigTimestamp.Set(fasttime.UnixTimestamp())
-	logger.Infof("successfully reloaded relabel configs")
-}
-
-var (
-	relabelConfigReloads      = metrics.NewCounter(`vmagent_relabel_config_reloads_total`)
-	relabelConfigReloadErrors = metrics.NewCounter(`vmagent_relabel_config_reloads_errors_total`)
-	relabelConfigSuccess      = metrics.NewGauge(`vmagent_relabel_config_last_reload_successful`, nil)
-	relabelConfigTimestamp    = metrics.NewCounter(`vmagent_relabel_config_last_reload_success_timestamp_seconds`)
-)
-
 func initRemoteWriteCtxs(urls []string) {
 	if len(urls) == 0 {
 		logger.Panicf("BUG: urls must be non-empty")

app/vminsert/common/streamaggr.go

@@ -45,10 +45,10 @@ var (
 	saCfgReloaderStopCh chan struct{}
 	saCfgReloaderWG     sync.WaitGroup
 
-	saCfgReloads   = metrics.NewCounter(`vminsert_streamagg_config_reloads_total`)
-	saCfgReloadErr = metrics.NewCounter(`vminsert_streamagg_config_reloads_errors_total`)
-	saCfgSuccess   = metrics.NewGauge(`vminsert_streamagg_config_last_reload_successful`, nil)
-	saCfgTimestamp = metrics.NewCounter(`vminsert_streamagg_config_last_reload_success_timestamp_seconds`)
+	saCfgReloads   *metrics.Counter
+	saCfgReloadErr *metrics.Counter
+	saCfgSuccess   *metrics.Gauge
+	saCfgTimestamp *metrics.Counter
 
 	sasGlobal    atomic.Pointer[streamaggr.Aggregators]
 	deduplicator *streamaggr.Deduplicator
@@ -87,6 +87,11 @@ func InitStreamAggr() {
 		return
 	}
 
+	saCfgReloads = metrics.NewCounter(`vminsert_streamagg_config_reloads_total`)
+	saCfgReloadErr = metrics.NewCounter(`vminsert_streamagg_config_reloads_errors_total`)
+	saCfgSuccess = metrics.NewGauge(`vminsert_streamagg_config_last_reload_successful`, nil)
+	saCfgTimestamp = metrics.NewCounter(`vminsert_streamagg_config_last_reload_success_timestamp_seconds`)
+
 	sighupCh := procutil.NewSighupChan()
 
 	opts := &streamaggr.Options{

app/vminsert/relabel/relabel.go

@@ -34,13 +34,20 @@ func Init() {
 	if err != nil {
 		logger.Fatalf("cannot load relabelConfig: %s", err)
 	}
-	pcsGlobal.Store(pcs)
-	configSuccess.Set(1)
-	configTimestamp.Set(fasttime.UnixTimestamp())
 
 	if len(*relabelConfig) == 0 {
 		return
 	}
+
+	configReloads = metrics.NewCounter(`vm_relabel_config_reloads_total`)
+	configReloadErrors = metrics.NewCounter(`vm_relabel_config_reloads_errors_total`)
+	configSuccess = metrics.NewGauge(`vm_relabel_config_last_reload_successful`, nil)
+	configTimestamp = metrics.NewCounter(`vm_relabel_config_last_reload_success_timestamp_seconds`)
+
+	pcsGlobal.Store(pcs)
+	configSuccess.Set(1)
+	configTimestamp.Set(fasttime.UnixTimestamp())
+
 	go func() {
 		for range sighupCh {
 			configReloads.Inc()
@@ -61,10 +68,10 @@ func Init() {
 }
 
 var (
-	configReloads      = metrics.NewCounter(`vm_relabel_config_reloads_total`)
-	configReloadErrors = metrics.NewCounter(`vm_relabel_config_reloads_errors_total`)
-	configSuccess      = metrics.NewGauge(`vm_relabel_config_last_reload_successful`, nil)
-	configTimestamp    = metrics.NewCounter(`vm_relabel_config_last_reload_success_timestamp_seconds`)
+	configReloads      *metrics.Counter
+	configReloadErrors *metrics.Counter
+	configSuccess      *metrics.Gauge
+	configTimestamp    *metrics.Counter
 )
 
 var pcsGlobal atomic.Pointer[promrelabel.ParsedConfigs]

docs/victoriametrics/changelog/CHANGELOG.md

@@ -29,6 +29,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * BUGFIX: [vmalert-tool](https://docs.victoriametrics.com/victoriametrics/vmalert-tool/): fix parsing for (+/-)Inf values and scientific notation in `values` field. Thanks to @evkuzin for [#8847](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8847).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): use `retentionFilter` flag name in debugging interface to make it consistent with flag definition. Previously, flag name in debugging interface was different from command-line configuration so copying command-line flags for debugging produced an error. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8697).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix various UI glitches and tidy up the visual styles. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8773) for details.
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): create relabel config reload and stream aggregation config reload related metrics only if config is present. See [helm-charts#2119](https://github.com/VictoriaMetrics/helm-charts/issues/2119) for details.
 
 ## [v1.116.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.116.0)