relabel debug: support multiple lines metrics input

app/vmagent/remotewrite/relabel.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/metrics"
 	"gopkg.in/yaml.v2"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,30 +83,58 @@ func WriteRelabelConfigData(w io.Writer) {
 	_, _ = w.Write(*p)
 }
 
+// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
+func GetRemoteWriteRelabelConfigString() string {
+	var bb bytesutil.ByteBuffer
+	WriteRelabelConfigData(&bb)
+	if bb.Len() == 0 {
+		return ""
+	}
+	return string(bb.B)
+}
+
+type UrlRelabelCfg struct {
+	Url           string `yaml:"url"`
+	RelabelConfig any    `yaml:"relabel_config"`
+
+	RelabelConfigStr string
+}
+
 // WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
 func WriteURLRelabelConfigData(w io.Writer) {
-	p := remoteWriteURLRelabelConfigData.Load()
-	if p == nil {
+	cs := GetURLRelabelConfigData()
+	if cs == nil {
 		// Nothing to write to w
 		return
 	}
-	type urlRelabelCfg struct {
-		Url           string `yaml:"url"`
-		RelabelConfig any    `yaml:"relabel_config"`
+	d, _ := yaml.Marshal(cs)
+	_, _ = w.Write(d)
+}
+
+// GetURLRelabelConfigData is similar to WriteURLRelabelConfigData but returning data in []UrlRelabelCfg.
+func GetURLRelabelConfigData() []UrlRelabelCfg {
+	p := remoteWriteURLRelabelConfigData.Load()
+	if p == nil {
+		return nil
 	}
-	var cs []urlRelabelCfg
+	var cs []UrlRelabelCfg
 	for i, url := range *remoteWriteURLs {
 		cfgData := (*p)[i]
+		var cfgDataBytes []byte
+		if cfgData != nil {
+			cfgDataBytes, _ = yaml.Marshal(cfgData)
+		}
 		if !*showRemoteWriteURL {
 			url = fmt.Sprintf("%d:secret-url", i+1)
 		}
-		cs = append(cs, urlRelabelCfg{
+		cs = append(cs, UrlRelabelCfg{
 			Url:           url,
 			RelabelConfig: cfgData,
+
+			RelabelConfigStr: string(cfgDataBytes),
 		})
 	}
-	d, _ := yaml.Marshal(cs)
-	_, _ = w.Write(d)
+	return cs
 }
 
 func reloadRelabelConfigs() {

docs/victoriametrics/changelog/CHANGELOG.md

@@ -30,6 +30,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `access_log` configuration option for each user that will log requests to stdout, and support filtering by HTTP status codes. See more in [docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). See [#5936](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5936).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support negative values for the group `eval_offset` option, which allows starting group evaluation at `groupInterval-abs(eval_offset)` within `[0...groupInterval]`. See [#10424](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10424).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Disable `/graphite/tags/tagSeries` and `/graphite/tags/tagMultiSeries` for Graphite tag registration since it is unlikely it is used in context of VictoriaMetrics. See [10544](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10544).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): enhance metrics relabel debug by adding remote write relabel configs to the relabel configs input. See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918). 
 
 * BUGFIX: [dashboards/vmauth](https://grafana.com/grafana/dashboards/21394): fix `requested from system` and `heap inuse` expressions in the memory usage panel. See [#10574](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10574).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not enable ACL when uploading backups to S3-compatible endpoints by default. ACL is not always supported by S3-compatible endpoints and it is not recommended to use ACLs to limit access to objects. See [#10539](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10539) for more details.

lib/promrelabel/debug.go

@@ -35,12 +35,6 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 		return
 	}
 
-	labels, err := promutil.NewLabelsFromString(metric)
-	if err != nil {
-		err = fmt.Errorf("cannot parse metric: %w", err)
-		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
-		return
-	}
 	pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
 	if err != nil {
 		err = fmt.Errorf("cannot parse relabel configs: %w", err)
@@ -48,8 +42,33 @@ func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, rela
 		return
 	}
 
-	dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
-	WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, nil)
+	// metric input may contain multiple lines, and it's good to add support and print the debug steps for each of them.
+	// however, the target URL won't change
+	metrics := strings.Split(metric, "\n")
+	dsss := make([][]DebugStep, 0, len(metrics))
+	var lastErr error
+
+	for _, m := range metrics {
+		// try to trim all the special characters and space first
+		m = strings.TrimSpace(m)
+		labels, err := promutil.NewLabelsFromString(m)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		dss, tURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
+		targetURL = tURL
+		dsss = append(dsss, dss)
+	}
+
+	// break here if all failed
+	if len(dsss) == 0 && lastErr != nil {
+		err = fmt.Errorf("cannot parse metric: %w", err)
+		WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, lastErr)
+		return
+	}
+
+	WriteRelabelDebugSteps(w, targetURL, targetID, format, dsss, metric, relabelConfigs, nil)
 }
 
 func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutil.Labels, isTargetRelabel bool) ([]DebugStep, string) {

lib/promrelabel/debug.qtpl

@@ -6,15 +6,15 @@
 
 {% stripspace %}
 
-{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs  string, err error) %}
+{% func RelabelDebugSteps(targetURL, targetID, format string, dsss [][]DebugStep, metric string, relabelConfigs  string, err error) %}
   {% if format == "json" %}
-      {%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsJSON(targetURL, targetID, dsss, metric, relabelConfigs, err) %}
   {% else %}
-      {%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, err) %}
+      {%= RelabelDebugStepsHTML(targetURL, targetID, dsss, metric, relabelConfigs, err) %}
   {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsHTML(targetURL, targetID string, dsss [][]DebugStep, metric string, relabelConfigs string, err error) %}
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -60,11 +60,13 @@ function submitRelabelDebugForm(e) {
         </form>
         </div>
 
+        {% for _, dss := range dsss %}
         <div class="row">
             <main class="col-12">
               {%= relabelDebugSteps(dss, targetURL, targetID) %}
             </main>
         </div>
+        {% endfor %}
     </div>
 </body>
 </html>
@@ -77,7 +79,7 @@ function submitRelabelDebugForm(e) {
     </div>
 
     <div>
-        Labels:<br/>
+        Labels (One label set per line. Multiple lines will be treated as multiple label sets):<br/>
         <textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">{%s metric %}</textarea>
     </div>
 {% endfunc %}
@@ -151,45 +153,52 @@ function submitRelabelDebugForm(e) {
 {% endif %}
 {% endfunc %}
 
-{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
+{% func RelabelDebugStepsJSON(targetURL, targetID string, dsss [][]DebugStep, metric string, relabelConfigs string, err error) %}
 {
     {% if err != nil %}
         "status": "error",
         "error": {%q= fmt.Sprintf("Error: %s", err) %}
     {% else %}
-        {% code var hasError bool %}
         "status": "success",
-        "steps": [
-            {% for i, ds := range dss %}
-                {% code
-                    inLabels, inErr := promutil.NewLabelsFromString(ds.In)
-                    outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
-                    changedLabels := getChangedLabelNames(inLabels, outLabels)
-                %}
+        "metrics": [
+            {% for idx, dss := range dsss %}
+                {% code var hasError bool %}
                 {
-                    "inLabels": {%q= labelsWithHighlight(inLabels, changedLabels, "#D15757") %},
-                    "outLabels": {%q= labelsWithHighlight(outLabels, changedLabels, "#4495e0") %},
-                    "rule": {%q= ds.Rule %},
-                    "errors": {
-                        {% if inErr != nil %}
-                        "inLabels": {%q= `<span style="color: #D15757">`+inErr.Error()+`</span>` %}{% if outErr != nil %},{% endif %}
-                        {%code hasError = true %}
-                        {% else %}
-                        {% endif %}
-                        {% if outErr != nil %}
-                        "outLabels": {%q= `<span style="color: #D15757">`+outErr.Error()+`</span>` %}
-                        {%code hasError = true %}
-                        {% endif %}
-                    }
+                    "steps": [
+                        {% for i, ds := range dss %}
+                            {% code
+                                inLabels, inErr := promutil.NewLabelsFromString(ds.In)
+                                outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
+                                changedLabels := getChangedLabelNames(inLabels, outLabels)
+                            %}
+                            {
+                                "inLabels": {%q= labelsWithHighlight(inLabels, changedLabels, "#D15757") %},
+                                "outLabels": {%q= labelsWithHighlight(outLabels, changedLabels, "#4495e0") %},
+                                "rule": {%q= ds.Rule %},
+                                "errors": {
+                                    {% if inErr != nil %}
+                                    "inLabels": {%q= `<span style="color: #D15757">`+inErr.Error()+`</span>` %}{% if outErr != nil %},{% endif %}
+                                    {%code hasError = true %}
+                                    {% else %}
+                                    {% endif %}
+                                    {% if outErr != nil %}
+                                    "outLabels": {%q= `<span style="color: #D15757">`+outErr.Error()+`</span>` %}
+                                    {%code hasError = true %}
+                                    {% endif %}
+                                }
+                            }
+                            {% if i != len(dss)-1 %},{% endif %}
+                        {% endfor %}
+                    ]
+                    {% if len(dss) > 0 && !hasError %}
+                    ,
+                      "originalLabels": {%q= mustFormatLabels(dss[0].In) %},
+                      "resultingLabels": {%q= mustFormatLabels(dss[len(dss)-1].Out) %}
+                    {% endif %}
                 }
-                {% if i != len(dss)-1 %},{% endif %}
+                {% if idx != len(dsss)-1 %},{% endif %}
             {% endfor %}
         ]
-        {% if len(dss) > 0 && !hasError %}
-        ,
-          "originalLabels": {%q= mustFormatLabels(dss[0].In) %},
-          "resultingLabels": {%q= mustFormatLabels(dss[len(dss)-1].Out) %}
-        {% endif %}
     {% endif %}
 }
 {% endfunc %}

lib/promscrape/relabel_debug.go

@@ -3,7 +3,9 @@ package promscrape
 import (
 	"fmt"
 	"net/http"
+	"strings"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmagent/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
@@ -23,9 +25,39 @@ func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
 			targetID = ""
 		} else {
 			metric = labels.String()
-			relabelConfigs = pcs.String()
+			relabelConfigs += "# metrics_relabel_configs\n"
+			relabelConfigs += pcs.String()
+
+			rwRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
+			rwURLRelabelConfigs := remotewrite.GetURLRelabelConfigData()
+
+			relabelConfigs += "\n# -remoteWrite.relabelConfig"
+			relabelConfigs += "\n" + rwRelabelConfigs
+
+			// we could have different relabel config for different remote write URL, but there's no way to know which one the user wants to debug.
+			// so we append the 1st one here, and comment out the rest. user can see them on the page and edit to activate them.
+			for i := range rwURLRelabelConfigs {
+				if i == 0 {
+					relabelConfigs += "\n# -remoteWrite.urlRelabelConfig"
+
+					// append the URL info
+					relabelConfigs += "\n# " + rwURLRelabelConfigs[i].Url
+
+					// append the relabeling config string
+					relabelConfigs += "\n" + rwURLRelabelConfigs[i].RelabelConfigStr
+					continue
+				}
+
+				// for the rest URLs add comment # before every line.
+				relabelConfigs += "\n# " + rwURLRelabelConfigs[i].Url
+				lines := strings.Split(rwURLRelabelConfigs[i].RelabelConfigStr, "\n")
+				for _, line := range lines {
+					relabelConfigs += "\n#" + line
+				}
+			}
 		}
 	}
+
 	if format == "json" {
 		httpserver.EnableCORS(w, r)
 		w.Header().Set("Content-Type", "application/json")