app/vmctl/remote-read: allow providing multiple label filters

Previously, vmctl only accepted one label for filtering. Extend this to
allow providing multiple-filters at once. This is useful when migrating
large volumes of data as it allows narrowing down migration scope of
migration for one run so that the source side is not overwhelmed with
migration.

Related PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9917/

app/vmctl/flags.go

@@ -689,15 +689,15 @@ var (
 			Usage:  "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
 			Layout: time.RFC3339,
 		},
-		&cli.StringFlag{
-			Name:  remoteReadFilterLabel,
-			Usage: "Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
-			Value: "__name__",
+		&cli.StringSliceFlag{
+			Name:        remoteReadFilterLabel,
+			Usage:       "Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
+			DefaultText: "__name__",
 		},
-		&cli.StringFlag{
-			Name:  remoteReadFilterLabelValue,
-			Usage: fmt.Sprintf("Prometheus regular expression to filter label from %q flag.", remoteReadFilterLabelValue),
-			Value: ".*",
+		&cli.StringSliceFlag{
+			Name:        remoteReadFilterLabelValue,
+			Usage:       fmt.Sprintf("Prometheus regular expression to filter label from %q flag.", remoteReadFilterLabelValue),
+			DefaultText: ".*",
 		},
 		&cli.BoolFlag{
 			Name:  remoteRead,

app/vmctl/main.go

@@ -192,6 +192,14 @@ func main() {
 						return fmt.Errorf("failed to create transport for -%s=%q: %s", remoteReadSrcAddr, addr, err)
 					}
 
+					// Backwards compatible default values if none provided by user
+					rrLabelNames := c.StringSlice(remoteReadFilterLabel)
+					rrLabelValues := c.StringSlice(remoteReadFilterLabelValue)
+					if len(rrLabelNames) == 0 && len(rrLabelValues) == 0 {
+						rrLabelNames = []string{"__name__"}
+						rrLabelValues = []string{".*"}
+					}
+
 					rr, err := remoteread.NewClient(remoteread.Config{
 						Addr:              addr,
 						Transport:         tr,
@@ -200,8 +208,8 @@ func main() {
 						Timeout:           c.Duration(remoteReadHTTPTimeout),
 						UseStream:         c.Bool(remoteReadUseStream),
 						Headers:           c.String(remoteReadHeaders),
-						LabelName:         c.String(remoteReadFilterLabel),
-						LabelValue:        c.String(remoteReadFilterLabelValue),
+						LabelNames:        rrLabelNames,
+						LabelValues:       rrLabelValues,
 						DisablePathAppend: c.Bool(remoteReadDisablePathAppend),
 					})
 					if err != nil {

app/vmctl/remoteread/remoteread.go

@@ -11,14 +11,15 @@ import (
 	"strings"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/storage/remote"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 )
 
 const (
@@ -63,9 +64,9 @@ type Config struct {
 	UseStream bool
 	// Headers optional HTTP headers to send with each request to the corresponding remote storage
 	Headers string
-	// LabelName, LabelValue stands for label=~value pair used for read requests.
+	// LabelNames, LabelValues stands for label=~value pair used for read requests.
 	// Is optional.
-	LabelName, LabelValue string
+	LabelNames, LabelValues []string
 }
 
 // Filter defines a list of filters applied to requested data
@@ -94,12 +95,22 @@ func NewClient(cfg Config) (*Client, error) {
 		return nil, err
 	}
 
-	var m *prompb.LabelMatcher
-	if cfg.LabelName != "" && cfg.LabelValue != "" {
-		m = &prompb.LabelMatcher{
-			Type:  prompb.LabelMatcher_RE,
-			Name:  cfg.LabelName,
-			Value: cfg.LabelValue,
+	var matchers []*prompb.LabelMatcher
+	if len(cfg.LabelNames) > 0 || len(cfg.LabelValues) > 0 {
+		if len(cfg.LabelNames) != len(cfg.LabelValues) {
+			return nil, fmt.Errorf("the number of label names and label values must be the same")
+		}
+
+		for i := range cfg.LabelNames {
+			if cfg.LabelNames[i] == "" {
+				return nil, fmt.Errorf("label name cannot be empty")
+			}
+			matcher := &prompb.LabelMatcher{
+				Type:  prompb.LabelMatcher_RE,
+				Name:  cfg.LabelNames[i],
+				Value: cfg.LabelValues[i],
+			}
+			matchers = append(matchers, matcher)
 		}
 	}
 
@@ -116,7 +127,7 @@ func NewClient(cfg Config) (*Client, error) {
 		password:          cfg.Password,
 		useStream:         cfg.UseStream,
 		headers:           headers,
-		matchers:          []*prompb.LabelMatcher{m},
+		matchers:          matchers,
 	}
 
 	return c, nil

docs/victoriametrics/changelog/CHANGELOG.md

@@ -28,6 +28,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 * FEATURE: `vminsert` and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce new RPC protocol for insert-storage communication. See this PR [#9820](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9820) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): explicitly check response type for [range queries](https://docs.victoriametrics.com/keyConcepts.html#range-query) during [replay](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling) and return error on type mismatch. This change should reduce confusions like in [#9779](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9779).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): allow providing multiple filters for [remote-read migration mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) via multiple `--remote-read-filter-label` and `--remote-read-filter-label-value` flags. This is useful in order to narrow down the data being migrated by using more precise filters. See this PR [#9917](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9917) for details.
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support `alert_relabel_configs` per each notifier in `-notifier.config` file. See [#5980](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5980).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add template function `now` to return the Unix timestamp in seconds at the time of the template evaluation. For example, `{{ (now | toTime).Sub $activeAt }}` can calculate duration the alert has been active. See this issue [#9864](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9864) for more details. Thank you @nguu0123 for the pull request.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `/remotewrite-relabel-config` and `/api/v1/status/remotewrite-relabel-config` API handlers for returning content of `--remoteWrite.relabelConfig` cmd-line flag. Useful for inspecting and verifying the final global relabeling rules used by vmagent. Access to new handlers can be protected via `--configAuthKey` command-line flag. See [#9504](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9504).

docs/victoriametrics/vmctl/remoteread.md

@@ -47,6 +47,15 @@ Filtering by labels can be configured via flags `--remote-read-filter-label` and
 For example, `--remote-read-filter-label=tenant` and `--remote-read-filter-label-value="team-eu"` will select only series
 with `tenant="team-eu"` label-value pair.
 
+Filtering flags can be provided multiple times {{% available_from "#" %}} to narrow down the selection of timeseries to migrate.
+For example:
+```sh
+./vmctl remote-read \
+    --remote-read-filter-label=tenant --remote-read-filter-label-value="team-eu" \
+    --remote-read-filter-label=__name__ --remote-read-filter-label-value="cpu_.*"
+```
+will select only timeseries with `tenant="team-eu"` label and metric names matching `cpu_.*` regex.
+
 ## Configuration 
 
 Migrating big volumes of data may result in remote read client reaching the timeout. Increase the value of