lib/metricnamestats: add new matchNames stats query filter

Introduce new query filter option matchNames for metricnamestats query Requests.
It allows to fetch stats for exact metric names. Which is useful for Explore Cardinality page.
It's also allows 3rd party tool to check if application metrics are used
for query requests.

 This commit adds the following changes:
* replaces MetricNamesUsageStats inline func query args with dedicated struct.
* bumps cluster RPC metricNamesUsageStats_v1 to metricNamesUsageStats_v2
  in order to properly encode new RequestQuery params
* adds new methods for MetricNameTracker

Related issue:
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6145

app/vmselect/netstorage/netstorage.go

@@ -1369,10 +1369,11 @@ func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
 const maxFastAllocBlockSize = 32 * 1024
 
 // GetMetricNamesStats returns statistic for timeseries metric names usage.
-func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
-	qt = qt.NewChild("get metric names usage statistics with limit: %d, less or equal to: %d, match pattern=%q", limit, le, matchPattern)
+func GetMetricNamesStats(qt *querytracer.Tracer, statsQuery storage.MetricNamesStatsQuery) (storage.MetricNamesStatsResponse, error) {
+	qt = qt.NewChild("get metric names usage statistics with limit: %d, less or equal to: %d, match pattern=%q,match_names_len=%d",
+		statsQuery.Limit, statsQuery.Le, statsQuery.MatchPattern, len(statsQuery.MatchNames))
 	defer qt.Done()
-	return vmstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	return vmstorage.GetMetricNamesStats(qt, statsQuery)
 }
 
 // ResetMetricNamesStats resets state of metric names usage

app/vmselect/stats/stats.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
 )
 
 // MetricNamesStatsHandler returns timeseries metric names usage statistics
@@ -33,7 +34,18 @@ func MetricNamesStatsHandler(qt *querytracer.Tracer, w http.ResponseWriter, r *h
 		le = n
 	}
 	matchPattern := r.FormValue("match_pattern")
-	stats, err := netstorage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	matchNames := r.Form["match_names"]
+	statsQuery := storage.MetricNamesStatsQuery{
+		Limit:        limit,
+		Le:           le,
+		MatchNames:   matchNames,
+		MatchPattern: matchPattern,
+	}
+	if limit > 0 && len(matchNames) > limit {
+		return fmt.Errorf("match_names len=%d cannot exceed limit=%d", len(matchNames), limit)
+	}
+
+	stats, err := netstorage.GetMetricNamesStats(qt, statsQuery)
 	if err != nil {
 		return err
 	}

app/vmstorage/main.go

@@ -200,9 +200,9 @@ func DeleteSeries(qt *querytracer.Tracer, tfss []*storage.TagFilters, maxMetrics
 }
 
 // GetMetricNamesStats returns metric names usage stats with give limit and lte predicate
-func GetMetricNamesStats(qt *querytracer.Tracer, limit, le int, matchPattern string) (storage.MetricNamesStatsResponse, error) {
+func GetMetricNamesStats(qt *querytracer.Tracer, statsQuery storage.MetricNamesStatsQuery) (storage.MetricNamesStatsResponse, error) {
 	WG.Add(1)
-	r := Storage.GetMetricNamesStats(qt, limit, le, matchPattern)
+	r := Storage.GetMetricNamesStats(qt, statsQuery)
 	WG.Done()
 	return r, nil
 }

docs/victoriametrics/README.md

@@ -483,6 +483,7 @@ To get metric names usage statistics, use the `/prometheus/api/v1/status/metric_
 * `limit` - integer value to limit the number of metric names in response. By default, API returns 1000 records.
 * `le` -  `less than or equal`, is an integer threshold for filtering metric names by their usage count in queries. For example, with `?le=1` API returns metric names that were queried <=1 times.
 * `match_pattern` - a substring pattern to match metric names. For example, `?match_pattern=vm_` will match any metric names with `vm_` pattern, like `vm_http_requests`, `max_vm_memory_available`. It doesn't support regex syntax.
+* `match_names` - a list of metric names to query. If this param is defined, `le` and `match_pattern` options will be ignored.
 
  The API endpoint returns the following `JSON` response:
 

docs/victoriametrics/changelog/CHANGELOG.md

@@ -18,9 +18,12 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 
 ## tip
 
+** update note 1: change vmselect to vmstorage cluster RPC method from `metricNamesUsageStats_v1` to `metricNamesUsageStats_v2`
+
 * FEATURE: all the VictoriaMetrics components: mask `authKey` value from log messages. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5973) for details.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), [vmagent](https://docs.victoriametrics.com/vmagent/): add helpful hints to the unexpected EOF error message in the write concurrency limiter. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8704) for details.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): use [VM remote write protocol](https://docs.victoriametrics.com/vmagent/#victoriametrics-remote-write-protocol) by default with automatic downgrade in runtime to Prometheus protocol when needed. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8462) for details.
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): enhance  - `/api/v1/status/metric_names_stats` [API](https://docs.victoriametrics.com/keyconcepts/#structure-of-a-metric) with `match_names` query param. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6145) for details and related [docs](https://docs.victoriametrics.com/#track-ingested-metrics-usage)
 
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): properly init [enterprise](https://docs.victoriametrics.com/enterprise/) version for `linux/arm` and non-CGO buids. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6019) for details.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/): remote write client sets correct content encoding header based on actual body content, rather than relying on configuration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/8650).

lib/storage/metricnamestats/tracker.go

@@ -391,6 +391,71 @@ func (mt *Tracker) GetStatsForTenant(accountID, projectID uint32, limit, le int,
 	return result
 }
 
+// GetStatsForNamesMultitenant returns stats for metric names
+// ignores accountID and projectID
+//
+// SingleNode version must use GetStatsForNamesTenant with 0 account and project IDs
+func (mt *Tracker) GetStatsForNamesMultitenant(names []string) StatsResult {
+	var result StatsResult
+
+	result.CollectedSinceTs = mt.creationTs.Load()
+	result.TotalRecords = mt.currentItemsCount.Load()
+	result.MaxSizeBytes = mt.maxSizeBytes
+	result.CurrentSizeBytes = mt.currentSizeBytes.Load()
+
+	mt.mu.RLock()
+	for sk, si := range mt.store {
+		for _, mn := range names {
+			if mn == sk.metricName {
+				result.Records = append(result.Records, StatRecord{
+					MetricName:    sk.metricName,
+					RequestsCount: si.requestsCount.Load(),
+					LastRequestTs: si.lastRequestTs.Load(),
+				})
+				break
+			}
+		}
+	}
+	mt.mu.RUnlock()
+
+	result.sort()
+	result.DeduplicateMergeRecords()
+	return result
+}
+
+// GetStatsForNamesTenant returns stats for given accountID and projectID
+func (mt *Tracker) GetStatsForNamesTenant(accountID, projectID uint32, names []string) StatsResult {
+	var result StatsResult
+
+	result.CollectedSinceTs = mt.creationTs.Load()
+	result.TotalRecords = mt.currentItemsCount.Load()
+	result.MaxSizeBytes = mt.maxSizeBytes
+	result.CurrentSizeBytes = mt.currentSizeBytes.Load()
+
+	mt.mu.RLock()
+	for _, mn := range names {
+		sk := statKey{
+			accountID:  accountID,
+			projectID:  projectID,
+			metricName: mn,
+		}
+		si, ok := mt.store[sk]
+		if !ok {
+			continue
+		}
+		result.Records = append(result.Records, StatRecord{
+			MetricName:    sk.metricName,
+			RequestsCount: si.requestsCount.Load(),
+			LastRequestTs: si.lastRequestTs.Load(),
+		})
+	}
+
+	mt.mu.RUnlock()
+
+	result.sort()
+	return result
+}
+
 // GetStats returns stats response for the tracked metrics
 //
 // DeduplicateMergeRecords must be called at cluster version on returned result.

lib/storage/metricnamestats/tracker_test.go

@@ -562,3 +562,60 @@ func TestStatsResultMerge(t *testing.T) {
 	f(dst, src, expected)
 
 }
+
+func TestStatsForMetricNames(t *testing.T) {
+	type testOp struct {
+		o  byte
+		mg string
+	}
+
+	umt, err := loadFrom(t.TempDir()+t.Name(), storeOverhead+10*2)
+	if err != nil {
+		t.Fatalf("cannot load tracker: %s", err)
+	}
+	umt.getCurrentTs = func() uint64 { return 1 }
+	ops := []testOp{
+		{'i', "metric_1"},
+		{'r', "metric_2"},
+		{'r', "metric_1"},
+		{'i', "metric_2"},
+		{'i', "metric_3"},
+		{'i', "metric_4"},
+		{'r', "metric_1"},
+		{'r', "metric_2"},
+		{'r', "metric_2"},
+		{'r', "metric_2"},
+	}
+	for _, op := range ops {
+		switch op.o {
+		case 'i':
+			umt.RegisterIngestRequest(0, 0, []byte(op.mg))
+		case 'r':
+			umt.RegisterQueryRequest(0, 0, []byte(op.mg))
+		}
+	}
+
+	f := func(names []string, expected StatsResult) {
+		t.Helper()
+		got := umt.GetStatsForNamesTenant(0, 0, names)
+		if d := cmp.Diff(expected, got, statsResultCmpOpts); len(d) > 0 {
+			t.Fatalf("unexpected deduplicate result: %s", d)
+		}
+	}
+	want := StatsResult{
+		TotalRecords: 2,
+		Records: []StatRecord{
+			{MetricName: "metric_1", RequestsCount: 2, LastRequestTs: 1},
+		},
+	}
+	f([]string{"metric_1", "metric"}, want)
+	want = StatsResult{
+		TotalRecords: 2,
+		Records: []StatRecord{
+			{MetricName: "metric_1", RequestsCount: 2, LastRequestTs: 1},
+			{MetricName: "metric_2", RequestsCount: 3, LastRequestTs: 1},
+		},
+	}
+	f([]string{"metric_1", "metric_2"}, want)
+
+}

lib/storage/storage.go

@@ -2957,9 +2957,21 @@ func (s *Storage) wasMetricIDMissingBefore(metricID uint64) bool {
 // MetricNamesStatsResponse contains metric names usage stats API response
 type MetricNamesStatsResponse = metricnamestats.StatsResult
 
-// GetMetricNamesStats returns metric names usage stats with given limit and le predicate
-func (s *Storage) GetMetricNamesStats(_ *querytracer.Tracer, limit, le int, matchPattern string) MetricNamesStatsResponse {
-	return s.metricsTracker.GetStats(limit, le, matchPattern)
+// MetricNamesStatsQuery represents query params for Stats requests
+
+type MetricNamesStatsQuery struct {
+	Limit        int
+	Le           int
+	MatchPattern string
+	MatchNames   []string
+}
+
+// GetMetricNamesStats returns metric names usage stats for given Query params
+func (s *Storage) GetMetricNamesStats(_ *querytracer.Tracer, statsQuery MetricNamesStatsQuery) MetricNamesStatsResponse {
+	if len(statsQuery.MatchNames) > 0 {
+		return s.metricsTracker.GetStatsForNamesTenant(0, 0, statsQuery.MatchNames)
+	}
+	return s.metricsTracker.GetStats(statsQuery.Limit, statsQuery.Le, statsQuery.MatchPattern)
 }
 
 // ResetMetricNamesStats resets state for metric names usage tracker

lib/storage/storage_test.go

@@ -3262,9 +3262,11 @@ func TestStorageMetricTracker(t *testing.T) {
 		MinTimestamp: minTimestamp,
 		MaxTimestamp: maxTimestamp,
 	}
-
+	statsQuery := MetricNamesStatsQuery{
+		Limit: 10_000,
+	}
 	// check stats for metrics with 0 requests count
-	mus := s.GetMetricNamesStats(nil, 10_000, 0, "")
+	mus := s.GetMetricNamesStats(nil, statsQuery)
 	if len(mus.Records) != int(numRows) {
 		t.Fatalf("unexpected Stats records count=%d, want %d records", len(mus.Records), numRows)
 	}
@@ -3280,11 +3282,12 @@ func TestStorageMetricTracker(t *testing.T) {
 	}
 	sr.MustClose()
 
-	mus = s.GetMetricNamesStats(nil, 10_000, 0, "")
+	mus = s.GetMetricNamesStats(nil, statsQuery)
 	if len(mus.Records) != 0 {
 		t.Fatalf("unexpected Stats records count=%d; want 0 records", len(mus.Records))
 	}
-	mus = s.GetMetricNamesStats(nil, 10_000, 1, "")
+	statsQuery.Le = 1
+	mus = s.GetMetricNamesStats(nil, statsQuery)
 	if len(mus.Records) != int(numRows) {
 		t.Fatalf("unexpected Stats records count=%d, want %d records", len(mus.Records), numRows)
 	}