lib/storage: properly report dateMetricIDCache stats

A number of changes to `dateMetricIDCache` stats and configuration:

1. Export `SizeMaxBytes` metric and make the size configurable via a
flag
2. Fix `EntriesCount` and `SizeBytes` stats. Previously the cache
reported this stats for its immutable part only. Whereas there are cases
when the number of entries in its mutable part is comparable with the
number in immutable part. The stats from the mutable part remains
invisible until it is sync'ed to the immutable part. It is also possible
that the cache gets reset after the sync because the cache size exceeds
the max allowed size. Reporting the stats for both mutable and immutable
parts should provide a clear picture of the cache utilization.

Together, SizeBytes and SizeMaxBytes should enable tracking the cache
utilization properly. And take appropriate actions if necessary (such as
adjusting the memory resources and/or cache size limit via a flag).

Related issue https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10064

app/vmstorage/main.go

@@ -75,6 +75,8 @@ var (
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
 	cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+
 		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
+	cacheSizeIndexDBDateMetricID = flagutil.NewBytes("storage.cacheSizeIndexDBDateMetricID", 0, "Overrides max size for indexdb/date_metricID cache. "+
+		"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
 
 	disablePerDayIndex = flag.Bool("disablePerDayIndex", false, "Disable per-day index and use global index for all searches. "+
 		"This may improve performance and decrease disk space usage for the use cases with fixed set of timeseries scattered across a "+
@@ -125,6 +127,7 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
 	storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
 	storage.SetMetadataStorageSize(metadataStorageSize.IntN())
+	storage.SetDateMetricIDCacheSize(cacheSizeIndexDBDateMetricID.IntN())
 	mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
 	mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
 	mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())
@@ -634,13 +637,13 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/tsid"}`, m.TSIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricIDs"}`, m.MetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/metricName"}`, m.MetricNameCacheSize)
-	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/indexBlocks"}`, tm.IndexBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSize)
+	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSize)
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexps"}`, uint64(storage.RegexpCacheSize()))
 	metrics.WriteGaugeUint64(w, `vm_cache_entries{type="storage/regexpPrefixes"}`, uint64(storage.RegexpPrefixesCacheSize()))
@@ -652,9 +655,9 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeBytes)
-	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/hour_metric_ids"}`, m.HourMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/next_day_metric_ids"}`, m.NextDayMetricIDCacheSizeBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexps"}`, storage.RegexpCacheSizeBytes())
 	metrics.WriteGaugeUint64(w, `vm_cache_size_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheSizeBytes())
@@ -666,6 +669,7 @@ func writeStorageMetrics(w io.Writer, strg *storage.Storage) {
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocks"}`, idbm.DataBlocksCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/dataBlocksSparse"}`, idbm.DataBlocksSparseCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/indexBlocks"}`, idbm.IndexBlocksCacheSizeMaxBytes)
+	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/date_metricID"}`, idbm.DateMetricIDCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="indexdb/tagFiltersToMetricIDs"}`, idbm.TagFiltersToMetricIDsCacheSizeMaxBytes)
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexps"}`, storage.RegexpCacheMaxSizeBytes())
 	metrics.WriteGaugeUint64(w, `vm_cache_size_max_bytes{type="storage/regexpPrefixes"}`, storage.RegexpPrefixesCacheMaxSizeBytes())

lib/storage/date_metric_id_cache.go

@@ -9,13 +9,24 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
 )
 
+var maxDateMetricIDCacheSize uint64
+
+// SetDateMetricIDCacheSize overrides the default size of dateMetricIDCache
+func SetDateMetricIDCacheSize(size int) {
+	maxDateMetricIDCacheSize = uint64(size)
+}
+
+func getDateMetricIDCacheSize() uint64 {
+	if maxDateMetricIDCacheSize <= 0 {
+		return uint64(float64(memory.Allowed()) / 256)
+	}
+	return maxDateMetricIDCacheSize
+}
+
 // dateMetricIDCache is fast cache for holding (date, metricID) entries.
 //
 // It should be faster than map[date]*uint64set.Set on multicore systems.
 type dateMetricIDCache struct {
-	syncsCount  atomic.Uint64
-	resetsCount atomic.Uint64
-
 	// Contains immutable map
 	byDate atomic.Pointer[byDateMetricIDMap]
 
@@ -27,6 +38,12 @@ type dateMetricIDCache struct {
 	// Protected by mu.
 	slowHits int
 
+	// Protected by mu.
+	syncsCount uint64
+
+	// Protected by mu.
+	resetsCount uint64
+
 	mu sync.Mutex
 }
 
@@ -42,25 +59,38 @@ func (dmc *dateMetricIDCache) resetLocked() {
 	dmc.byDateMutable = newByDateMetricIDMap()
 	dmc.slowHits = 0
 
-	dmc.resetsCount.Add(1)
+	dmc.resetsCount++
 }
 
-func (dmc *dateMetricIDCache) EntriesCount() int {
-	byDate := dmc.byDate.Load()
-	n := 0
-	for _, metricIDs := range byDate.m {
-		n += metricIDs.Len()
-	}
-	return n
+type dateMetricIDCacheStats struct {
+	Size         uint64
+	SizeBytes    uint64
+	SizeMaxBytes uint64
+	ResetsCount  uint64
+	SyncsCount   uint64
 }
 
-func (dmc *dateMetricIDCache) SizeBytes() uint64 {
-	byDate := dmc.byDate.Load()
-	n := uint64(0)
-	for _, metricIDs := range byDate.m {
-		n += metricIDs.SizeBytes()
+func (dmc *dateMetricIDCache) Stats() dateMetricIDCacheStats {
+	s := dateMetricIDCacheStats{
+		SizeMaxBytes: getDateMetricIDCacheSize(),
 	}
-	return n
+
+	dmc.mu.Lock()
+	defer dmc.mu.Unlock()
+
+	for _, metricIDs := range dmc.byDate.Load().m {
+		s.Size += uint64(metricIDs.Len())
+		s.SizeBytes += metricIDs.SizeBytes()
+	}
+	for _, metricIDs := range dmc.byDateMutable.m {
+		s.Size += uint64(metricIDs.Len())
+		s.SizeBytes += metricIDs.SizeBytes()
+	}
+
+	s.ResetsCount = dmc.resetsCount
+	s.SyncsCount = dmc.syncsCount
+
+	return s
 }
 
 func (dmc *dateMetricIDCache) Has(date, metricID uint64) bool {
@@ -163,13 +193,18 @@ func (dmc *dateMetricIDCache) syncLocked() {
 		}
 	}
 
+	var sizeBytes uint64
+	for _, v := range dmc.byDateMutable.m {
+		sizeBytes += v.SizeBytes()
+	}
+
 	// Atomically replace byDate with byDateMutable
 	dmc.byDate.Store(dmc.byDateMutable)
 	dmc.byDateMutable = newByDateMetricIDMap()
 
-	dmc.syncsCount.Add(1)
+	dmc.syncsCount++
 
-	if dmc.SizeBytes() > uint64(memory.Allowed())/256 {
+	if sizeBytes > getDateMetricIDCacheSize() {
 		dmc.resetLocked()
 	}
 }

lib/storage/date_metric_id_cache_test.go

@@ -5,6 +5,9 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
 )
 
 func TestDateMetricIDCacheSerial(t *testing.T) {
@@ -85,13 +88,13 @@ func testDateMetricIDCache(c *dateMetricIDCache, concurrent bool) error {
 	}
 
 	// Verify c.Reset
-	if n := c.EntriesCount(); !concurrent && n < 123 {
+	if n := c.Stats().Size; !concurrent && n < 123 {
 		return fmt.Errorf("c.EntriesCount must return at least 123; returned %d", n)
 	}
 	c.mu.Lock()
 	c.resetLocked()
 	c.mu.Unlock()
-	if n := c.EntriesCount(); !concurrent && n > 0 {
+	if n := c.Stats().Size; !concurrent && n > 0 {
 		return fmt.Errorf("c.EntriesCount must return 0 after reset; returned %d", n)
 	}
 	return nil
@@ -120,3 +123,85 @@ func TestDateMetricIDCacheIsConsistent(_ *testing.T) {
 	}
 	wg.Wait()
 }
+
+func TestDateMetricIDCache_SizeMaxBytes(t *testing.T) {
+	defer SetDateMetricIDCacheSize(0)
+
+	assertSizeMaxBytes := func(dmc *dateMetricIDCache, want uint64) {
+		t.Helper()
+		if got := dmc.Stats().SizeMaxBytes; got != want {
+			t.Fatalf("unexpected sizeMaxBytes: got %d, want %d", got, want)
+		}
+	}
+
+	defaultSizeMaxBytes := uint64(float64(memory.Allowed()) / 256)
+	var dmc *dateMetricIDCache
+
+	// Default.
+	dmc = newDateMetricIDCache()
+	assertSizeMaxBytes(dmc, defaultSizeMaxBytes)
+
+	// Overriden.
+	SetDateMetricIDCacheSize(1024)
+	dmc = newDateMetricIDCache()
+	assertSizeMaxBytes(dmc, 1024)
+
+	// Overriden at runtime.
+	SetDateMetricIDCacheSize(2048)
+	assertSizeMaxBytes(dmc, 2048)
+
+	// Reset to default at runtime.
+	SetDateMetricIDCacheSize(0)
+	assertSizeMaxBytes(dmc, defaultSizeMaxBytes)
+}
+
+func TestDateMetricIDCache_Size(t *testing.T) {
+	dmc := newDateMetricIDCache()
+	for i := range 100_000 {
+		date := 12345 + uint64(i%30)
+		metricID := uint64(i)
+		dmc.Set(date, metricID)
+
+		if got, want := dmc.Stats().Size, uint64(i+1); got != want {
+			t.Fatalf("unexpected size: got %d, want %d", got, want)
+		}
+	}
+
+	// Retrieve all entries and check the cache size again.
+	for i := range 100_000 {
+		date := 12345 + uint64(i%30)
+		metricID := uint64(i)
+		if !dmc.Has(date, metricID) {
+			t.Fatalf("entry not in cache: (date=%d, metricID=%d)", date, metricID)
+		}
+	}
+	if got, want := dmc.Stats().Size, uint64(100_000); got != want {
+		t.Fatalf("unexpected size: got %d, want %d", got, want)
+	}
+}
+
+func TestDateMetricIDCache_SizeBytes(t *testing.T) {
+	dmc := newDateMetricIDCache()
+	metricIDs := &uint64set.Set{}
+	for i := range 100_000 {
+		date := uint64(123)
+		metricID := uint64(i)
+		metricIDs.Add(metricID)
+		dmc.Set(date, metricID)
+	}
+	if got, want := dmc.Stats().SizeBytes, metricIDs.SizeBytes(); got != want {
+		t.Fatalf("unexpected sizeBytes: got %d, want %d", got, want)
+	}
+
+	// Retrieve all entries and check the cache sizeBytes again.
+	for i := range 100_000 {
+		date := uint64(123)
+		metricID := uint64(i)
+		if !dmc.Has(date, metricID) {
+			t.Fatalf("entry not in cache: (date=%d, metricID=%d)", date, metricID)
+		}
+	}
+	if got, want := dmc.Stats().SizeBytes, metricIDs.SizeBytes(); got != want {
+		t.Fatalf("unexpected sizeBytes: got %d, want %d", got, want)
+	}
+}

lib/storage/index_db.go

@@ -202,10 +202,11 @@ type IndexDBMetrics struct {
 	TagFiltersToMetricIDsCacheMisses       uint64
 	TagFiltersToMetricIDsCacheResets       uint64
 
-	DateMetricIDCacheSize        uint64
-	DateMetricIDCacheSizeBytes   uint64
-	DateMetricIDCacheSyncsCount  uint64
-	DateMetricIDCacheResetsCount uint64
+	DateMetricIDCacheSize         uint64
+	DateMetricIDCacheSizeBytes    uint64
+	DateMetricIDCacheSizeMaxBytes uint64
+	DateMetricIDCacheSyncsCount   uint64
+	DateMetricIDCacheResetsCount  uint64
 
 	IndexDBRefCount uint64
 
@@ -250,10 +251,12 @@ func (db *indexDB) UpdateMetrics(m *IndexDBMetrics) {
 	m.TagFiltersToMetricIDsCacheMisses += db.tagFiltersToMetricIDsCache.Misses()
 	m.TagFiltersToMetricIDsCacheResets += db.tagFiltersToMetricIDsCache.Resets()
 
-	m.DateMetricIDCacheSize += uint64(db.dateMetricIDCache.EntriesCount())
-	m.DateMetricIDCacheSizeBytes += uint64(db.dateMetricIDCache.SizeBytes())
-	m.DateMetricIDCacheSyncsCount += db.dateMetricIDCache.syncsCount.Load()
-	m.DateMetricIDCacheResetsCount += db.dateMetricIDCache.resetsCount.Load()
+	dmcs := db.dateMetricIDCache.Stats()
+	m.DateMetricIDCacheSize += dmcs.Size
+	m.DateMetricIDCacheSizeBytes += dmcs.SizeBytes
+	m.DateMetricIDCacheSizeMaxBytes += dmcs.SizeMaxBytes
+	m.DateMetricIDCacheSyncsCount += dmcs.SyncsCount
+	m.DateMetricIDCacheResetsCount += dmcs.ResetsCount
 
 	m.IndexDBRefCount += uint64(db.refCount.Load())