Compare commits

...

5 Commits

Author SHA1 Message Date
Artem Fetishev
137501d61b extend select existing unit tests with cases when global index is disabled
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-07-28 13:37:59 +02:00
Artem Fetishev
b8eefa75fb fix TestSearchTSIDWithTimeRange and TestSearchLabelValues tests
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-07-28 13:37:55 +02:00
Artem Fetishev
1bf0cc4f32 TestIndexDB: add the cases for disabled global index
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-07-28 13:31:25 +02:00
Artem Fetishev
e0e0c63f12 update TestStorageDeleteSeries with cases when global index is disabled
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-07-28 13:31:24 +02:00
Artem Fetishev
886b82be53 ib/storage: initial implementation of disabling global index
Existing tests pass, all except TestUnitTest in app/vmalert-tool/unittest/unittest_test.go.
Some changes to existing unit and apptests tests were necessary (such as explicitly enabling global index).
Tests that verify the behavior with disabled global index will be added in subsequent commits.

Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-07-28 13:31:24 +02:00
7 changed files with 395 additions and 204 deletions

View File

@@ -94,9 +94,13 @@ var (
cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
enableGlobalIndex = flag.Bool("enableGlobalIndex", false, "Enable global index. "+
"Deployments with high churn rate should have this index disabled as this decreases disk space usage. "+
"Such deployments may enable global index if the dominant query time range is > 1m as it may slightly improve query performance. "+
"Also see -disablePerDayIndex and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning")
disablePerDayIndex = flag.Bool("disablePerDayIndex", false, "Disable per-day index and use global index for all searches. "+
"This may improve performance and decrease disk space usage for the use cases with fixed set of timeseries scattered across a "+
"big time range (for example, when loading years of historical data). "+
"This may improve performance and decrease disk space usage for deployment with no/low churn rate. "+
"Disabling per-day index forces enabling global index and the -enableGlobalIndex flag value is ignored."+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning")
trackMetricNamesStats = flag.Bool("storage.trackMetricNamesStats", true, "Whether to track ingest and query requests for timeseries metric names. "+
"This feature allows to track metric names unused at query requests. "+
@@ -148,6 +152,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
if *idbPrefillStart > 23*time.Hour {
logger.Panicf("-storage.idbPrefillStart cannot exceed 23 hours; got %s", idbPrefillStart)
}
disableGlobalIndex := !*enableGlobalIndex
if *disablePerDayIndex {
// In case if per-day index has been disabled, forcibly enable global
// index even if -enableGlobalIndex flag is false.
disableGlobalIndex = false
}
fs.RegisterPathFsMetrics(*storageDataPath)
logger.Infof("opening storage at %q with -retentionPeriod=%s", *storageDataPath, retentionPeriod)
startTime := time.Now()
@@ -158,6 +168,7 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
MaxHourlySeries: getMaxHourlySeries(),
MaxDailySeries: getMaxDailySeries(),
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: *disablePerDayIndex,
TrackMetricNamesStats: *trackMetricNamesStats,
IDBPrefillStart: *idbPrefillStart,

View File

@@ -16,6 +16,7 @@ func TestSingleSearchWithDisabledPerDayIndex(t *testing.T) {
"-storageDataPath=" + tc.Dir() + "/vmsingle",
"-retentionPeriod=100y",
"-search.maxStalenessInterval=1m",
"-enableGlobalIndex=true",
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
})
})
@@ -34,6 +35,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
"-httpListenAddr=127.0.0.1:61001",
"-vminsertAddr=127.0.0.1:61002",
"-vmselectAddr=127.0.0.1:61003",
"-enableGlobalIndex=true",
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
})
vmstorage2 := tc.MustStartVmstorage("vmstorage2-"+name, []string{
@@ -42,6 +44,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
"-httpListenAddr=127.0.0.1:62001",
"-vminsertAddr=127.0.0.1:62002",
"-vmselectAddr=127.0.0.1:62003",
"-enableGlobalIndex=true",
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
})
vminsert := tc.MustStartVminsert("vminsert-"+name, []string{

View File

@@ -436,18 +436,6 @@ func (db *indexDB) createGlobalIndexes(tsid *TSID, mn *MetricName) {
ii := getIndexItems()
defer putIndexItems(ii)
if db.s.disablePerDayIndex {
// Create metricName -> TSID entry.
// This index is used for searching a TSID by metric name during data
// ingestion or metric name registration when -disablePerDayIndex flag
// is set.
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricNameToTSID)
ii.B = mn.Marshal(ii.B)
ii.B = append(ii.B, kvSeparatorChar)
ii.B = tsid.Marshal(ii.B)
ii.Next()
}
// Create metricID -> metricName entry.
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricIDToMetricName)
ii.B = encoding.MarshalUint64(ii.B, tsid.MetricID)
@@ -460,11 +448,20 @@ func (db *indexDB) createGlobalIndexes(tsid *TSID, mn *MetricName) {
ii.B = tsid.Marshal(ii.B)
ii.Next()
// Create tag -> metricID entries for every tag in mn.
kb := kbPool.Get()
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
ii.registerTagIndexes(kb.B, mn, tsid.MetricID)
kbPool.Put(kb)
if !db.s.disableGlobalIndex {
// Create metricName -> TSID entry.
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricNameToTSID)
ii.B = mn.Marshal(ii.B)
ii.B = append(ii.B, kvSeparatorChar)
ii.B = tsid.Marshal(ii.B)
ii.Next()
// Create tag -> metricID entries for every tag in mn.
kb := kbPool.Get()
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
ii.registerTagIndexes(kb.B, mn, tsid.MetricID)
kbPool.Put(kb)
}
db.tb.AddItems(ii.Items)
}
@@ -759,6 +756,7 @@ func (db *indexDB) SearchLabelValues(qt *querytracer.Tracer, labelName string, t
func filterLabelValues(lvs map[string]struct{}, tf *tagFilter, key string) {
var b []byte
for lv := range lvs {
// TODO
b = marshalCommonPrefix(b[:0], nsPrefixTagToMetricIDs)
b = marshalTagValue(b, bytesutil.ToUnsafeBytes(key))
b = marshalTagValue(b, bytesutil.ToUnsafeBytes(lv))
@@ -1517,10 +1515,11 @@ func (db *indexDB) DeleteSeries(qt *querytracer.Tracer, tfss []*TagFilters, maxM
is := db.getIndexSearch(noDeadline)
defer db.putIndexSearch(is)
// Unconditionally search global index since a given day in per-day
// index may not contain the full set of metricIDs that correspond
// to the tfss.
metricIDs, err := is.searchMetricIDs(qt, tfss, globalIndexTimeRange, maxMetrics)
tr := globalIndexTimeRange
if db.s.disableGlobalIndex {
tr = db.tr
}
metricIDs, err := is.searchMetricIDs(qt, tfss, tr, maxMetrics)
if err != nil {
return nil, db.wrapError("delete series", err)
}
@@ -1967,6 +1966,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(qt *querytracer.Tracer,
qt.Printf("sort %d metric ids", len(sortedMetricIDs))
kb := &is.kb
// TODO
kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
tfs = removeCompositeTagFilters(tfs, kb.B)
@@ -2077,6 +2077,7 @@ func hasCompositeTagFilters(tfs []*tagFilter, prefix []byte) bool {
}
func matchTagFilters(mn *MetricName, tfs []*tagFilter, kb *bytesutil.ByteBuffer) (bool, error) {
// TODO
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
for i, tf := range tfs {
if bytes.Equal(tf.key, graphiteReverseTagKey) {
@@ -3236,6 +3237,7 @@ func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter, negativeFilter
}
func mergeTagToMetricIDsRows(data []byte, items []mergeset.Item) ([]byte, []mergeset.Item) {
// TODO
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixTagToMetricIDs)
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixDateTagToMetricIDs)
return data, items

View File

@@ -2,10 +2,10 @@ package storage
import (
"bytes"
"math"
"path/filepath"
"strconv"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -79,7 +79,7 @@ func mustOpenLegacyIndexDB(path string, s *Storage) *legacyIndexDB {
tr := TimeRange{
MinTimestamp: 0,
MaxTimestamp: math.MaxInt64,
MaxTimestamp: time.Now().UnixMilli(),
}
idb := mustOpenIndexDB(id, tr, name, path, s, &s.isReadOnly, true)
legacyIDB := &legacyIndexDB{idb: idb}

View File

@@ -486,7 +486,7 @@ func TestIndexDBOpenClose(t *testing.T) {
func TestIndexDB(t *testing.T) {
defer testRemoveAll(t)
f := func(t *testing.T, concurrency int, disablePerDayIndex bool) {
f := func(t *testing.T, concurrency int, disableGlobalIndex, disablePerDayIndex bool) {
const metricGroups = 10
now := time.Now().UTC()
timestamp := now.UnixMilli()
@@ -500,6 +500,7 @@ func TestIndexDB(t *testing.T) {
}
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -531,13 +532,19 @@ func TestIndexDB(t *testing.T) {
}
for _, concurrency := range []int{1, 4} {
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
f(t, concurrency, disablePerDayIndex)
// Repeat the same test on non-empty reopened storage.
f(t, concurrency, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("concurrency=%d/disableGlobalIndex=%t/disablePerDayIndex=%t", concurrency, disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
f(t, concurrency, disableGlobalIndex, disablePerDayIndex)
// Repeat the same test on non-empty reopened storage.
f(t, concurrency, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
}
@@ -1400,15 +1407,21 @@ func TestMatchTagFilters(t *testing.T) {
func TestIndexDBSearchTSIDs(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBSearchTSIDs(t, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBSearchTSIDs(t, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
func testIndexDBSearchTSIDs(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
const days = 5
const metricsPerDay = 1000
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
@@ -1417,6 +1430,7 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
allMetricIDs := &uint64set.Set{}
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -1474,8 +1488,10 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
assertMetricIDs(date, metricsPerDay, perDayMetricIDs[date])
}
}
// Check that all the metrics are found in global index
assertMetricIDs(globalIndexDate, metricsPerDay*days, allMetricIDs)
if !disableGlobalIndex {
// Check that all the metrics are found in global index
assertMetricIDs(globalIndexDate, metricsPerDay*days, allMetricIDs)
}
db.putIndexSearch(is2)
@@ -1522,21 +1538,28 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
func TestIndexDBSearchLabelNames(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBSearchLabelNames(t, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBSearchLabelNames(t, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
func testIndexDBSearchLabelNames(t *testing.T, disablePerDayIndex bool) {
func testIndexDBSearchLabelNames(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
const days = 5
const metricsPerDay = 1000
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
baseDate := uint64(timestamp) / msecPerDay
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -1633,16 +1656,21 @@ func testIndexDBSearchLabelNames(t *testing.T, disablePerDayIndex bool) {
func TestIndexDBSearchLabelValues(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBSearchLabelValues(t, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBSearchLabelValues(t, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
func testIndexDBSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
func testIndexDBSearchLabelValues(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
const days = 5
const metricsPerDay = 1000
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
@@ -1650,6 +1678,7 @@ func testIndexDBSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
var allMetricNames []string
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -1816,15 +1845,21 @@ func TestFilterLabelValues(t *testing.T) {
func TestIndexDBDeleteSeries(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBDeleteSeries(t, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBDeleteSeries(t, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
func testIndexDBDeleteSeries(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
const days = 5
const metricsPerDay = 1000
trAllDays := TimeRange{
@@ -1837,6 +1872,7 @@ func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
metricNamesByDate := make(map[uint64][]string)
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -1922,21 +1958,28 @@ func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
func TestIndexDBGetTSDBStatus(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBGetTSDBStatus(t, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testIndexDBGetTSDBStatus(t, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
func testIndexDBGetTSDBStatus(t *testing.T, disablePerDayIndex bool) {
func testIndexDBGetTSDBStatus(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
const days = 5
const metricsPerDay = 1000
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
baseDate := uint64(timestamp) / msecPerDay
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()

View File

@@ -80,6 +80,7 @@ type Storage struct {
// compatibility with partition index.
legacyIndexDBs atomic.Pointer[legacyIndexDBs]
disableGlobalIndex bool
disablePerDayIndex bool
tb *table
@@ -169,6 +170,7 @@ type OpenOptions struct {
DenyQueriesOutsideRetention bool
MaxHourlySeries int
MaxDailySeries int
DisableGlobalIndex bool
DisablePerDayIndex bool
TrackMetricNamesStats bool
IDBPrefillStart time.Duration
@@ -269,6 +271,10 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
fs.MustMkdirIfNotExist(metadataDir)
s.minTimestampForCompositeIndex = mustGetMinTimestampForCompositeIndex(metadataDir, isEmptyDB)
if opts.DisableGlobalIndex && opts.DisablePerDayIndex {
logger.Panicf("BUG: global and per-day indexes cannot be disabled at the same time")
}
s.disableGlobalIndex = opts.DisableGlobalIndex
s.disablePerDayIndex = opts.DisablePerDayIndex
// Load legacy indexDBs.
@@ -1724,6 +1730,12 @@ const maxDaysForPerDaySearch = 40
// globalIndexTimeRange based on the time range length and -disablePerDayIndex
// flag.
func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
// if the global index is disabled and the searchTR is the
// globalIndexTimeRange, use idbTR.
if s.disableGlobalIndex && searchTR == globalIndexTimeRange {
return idbTR
}
// If the per day index is disabled, unconditionally search global index.
if s.disablePerDayIndex {
return globalIndexTimeRange
@@ -1740,7 +1752,7 @@ func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
// For legacy IndexDBs only, partition indexDBs can't span more than a
// month.
minDate, maxDate := tr.DateRange()
if maxDate-minDate > maxDaysForPerDaySearch {
if !s.disableGlobalIndex && maxDate-minDate > maxDaysForPerDaySearch {
return globalIndexTimeRange
}
@@ -1748,7 +1760,7 @@ func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
// the idb time range, then return globalIndexTimeRange to indicate that we
// want to search the global index since the entire index db needs to be
// searched anyway.
if tr == idbTR {
if !s.disableGlobalIndex && tr == idbTR {
return globalIndexTimeRange
}

View File

@@ -531,22 +531,29 @@ func TestStorageDeleteSeries(t *testing.T) {
defer testRemoveAll(t)
for _, concurrency := range []int{1, 4} {
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testStorageDeleteSeries(t, concurrency, disablePerDayIndex)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("concurrency=%d/disableGlobalIndex=%t/disablePerDayIndex=%t", concurrency, disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testStorageDeleteSeries(t, concurrency, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
}
func testStorageDeleteSeries(t *testing.T, concurrency int, disablePerDayIndex bool) {
func testStorageDeleteSeries(t *testing.T, concurrency int, disableGlobalIndex, disablePerDayIndex bool) {
tr := TimeRange{
MinTimestamp: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(2026, 1, 15, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
}
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -2821,10 +2828,11 @@ func TestStorageGetTSDBStatus(t *testing.T) {
func TestStorageAdjustTimeRange(t *testing.T) {
defer testRemoveAll(t)
f := func(disablePerDayIndex bool, searchTR, idbTR, want TimeRange) {
f := func(disableGlobalIndex, disablePerDayIndex bool, searchTR, idbTR, want TimeRange) {
t.Helper()
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -2835,7 +2843,7 @@ func TestStorageAdjustTimeRange(t *testing.T) {
legacyIDBTimeRange := TimeRange{
MinTimestamp: 0,
MaxTimestamp: math.MaxInt64,
MaxTimestamp: time.Now().UnixMilli(),
}
partitionIDBTimeRange := TimeRange{
MinTimestamp: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
@@ -2843,113 +2851,103 @@ func TestStorageAdjustTimeRange(t *testing.T) {
}
var searchTimeRange TimeRange
// Zero search time range is adjusted to globalIndexTimeRange regardless
// whether the -disablePerDayIndex flag is set or not.
searchTimeRange = TimeRange{}
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
// Search time range is the same as globalIndexTimeRange.
searchTimeRange = globalIndexTimeRange
f(false, false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, legacyIDBTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
// The search time range is smaller than a month (and therefore < 40 days)
// and is fully included into the partition idb time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange. Otherwise it must remain the same
// after the adjustment.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
}
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
// The search time range is the same as partition idb time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb: it must remain the same
// - For the partition idb: it must be replaced with globalIndexTimeRange.
searchTimeRange = partitionIDBTimeRange
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
// The search time range is smaller than 40 days and fully includes the
// partition idb time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb: it must remain the same
// - For the partition idb: it must be replaced with globalIndexTimeRange.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
}
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
// The search time range is 41 days and fully includes the partition idb
// time range.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise it must be replaced with globalIndexTimeRange for both legacy
// and partition idbs.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + 41*msecPerDay,
}
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
// The search time range is smaller than 40 days and overlaps with partition
// idb time range on the left.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb: it must remain the same
// - For the partition idb: the MinTimestamp must be adjusted to match the
// partition idb time range MinTimestamp.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
}
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, TimeRange{
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp,
MaxTimestamp: searchTimeRange.MaxTimestamp,
})
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
MinTimestamp: partitionIDBTimeRange.MinTimestamp,
MaxTimestamp: searchTimeRange.MaxTimestamp,
})
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
// The search time range is smaller than 40 days and overlaps with partition
// idb time range on the right.
// If -disablePerDayIndex is set, the effective search time range is
// expected to be globalIndexTimeRange for both legacy and parition idb.
// Otherwise:
// - For the legacy idb, it must remain the same
// - For the partition idb: its MaxTimestamp must be adjusted to match the
// partition idb time range MaxTimestamp.
searchTimeRange = TimeRange{
MinTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
}
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, searchTimeRange, partitionIDBTimeRange, TimeRange{
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(false, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
MinTimestamp: searchTimeRange.MinTimestamp,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp,
})
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
f(true, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
MinTimestamp: searchTimeRange.MinTimestamp,
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp,
})
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
}
type testStorageSearchWithoutPerDayIndexOptions struct {
type testStorageSearchWithoutIndexOptions struct {
mrs []MetricRow
assertSearchResult func(t *testing.T, s *Storage, tr TimeRange, want any)
alwaysPerTimeRange bool // If true, use wantPerTimeRange instead of wantAll
@@ -2958,17 +2956,18 @@ type testStorageSearchWithoutPerDayIndexOptions struct {
wantEmpty any
}
// testStorageSearchWithoutPerDayIndex tests how the search behaves when the
// testStorageSearchWithoutIndex tests how the search behaves when the
// per-day index is disabled. This function is expected to be called by
// functions that test a particular search operation, such as GetTSDBStatus(),
// SearchMetricNames(), etc.
func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWithoutPerDayIndexOptions) {
func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutIndexOptions) {
defer testRemoveAll(t)
// The data is inserted and the search is performed when the per-day index
// is enabled.
t.Run("InsertAndSearchWithPerDayIndex", func(t *testing.T) {
// The data is inserted and the search is performed when both global and
// per-day indexes are enabled.
t.Run("Add-Global-PerDay/Search-Global-PerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: false,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
@@ -2979,10 +2978,11 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
s.MustClose()
})
// The data is inserted and the search is performed when the per-day index
// is disabled.
t.Run("InsertAndSearchWithoutPerDayIndex", func(t *testing.T) {
// The data is inserted and the search is performed when global index is
// enabled and per-day index is disabled.
t.Run("Add-Global-noPerDay/Search-Global-noPerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: true,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
@@ -2996,10 +2996,12 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
s.MustClose()
})
// The data is inserted when the per-day index is enabled but the search is
// performed when the per-day index is disabled.
t.Run("InsertWithPerDayIndexSearchWithout", func(t *testing.T) {
// The data is inserted when both global and per-day indexes are enabled.
// The search is performed when the global index is enabled and per-day
// index is disabled.
t.Run("Add-Global-PerDay/Search-Global-noPerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: false,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
@@ -3007,6 +3009,7 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: true,
})
for tr, want := range opts.wantPerTimeRange {
@@ -3018,11 +3021,14 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
s.MustClose()
})
// The data is inserted when the per-day index is disabled but the search is
// performed when the per-day index is enabled. This case also shows that
// registering metric names recovers the per-day index.
t.Run("InsertWithoutPerDayIndexSearchWith", func(t *testing.T) {
// The data is inserted when global index is enabled and per-day index is
// disabled.
// The search is performed when both global and per-day index is enabled.
// This case also shows that registering metric names recovers the per-day
// index.
t.Run("Add-Global-noPerDay/Search-Global-PerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: true,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
@@ -3030,6 +3036,7 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: false,
})
@@ -3046,15 +3053,120 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
}
s.MustClose()
})
// The data is inserted and the search is performed when global index is
// disabled and per-day index is enabled.
t.Run("Add-noGlobal-PerDay/Search-noGlobal-PerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: true,
DisablePerDayIndex: false,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
s.DebugFlush()
for tr, want := range opts.wantPerTimeRange {
opts.assertSearchResult(t, s, tr, want)
}
s.MustClose()
})
// The data is inserted when both global and per-day indexes are enabled.
// The search is performed when the global index is disabled and per-day
// index is enabled.
t.Run("Add-Global-PerDay/Search-noGlobal-PerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: false,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
s.DebugFlush()
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: true,
DisablePerDayIndex: false,
})
for tr, want := range opts.wantPerTimeRange {
opts.assertSearchResult(t, s, tr, want)
}
s.MustClose()
})
// The data is inserted when global index is disabled and per-day index is
// enabled.
// The search is performed when both global and per-day indexes are enabled.
t.Run("Add-noGlobal-PerDay/Search-Global-PerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: true,
DisablePerDayIndex: false,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
s.DebugFlush()
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: false,
})
for tr, want := range opts.wantPerTimeRange {
opts.assertSearchResult(t, s, tr, want)
}
s.MustClose()
})
// The data is inserted when global index is disabled and per-day index is
// enabled.
// The search is performed when global index is enabled and per-day index is
// disabled.
t.Run("Add-noGlobal-PerDay/Search-Global-noPerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: true,
DisablePerDayIndex: false,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
s.DebugFlush()
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: true,
})
for tr := range opts.wantPerTimeRange {
opts.assertSearchResult(t, s, tr, opts.wantEmpty)
}
s.MustClose()
})
// The data is inserted when global index is enabled and per-day index is
// disabled.
// The search is performed when the global index is disabled and per-day
// index is enabled.
t.Run("Add-Global-noPerDay/Search-noGlobal-PerDay", func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: false,
DisablePerDayIndex: true,
})
s.AddRows(opts.mrs, defaultPrecisionBits)
s.DebugFlush()
s.MustClose()
s = MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: true,
DisablePerDayIndex: false,
})
for tr := range opts.wantPerTimeRange {
opts.assertSearchResult(t, s, tr, opts.wantEmpty)
}
s.MustClose()
})
}
func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: &TSDBStatus{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: &TSDBStatus{TotalSeries: days * rows},
@@ -3094,16 +3206,16 @@ func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3155,16 +3267,16 @@ func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3209,17 +3321,17 @@ func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
labelName = "job"
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3263,17 +3375,17 @@ func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
tagValuePrefix = "metric."
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3313,16 +3425,16 @@ func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: []string{},
wantPerTimeRange: make(map[TimeRange]any),
wantAll: []string{},
@@ -3363,16 +3475,16 @@ func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
func TestStorageQueryWithoutIndex(t *testing.T) {
const (
days = 4
rows = 10
)
rng := rand.New(rand.NewSource(1))
opts := testStorageSearchWithoutPerDayIndexOptions{
opts := testStorageSearchWithoutIndexOptions{
wantEmpty: []MetricRow(nil),
wantPerTimeRange: make(map[TimeRange]any),
alwaysPerTimeRange: true,
@@ -3413,16 +3525,17 @@ func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
}
}
testStorageSearchWithoutPerDayIndex(t, &opts)
testStorageSearchWithoutIndex(t, &opts)
}
func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
defer testRemoveAll(t)
f := func(t *testing.T, disablePerDayIndex bool) {
f := func(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
t.Helper()
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
@@ -3455,12 +3568,21 @@ func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
}
}
t.Run("disablePerDayIndex=false", func(t *testing.T) {
f(t, false)
})
t.Run("disablePerDayIndex=true", func(t *testing.T) {
f(t, true)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
if disableGlobalIndex {
t.Skip("TODO(@rtm0): zero timestamps cannot be searched with disabled global index")
}
f(t, disableGlobalIndex, disablePerDayIndex)
})
}
}
}
// testSearchMetricIDs returns metricIDs for the given tfss and tr.
@@ -3497,13 +3619,13 @@ func testCountAllMetricIDs(s *Storage, tr TimeRange) int {
return len(ids)
}
func TestStorageRegisterMetricNamesForVariousDataPatternsConcurrently(t *testing.T) {
func TestStorageRegisterMetricNamesVariousDataPatterns(t *testing.T) {
testStorageVariousDataPatternsConcurrently(t, true, func(s *Storage, mrs []MetricRow) {
s.RegisterMetricNames(nil, mrs)
})
}
func TestStorageAddRowsForVariousDataPatternsConcurrently(t *testing.T) {
func TestStorageAddRowsVariousDataPatterns(t *testing.T) {
testStorageVariousDataPatternsConcurrently(t, false, func(s *Storage, mrs []MetricRow) {
s.AddRows(mrs, defaultPrecisionBits)
})
@@ -3520,27 +3642,24 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
const concurrency = 4
disablePerDayIndex := false
t.Run("perDayIndexes/serial", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
})
t.Run("perDayIndexes/concurrentRows", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
})
t.Run("perDayIndexes/concurrentBatches", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
})
disablePerDayIndex = true
t.Run("noPerDayIndexes/serial", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
})
t.Run("noPerDayIndexes/concurrentRows", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
})
t.Run("noPerDayIndexes/concurrentBatches", func(t *testing.T) {
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
})
for _, disableGlobalIndex := range []bool{false, true} {
for _, disablePerDayIndex := range []bool{false, true} {
if disableGlobalIndex && disablePerDayIndex {
// Both indexes cannot be disabled at the same time.
continue
}
prefix := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
t.Run(prefix+"/serial", func(t *testing.T) {
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, 1, false)
})
t.Run(prefix+"/concurrentRows", func(t *testing.T) {
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, concurrency, true)
})
t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, concurrency, false)
})
}
}
}
// testStorageVariousDataPatterns tests the ingestion of different combinations
@@ -3549,7 +3668,7 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
// The function is intended to be used by other tests that define the
// concurrency, the per-day index setting, and the operation (AddRows or
// RegisterMetricNames) under test.
func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOnly bool, op func(s *Storage, mrs []MetricRow), concurrency int, splitBatches bool) {
func testStorageVariousDataPatterns(t *testing.T, disableGlobalIndex, disablePerDayIndex, registerOnly bool, op func(s *Storage, mrs []MetricRow), concurrency int, splitBatches bool) {
f := func(t *testing.T, sameBatchMetricNames, sameRowMetricNames, sameBatchDates, sameRowDates bool) {
batches, wantCounts := testGenerateMetricRowBatches(&batchOptions{
numBatches: 3,
@@ -3566,6 +3685,7 @@ func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOn
rowsAddedTotal := wantCounts.metrics.RowsAddedTotal
s := MustOpenStorage(t.Name(), OpenOptions{
DisableGlobalIndex: disableGlobalIndex,
DisablePerDayIndex: disablePerDayIndex,
})