mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-07-29 12:01:55 +03:00
Compare commits
5 Commits
dependabot
...
disable-gl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
137501d61b | ||
|
|
b8eefa75fb | ||
|
|
1bf0cc4f32 | ||
|
|
e0e0c63f12 | ||
|
|
886b82be53 |
@@ -94,9 +94,13 @@ var (
|
||||
cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
|
||||
enableGlobalIndex = flag.Bool("enableGlobalIndex", false, "Enable global index. "+
|
||||
"Deployments with high churn rate should have this index disabled as this decreases disk space usage. "+
|
||||
"Such deployments may enable global index if the dominant query time range is > 1m as it may slightly improve query performance. "+
|
||||
"Also see -disablePerDayIndex and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning")
|
||||
disablePerDayIndex = flag.Bool("disablePerDayIndex", false, "Disable per-day index and use global index for all searches. "+
|
||||
"This may improve performance and decrease disk space usage for the use cases with fixed set of timeseries scattered across a "+
|
||||
"big time range (for example, when loading years of historical data). "+
|
||||
"This may improve performance and decrease disk space usage for deployment with no/low churn rate. "+
|
||||
"Disabling per-day index forces enabling global index and the -enableGlobalIndex flag value is ignored."+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning")
|
||||
trackMetricNamesStats = flag.Bool("storage.trackMetricNamesStats", true, "Whether to track ingest and query requests for timeseries metric names. "+
|
||||
"This feature allows to track metric names unused at query requests. "+
|
||||
@@ -148,6 +152,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
|
||||
if *idbPrefillStart > 23*time.Hour {
|
||||
logger.Panicf("-storage.idbPrefillStart cannot exceed 23 hours; got %s", idbPrefillStart)
|
||||
}
|
||||
disableGlobalIndex := !*enableGlobalIndex
|
||||
if *disablePerDayIndex {
|
||||
// In case if per-day index has been disabled, forcibly enable global
|
||||
// index even if -enableGlobalIndex flag is false.
|
||||
disableGlobalIndex = false
|
||||
}
|
||||
fs.RegisterPathFsMetrics(*storageDataPath)
|
||||
logger.Infof("opening storage at %q with -retentionPeriod=%s", *storageDataPath, retentionPeriod)
|
||||
startTime := time.Now()
|
||||
@@ -158,6 +168,7 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
|
||||
DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
|
||||
MaxHourlySeries: getMaxHourlySeries(),
|
||||
MaxDailySeries: getMaxDailySeries(),
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: *disablePerDayIndex,
|
||||
TrackMetricNamesStats: *trackMetricNamesStats,
|
||||
IDBPrefillStart: *idbPrefillStart,
|
||||
|
||||
@@ -16,6 +16,7 @@ func TestSingleSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
"-storageDataPath=" + tc.Dir() + "/vmsingle",
|
||||
"-retentionPeriod=100y",
|
||||
"-search.maxStalenessInterval=1m",
|
||||
"-enableGlobalIndex=true",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
})
|
||||
@@ -34,6 +35,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
"-httpListenAddr=127.0.0.1:61001",
|
||||
"-vminsertAddr=127.0.0.1:61002",
|
||||
"-vmselectAddr=127.0.0.1:61003",
|
||||
"-enableGlobalIndex=true",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
vmstorage2 := tc.MustStartVmstorage("vmstorage2-"+name, []string{
|
||||
@@ -42,6 +44,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
"-httpListenAddr=127.0.0.1:62001",
|
||||
"-vminsertAddr=127.0.0.1:62002",
|
||||
"-vmselectAddr=127.0.0.1:62003",
|
||||
"-enableGlobalIndex=true",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
vminsert := tc.MustStartVminsert("vminsert-"+name, []string{
|
||||
|
||||
@@ -436,18 +436,6 @@ func (db *indexDB) createGlobalIndexes(tsid *TSID, mn *MetricName) {
|
||||
ii := getIndexItems()
|
||||
defer putIndexItems(ii)
|
||||
|
||||
if db.s.disablePerDayIndex {
|
||||
// Create metricName -> TSID entry.
|
||||
// This index is used for searching a TSID by metric name during data
|
||||
// ingestion or metric name registration when -disablePerDayIndex flag
|
||||
// is set.
|
||||
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricNameToTSID)
|
||||
ii.B = mn.Marshal(ii.B)
|
||||
ii.B = append(ii.B, kvSeparatorChar)
|
||||
ii.B = tsid.Marshal(ii.B)
|
||||
ii.Next()
|
||||
}
|
||||
|
||||
// Create metricID -> metricName entry.
|
||||
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricIDToMetricName)
|
||||
ii.B = encoding.MarshalUint64(ii.B, tsid.MetricID)
|
||||
@@ -460,11 +448,20 @@ func (db *indexDB) createGlobalIndexes(tsid *TSID, mn *MetricName) {
|
||||
ii.B = tsid.Marshal(ii.B)
|
||||
ii.Next()
|
||||
|
||||
// Create tag -> metricID entries for every tag in mn.
|
||||
kb := kbPool.Get()
|
||||
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
ii.registerTagIndexes(kb.B, mn, tsid.MetricID)
|
||||
kbPool.Put(kb)
|
||||
if !db.s.disableGlobalIndex {
|
||||
// Create metricName -> TSID entry.
|
||||
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricNameToTSID)
|
||||
ii.B = mn.Marshal(ii.B)
|
||||
ii.B = append(ii.B, kvSeparatorChar)
|
||||
ii.B = tsid.Marshal(ii.B)
|
||||
ii.Next()
|
||||
|
||||
// Create tag -> metricID entries for every tag in mn.
|
||||
kb := kbPool.Get()
|
||||
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
ii.registerTagIndexes(kb.B, mn, tsid.MetricID)
|
||||
kbPool.Put(kb)
|
||||
}
|
||||
|
||||
db.tb.AddItems(ii.Items)
|
||||
}
|
||||
@@ -759,6 +756,7 @@ func (db *indexDB) SearchLabelValues(qt *querytracer.Tracer, labelName string, t
|
||||
func filterLabelValues(lvs map[string]struct{}, tf *tagFilter, key string) {
|
||||
var b []byte
|
||||
for lv := range lvs {
|
||||
// TODO
|
||||
b = marshalCommonPrefix(b[:0], nsPrefixTagToMetricIDs)
|
||||
b = marshalTagValue(b, bytesutil.ToUnsafeBytes(key))
|
||||
b = marshalTagValue(b, bytesutil.ToUnsafeBytes(lv))
|
||||
@@ -1517,10 +1515,11 @@ func (db *indexDB) DeleteSeries(qt *querytracer.Tracer, tfss []*TagFilters, maxM
|
||||
is := db.getIndexSearch(noDeadline)
|
||||
defer db.putIndexSearch(is)
|
||||
|
||||
// Unconditionally search global index since a given day in per-day
|
||||
// index may not contain the full set of metricIDs that correspond
|
||||
// to the tfss.
|
||||
metricIDs, err := is.searchMetricIDs(qt, tfss, globalIndexTimeRange, maxMetrics)
|
||||
tr := globalIndexTimeRange
|
||||
if db.s.disableGlobalIndex {
|
||||
tr = db.tr
|
||||
}
|
||||
metricIDs, err := is.searchMetricIDs(qt, tfss, tr, maxMetrics)
|
||||
if err != nil {
|
||||
return nil, db.wrapError("delete series", err)
|
||||
}
|
||||
@@ -1967,6 +1966,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(qt *querytracer.Tracer,
|
||||
qt.Printf("sort %d metric ids", len(sortedMetricIDs))
|
||||
|
||||
kb := &is.kb
|
||||
// TODO
|
||||
kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
tfs = removeCompositeTagFilters(tfs, kb.B)
|
||||
|
||||
@@ -2077,6 +2077,7 @@ func hasCompositeTagFilters(tfs []*tagFilter, prefix []byte) bool {
|
||||
}
|
||||
|
||||
func matchTagFilters(mn *MetricName, tfs []*tagFilter, kb *bytesutil.ByteBuffer) (bool, error) {
|
||||
// TODO
|
||||
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
for i, tf := range tfs {
|
||||
if bytes.Equal(tf.key, graphiteReverseTagKey) {
|
||||
@@ -3236,6 +3237,7 @@ func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter, negativeFilter
|
||||
}
|
||||
|
||||
func mergeTagToMetricIDsRows(data []byte, items []mergeset.Item) ([]byte, []mergeset.Item) {
|
||||
// TODO
|
||||
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixTagToMetricIDs)
|
||||
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixDateTagToMetricIDs)
|
||||
return data, items
|
||||
|
||||
@@ -2,10 +2,10 @@ package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
@@ -79,7 +79,7 @@ func mustOpenLegacyIndexDB(path string, s *Storage) *legacyIndexDB {
|
||||
|
||||
tr := TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: math.MaxInt64,
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
}
|
||||
idb := mustOpenIndexDB(id, tr, name, path, s, &s.isReadOnly, true)
|
||||
legacyIDB := &legacyIndexDB{idb: idb}
|
||||
|
||||
@@ -486,7 +486,7 @@ func TestIndexDBOpenClose(t *testing.T) {
|
||||
func TestIndexDB(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
f := func(t *testing.T, concurrency int, disablePerDayIndex bool) {
|
||||
f := func(t *testing.T, concurrency int, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const metricGroups = 10
|
||||
now := time.Now().UTC()
|
||||
timestamp := now.UnixMilli()
|
||||
@@ -500,6 +500,7 @@ func TestIndexDB(t *testing.T) {
|
||||
}
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -531,13 +532,19 @@ func TestIndexDB(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, concurrency := range []int{1, 4} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f(t, concurrency, disablePerDayIndex)
|
||||
// Repeat the same test on non-empty reopened storage.
|
||||
f(t, concurrency, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("concurrency=%d/disableGlobalIndex=%t/disablePerDayIndex=%t", concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f(t, concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
// Repeat the same test on non-empty reopened storage.
|
||||
f(t, concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1400,15 +1407,21 @@ func TestMatchTagFilters(t *testing.T) {
|
||||
|
||||
func TestIndexDBSearchTSIDs(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchTSIDs(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchTSIDs(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBSearchTSIDs(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
@@ -1417,6 +1430,7 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
allMetricIDs := &uint64set.Set{}
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1474,8 +1488,10 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
assertMetricIDs(date, metricsPerDay, perDayMetricIDs[date])
|
||||
}
|
||||
}
|
||||
// Check that all the metrics are found in global index
|
||||
assertMetricIDs(globalIndexDate, metricsPerDay*days, allMetricIDs)
|
||||
if !disableGlobalIndex {
|
||||
// Check that all the metrics are found in global index
|
||||
assertMetricIDs(globalIndexDate, metricsPerDay*days, allMetricIDs)
|
||||
}
|
||||
|
||||
db.putIndexSearch(is2)
|
||||
|
||||
@@ -1522,21 +1538,28 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
|
||||
func TestIndexDBSearchLabelNames(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelNames(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelNames(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBSearchLabelNames(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBSearchLabelNames(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
baseDate := uint64(timestamp) / msecPerDay
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1633,16 +1656,21 @@ func testIndexDBSearchLabelNames(t *testing.T, disablePerDayIndex bool) {
|
||||
|
||||
func TestIndexDBSearchLabelValues(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelValues(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelValues(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBSearchLabelValues(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
@@ -1650,6 +1678,7 @@ func testIndexDBSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
|
||||
var allMetricNames []string
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1816,15 +1845,21 @@ func TestFilterLabelValues(t *testing.T) {
|
||||
|
||||
func TestIndexDBDeleteSeries(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBDeleteSeries(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBDeleteSeries(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBDeleteSeries(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
trAllDays := TimeRange{
|
||||
@@ -1837,6 +1872,7 @@ func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
|
||||
metricNamesByDate := make(map[uint64][]string)
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1922,21 +1958,28 @@ func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
|
||||
|
||||
func TestIndexDBGetTSDBStatus(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBGetTSDBStatus(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBGetTSDBStatus(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBGetTSDBStatus(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBGetTSDBStatus(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
baseDate := uint64(timestamp) / msecPerDay
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
|
||||
@@ -80,6 +80,7 @@ type Storage struct {
|
||||
// compatibility with partition index.
|
||||
legacyIndexDBs atomic.Pointer[legacyIndexDBs]
|
||||
|
||||
disableGlobalIndex bool
|
||||
disablePerDayIndex bool
|
||||
|
||||
tb *table
|
||||
@@ -169,6 +170,7 @@ type OpenOptions struct {
|
||||
DenyQueriesOutsideRetention bool
|
||||
MaxHourlySeries int
|
||||
MaxDailySeries int
|
||||
DisableGlobalIndex bool
|
||||
DisablePerDayIndex bool
|
||||
TrackMetricNamesStats bool
|
||||
IDBPrefillStart time.Duration
|
||||
@@ -269,6 +271,10 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
|
||||
fs.MustMkdirIfNotExist(metadataDir)
|
||||
s.minTimestampForCompositeIndex = mustGetMinTimestampForCompositeIndex(metadataDir, isEmptyDB)
|
||||
|
||||
if opts.DisableGlobalIndex && opts.DisablePerDayIndex {
|
||||
logger.Panicf("BUG: global and per-day indexes cannot be disabled at the same time")
|
||||
}
|
||||
s.disableGlobalIndex = opts.DisableGlobalIndex
|
||||
s.disablePerDayIndex = opts.DisablePerDayIndex
|
||||
|
||||
// Load legacy indexDBs.
|
||||
@@ -1724,6 +1730,12 @@ const maxDaysForPerDaySearch = 40
|
||||
// globalIndexTimeRange based on the time range length and -disablePerDayIndex
|
||||
// flag.
|
||||
func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
|
||||
// if the global index is disabled and the searchTR is the
|
||||
// globalIndexTimeRange, use idbTR.
|
||||
if s.disableGlobalIndex && searchTR == globalIndexTimeRange {
|
||||
return idbTR
|
||||
}
|
||||
|
||||
// If the per day index is disabled, unconditionally search global index.
|
||||
if s.disablePerDayIndex {
|
||||
return globalIndexTimeRange
|
||||
@@ -1740,7 +1752,7 @@ func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
|
||||
// For legacy IndexDBs only, partition indexDBs can't span more than a
|
||||
// month.
|
||||
minDate, maxDate := tr.DateRange()
|
||||
if maxDate-minDate > maxDaysForPerDaySearch {
|
||||
if !s.disableGlobalIndex && maxDate-minDate > maxDaysForPerDaySearch {
|
||||
return globalIndexTimeRange
|
||||
}
|
||||
|
||||
@@ -1748,7 +1760,7 @@ func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
|
||||
// the idb time range, then return globalIndexTimeRange to indicate that we
|
||||
// want to search the global index since the entire index db needs to be
|
||||
// searched anyway.
|
||||
if tr == idbTR {
|
||||
if !s.disableGlobalIndex && tr == idbTR {
|
||||
return globalIndexTimeRange
|
||||
}
|
||||
|
||||
|
||||
@@ -531,22 +531,29 @@ func TestStorageDeleteSeries(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
for _, concurrency := range []int{1, 4} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageDeleteSeries(t, concurrency, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("concurrency=%d/disableGlobalIndex=%t/disablePerDayIndex=%t", concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageDeleteSeries(t, concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testStorageDeleteSeries(t *testing.T, concurrency int, disablePerDayIndex bool) {
|
||||
func testStorageDeleteSeries(t *testing.T, concurrency int, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
tr := TimeRange{
|
||||
MinTimestamp: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC).UnixMilli(),
|
||||
MaxTimestamp: time.Date(2026, 1, 15, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
|
||||
}
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -2821,10 +2828,11 @@ func TestStorageGetTSDBStatus(t *testing.T) {
|
||||
func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
f := func(disablePerDayIndex bool, searchTR, idbTR, want TimeRange) {
|
||||
f := func(disableGlobalIndex, disablePerDayIndex bool, searchTR, idbTR, want TimeRange) {
|
||||
t.Helper()
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -2835,7 +2843,7 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
legacyIDBTimeRange := TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: math.MaxInt64,
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
}
|
||||
partitionIDBTimeRange := TimeRange{
|
||||
MinTimestamp: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
|
||||
@@ -2843,113 +2851,103 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
}
|
||||
var searchTimeRange TimeRange
|
||||
|
||||
// Zero search time range is adjusted to globalIndexTimeRange regardless
|
||||
// whether the -disablePerDayIndex flag is set or not.
|
||||
searchTimeRange = TimeRange{}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
// Search time range is the same as globalIndexTimeRange.
|
||||
searchTimeRange = globalIndexTimeRange
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, legacyIDBTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
|
||||
|
||||
// The search time range is smaller than a month (and therefore < 40 days)
|
||||
// and is fully included into the partition idb time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange. Otherwise it must remain the same
|
||||
// after the adjustment.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
|
||||
// The search time range is the same as partition idb time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb: it must remain the same
|
||||
// - For the partition idb: it must be replaced with globalIndexTimeRange.
|
||||
searchTimeRange = partitionIDBTimeRange
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
|
||||
// The search time range is smaller than 40 days and fully includes the
|
||||
// partition idb time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb: it must remain the same
|
||||
// - For the partition idb: it must be replaced with globalIndexTimeRange.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
|
||||
|
||||
// The search time range is 41 days and fully includes the partition idb
|
||||
// time range.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise it must be replaced with globalIndexTimeRange for both legacy
|
||||
// and partition idbs.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + 41*msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
|
||||
|
||||
// The search time range is smaller than 40 days and overlaps with partition
|
||||
// idb time range on the left.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb: it must remain the same
|
||||
// - For the partition idb: the MinTimestamp must be adjusted to match the
|
||||
// partition idb time range MinTimestamp.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp,
|
||||
MaxTimestamp: searchTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp,
|
||||
MaxTimestamp: searchTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
|
||||
// The search time range is smaller than 40 days and overlaps with partition
|
||||
// idb time range on the right.
|
||||
// If -disablePerDayIndex is set, the effective search time range is
|
||||
// expected to be globalIndexTimeRange for both legacy and parition idb.
|
||||
// Otherwise:
|
||||
// - For the legacy idb, it must remain the same
|
||||
// - For the partition idb: its MaxTimestamp must be adjusted to match the
|
||||
// partition idb time range MaxTimestamp.
|
||||
searchTimeRange = TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: searchTimeRange.MinTimestamp,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: searchTimeRange.MinTimestamp,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
}
|
||||
|
||||
type testStorageSearchWithoutPerDayIndexOptions struct {
|
||||
type testStorageSearchWithoutIndexOptions struct {
|
||||
mrs []MetricRow
|
||||
assertSearchResult func(t *testing.T, s *Storage, tr TimeRange, want any)
|
||||
alwaysPerTimeRange bool // If true, use wantPerTimeRange instead of wantAll
|
||||
@@ -2958,17 +2956,18 @@ type testStorageSearchWithoutPerDayIndexOptions struct {
|
||||
wantEmpty any
|
||||
}
|
||||
|
||||
// testStorageSearchWithoutPerDayIndex tests how the search behaves when the
|
||||
// testStorageSearchWithoutIndex tests how the search behaves when the
|
||||
// per-day index is disabled. This function is expected to be called by
|
||||
// functions that test a particular search operation, such as GetTSDBStatus(),
|
||||
// SearchMetricNames(), etc.
|
||||
func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWithoutPerDayIndexOptions) {
|
||||
func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutIndexOptions) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
// The data is inserted and the search is performed when the per-day index
|
||||
// is enabled.
|
||||
t.Run("InsertAndSearchWithPerDayIndex", func(t *testing.T) {
|
||||
// The data is inserted and the search is performed when both global and
|
||||
// per-day indexes are enabled.
|
||||
t.Run("Add-Global-PerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -2979,10 +2978,11 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted and the search is performed when the per-day index
|
||||
// is disabled.
|
||||
t.Run("InsertAndSearchWithoutPerDayIndex", func(t *testing.T) {
|
||||
// The data is inserted and the search is performed when global index is
|
||||
// enabled and per-day index is disabled.
|
||||
t.Run("Add-Global-noPerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -2996,10 +2996,12 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when the per-day index is enabled but the search is
|
||||
// performed when the per-day index is disabled.
|
||||
t.Run("InsertWithPerDayIndexSearchWithout", func(t *testing.T) {
|
||||
// The data is inserted when both global and per-day indexes are enabled.
|
||||
// The search is performed when the global index is enabled and per-day
|
||||
// index is disabled.
|
||||
t.Run("Add-Global-PerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -3007,6 +3009,7 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
@@ -3018,11 +3021,14 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when the per-day index is disabled but the search is
|
||||
// performed when the per-day index is enabled. This case also shows that
|
||||
// registering metric names recovers the per-day index.
|
||||
t.Run("InsertWithoutPerDayIndexSearchWith", func(t *testing.T) {
|
||||
// The data is inserted when global index is enabled and per-day index is
|
||||
// disabled.
|
||||
// The search is performed when both global and per-day index is enabled.
|
||||
// This case also shows that registering metric names recovers the per-day
|
||||
// index.
|
||||
t.Run("Add-Global-noPerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -3030,6 +3036,7 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
|
||||
@@ -3046,15 +3053,120 @@ func testStorageSearchWithoutPerDayIndex(t *testing.T, opts *testStorageSearchWi
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted and the search is performed when global index is
|
||||
// disabled and per-day index is enabled.
|
||||
t.Run("Add-noGlobal-PerDay/Search-noGlobal-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, want)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when both global and per-day indexes are enabled.
|
||||
// The search is performed when the global index is disabled and per-day
|
||||
// index is enabled.
|
||||
t.Run("Add-Global-PerDay/Search-noGlobal-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, want)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when global index is disabled and per-day index is
|
||||
// enabled.
|
||||
// The search is performed when both global and per-day indexes are enabled.
|
||||
t.Run("Add-noGlobal-PerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, want)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when global index is disabled and per-day index is
|
||||
// enabled.
|
||||
// The search is performed when global index is enabled and per-day index is
|
||||
// disabled.
|
||||
t.Run("Add-noGlobal-PerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
for tr := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, opts.wantEmpty)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when global index is enabled and per-day index is
|
||||
// disabled.
|
||||
// The search is performed when the global index is disabled and per-day
|
||||
// index is enabled.
|
||||
t.Run("Add-Global-noPerDay/Search-noGlobal-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
for tr := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, opts.wantEmpty)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
}
|
||||
|
||||
func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: &TSDBStatus{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: &TSDBStatus{TotalSeries: days * rows},
|
||||
@@ -3094,16 +3206,16 @@ func TestStorageGetTSDBStatusWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageSearchMetricNamesWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3155,16 +3267,16 @@ func TestStorageSearchMetricNamesWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageSearchLabelNamesWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3209,17 +3321,17 @@ func TestStorageSearchLabelNamesWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageSearchLabelValuesWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
labelName = "job"
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3263,17 +3375,17 @@ func TestStorageSearchLabelValuesWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageSearchTagValueSuffixesWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
tagValuePrefix = "metric."
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3313,16 +3425,16 @@ func TestStorageSearchTagValueSuffixesWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageSearchGraphitePathsWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: []string{},
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
wantAll: []string{},
|
||||
@@ -3363,16 +3475,16 @@ func TestStorageSearchGraphitePathsWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
|
||||
func TestStorageQueryWithoutIndex(t *testing.T) {
|
||||
const (
|
||||
days = 4
|
||||
rows = 10
|
||||
)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
opts := testStorageSearchWithoutPerDayIndexOptions{
|
||||
opts := testStorageSearchWithoutIndexOptions{
|
||||
wantEmpty: []MetricRow(nil),
|
||||
wantPerTimeRange: make(map[TimeRange]any),
|
||||
alwaysPerTimeRange: true,
|
||||
@@ -3413,16 +3525,17 @@ func TestStorageQueryWithoutPerDayIndex(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
testStorageSearchWithoutPerDayIndex(t, &opts)
|
||||
testStorageSearchWithoutIndex(t, &opts)
|
||||
}
|
||||
|
||||
func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
f := func(t *testing.T, disablePerDayIndex bool) {
|
||||
f := func(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
t.Helper()
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -3455,12 +3568,21 @@ func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("disablePerDayIndex=false", func(t *testing.T) {
|
||||
f(t, false)
|
||||
})
|
||||
t.Run("disablePerDayIndex=true", func(t *testing.T) {
|
||||
f(t, true)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if disableGlobalIndex {
|
||||
t.Skip("TODO(@rtm0): zero timestamps cannot be searched with disabled global index")
|
||||
}
|
||||
f(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testSearchMetricIDs returns metricIDs for the given tfss and tr.
|
||||
@@ -3497,13 +3619,13 @@ func testCountAllMetricIDs(s *Storage, tr TimeRange) int {
|
||||
return len(ids)
|
||||
}
|
||||
|
||||
func TestStorageRegisterMetricNamesForVariousDataPatternsConcurrently(t *testing.T) {
|
||||
func TestStorageRegisterMetricNamesVariousDataPatterns(t *testing.T) {
|
||||
testStorageVariousDataPatternsConcurrently(t, true, func(s *Storage, mrs []MetricRow) {
|
||||
s.RegisterMetricNames(nil, mrs)
|
||||
})
|
||||
}
|
||||
|
||||
func TestStorageAddRowsForVariousDataPatternsConcurrently(t *testing.T) {
|
||||
func TestStorageAddRowsVariousDataPatterns(t *testing.T) {
|
||||
testStorageVariousDataPatternsConcurrently(t, false, func(s *Storage, mrs []MetricRow) {
|
||||
s.AddRows(mrs, defaultPrecisionBits)
|
||||
})
|
||||
@@ -3520,27 +3642,24 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
|
||||
|
||||
const concurrency = 4
|
||||
|
||||
disablePerDayIndex := false
|
||||
t.Run("perDayIndexes/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run("perDayIndexes/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run("perDayIndexes/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
|
||||
disablePerDayIndex = true
|
||||
t.Run("noPerDayIndexes/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run("noPerDayIndexes/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run("noPerDayIndexes/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
prefix := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(prefix+"/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run(prefix+"/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testStorageVariousDataPatterns tests the ingestion of different combinations
|
||||
@@ -3549,7 +3668,7 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
|
||||
// The function is intended to be used by other tests that define the
|
||||
// concurrency, the per-day index setting, and the operation (AddRows or
|
||||
// RegisterMetricNames) under test.
|
||||
func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOnly bool, op func(s *Storage, mrs []MetricRow), concurrency int, splitBatches bool) {
|
||||
func testStorageVariousDataPatterns(t *testing.T, disableGlobalIndex, disablePerDayIndex, registerOnly bool, op func(s *Storage, mrs []MetricRow), concurrency int, splitBatches bool) {
|
||||
f := func(t *testing.T, sameBatchMetricNames, sameRowMetricNames, sameBatchDates, sameRowDates bool) {
|
||||
batches, wantCounts := testGenerateMetricRowBatches(&batchOptions{
|
||||
numBatches: 3,
|
||||
@@ -3566,6 +3685,7 @@ func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOn
|
||||
rowsAddedTotal := wantCounts.metrics.RowsAddedTotal
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user