mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-08-08 00:35:50 +03:00
Compare commits
2 Commits
docs/remov
...
disable-gl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6e70f5859 | ||
|
|
4cb3f50477 |
@@ -94,9 +94,13 @@ var (
|
||||
cacheSizeIndexDBTagFilters = flagutil.NewBytes("storage.cacheSizeIndexDBTagFilters", 0, "Overrides max size for indexdb/tagFiltersToMetricIDs cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
|
||||
enableGlobalIndex = flag.Bool("enableGlobalIndex", false, "Enable global index. "+
|
||||
"Deployments with high churn rate should have this index disabled as this decreases disk space usage. "+
|
||||
"Such deployments may enable global index if the dominant query time range is > 1m as it may slightly improve query performance. "+
|
||||
"Also see -disablePerDayIndex and https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning")
|
||||
disablePerDayIndex = flag.Bool("disablePerDayIndex", false, "Disable per-day index and use global index for all searches. "+
|
||||
"This may improve performance and decrease disk space usage for the use cases with fixed set of timeseries scattered across a "+
|
||||
"big time range (for example, when loading years of historical data). "+
|
||||
"This may improve performance and decrease disk space usage for deployment with no/low churn rate. "+
|
||||
"Disabling per-day index forces enabling global index and the -enableGlobalIndex flag value is ignored."+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#index-tuning")
|
||||
trackMetricNamesStats = flag.Bool("storage.trackMetricNamesStats", true, "Whether to track ingest and query requests for timeseries metric names. "+
|
||||
"This feature allows to track metric names unused at query requests. "+
|
||||
@@ -148,6 +152,12 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
|
||||
if *idbPrefillStart > 23*time.Hour {
|
||||
logger.Panicf("-storage.idbPrefillStart cannot exceed 23 hours; got %s", idbPrefillStart)
|
||||
}
|
||||
disableGlobalIndex := !*enableGlobalIndex
|
||||
if *disablePerDayIndex {
|
||||
// In case if per-day index has been disabled, forcibly enable global
|
||||
// index even if -enableGlobalIndex flag is false.
|
||||
disableGlobalIndex = false
|
||||
}
|
||||
fs.RegisterPathFsMetrics(*storageDataPath)
|
||||
logger.Infof("opening storage at %q with -retentionPeriod=%s", *storageDataPath, retentionPeriod)
|
||||
startTime := time.Now()
|
||||
@@ -158,6 +168,7 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
|
||||
DenyQueriesOutsideRetention: *denyQueriesOutsideRetention,
|
||||
MaxHourlySeries: getMaxHourlySeries(),
|
||||
MaxDailySeries: getMaxDailySeries(),
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: *disablePerDayIndex,
|
||||
TrackMetricNamesStats: *trackMetricNamesStats,
|
||||
IDBPrefillStart: *idbPrefillStart,
|
||||
|
||||
@@ -16,6 +16,7 @@ func TestSingleSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
"-storageDataPath=" + tc.Dir() + "/vmsingle",
|
||||
"-retentionPeriod=100y",
|
||||
"-search.maxStalenessInterval=1m",
|
||||
"-enableGlobalIndex=true",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
})
|
||||
@@ -29,6 +30,7 @@ func TestClusterSearchWithDisabledPerDayIndex(t *testing.T) {
|
||||
vmstorage := tc.MustStartVmstorage("vmstorage-"+name, []string{
|
||||
"-storageDataPath=" + tc.Dir() + "/vmstorage",
|
||||
"-retentionPeriod=100y",
|
||||
"-enableGlobalIndex=true",
|
||||
fmt.Sprintf("-disablePerDayIndex=%t", disablePerDayIndex),
|
||||
})
|
||||
vminsert := tc.MustStartVminsert("vminsert-"+name, []string{
|
||||
|
||||
@@ -436,18 +436,6 @@ func (db *indexDB) createGlobalIndexes(tsid *TSID, mn *MetricName) {
|
||||
ii := getIndexItems()
|
||||
defer putIndexItems(ii)
|
||||
|
||||
if db.s.disablePerDayIndex {
|
||||
// Create metricName -> TSID entry.
|
||||
// This index is used for searching a TSID by metric name during data
|
||||
// ingestion or metric name registration when -disablePerDayIndex flag
|
||||
// is set.
|
||||
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricNameToTSID)
|
||||
ii.B = mn.Marshal(ii.B)
|
||||
ii.B = append(ii.B, kvSeparatorChar)
|
||||
ii.B = tsid.Marshal(ii.B)
|
||||
ii.Next()
|
||||
}
|
||||
|
||||
// Create metricID -> metricName entry.
|
||||
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricIDToMetricName)
|
||||
ii.B = encoding.MarshalUint64(ii.B, tsid.MetricID)
|
||||
@@ -460,11 +448,20 @@ func (db *indexDB) createGlobalIndexes(tsid *TSID, mn *MetricName) {
|
||||
ii.B = tsid.Marshal(ii.B)
|
||||
ii.Next()
|
||||
|
||||
// Create tag -> metricID entries for every tag in mn.
|
||||
kb := kbPool.Get()
|
||||
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
ii.registerTagIndexes(kb.B, mn, tsid.MetricID)
|
||||
kbPool.Put(kb)
|
||||
if !db.s.disableGlobalIndex {
|
||||
// Create metricName -> TSID entry.
|
||||
ii.B = marshalCommonPrefix(ii.B, nsPrefixMetricNameToTSID)
|
||||
ii.B = mn.Marshal(ii.B)
|
||||
ii.B = append(ii.B, kvSeparatorChar)
|
||||
ii.B = tsid.Marshal(ii.B)
|
||||
ii.Next()
|
||||
|
||||
// Create tag -> metricID entries for every tag in mn.
|
||||
kb := kbPool.Get()
|
||||
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
ii.registerTagIndexes(kb.B, mn, tsid.MetricID)
|
||||
kbPool.Put(kb)
|
||||
}
|
||||
|
||||
db.tb.AddItems(ii.Items)
|
||||
}
|
||||
@@ -759,6 +756,7 @@ func (db *indexDB) SearchLabelValues(qt *querytracer.Tracer, labelName string, t
|
||||
func filterLabelValues(lvs map[string]struct{}, tf *tagFilter, key string) {
|
||||
var b []byte
|
||||
for lv := range lvs {
|
||||
// TODO
|
||||
b = marshalCommonPrefix(b[:0], nsPrefixTagToMetricIDs)
|
||||
b = marshalTagValue(b, bytesutil.ToUnsafeBytes(key))
|
||||
b = marshalTagValue(b, bytesutil.ToUnsafeBytes(lv))
|
||||
@@ -1517,10 +1515,11 @@ func (db *indexDB) DeleteSeries(qt *querytracer.Tracer, tfss []*TagFilters, maxM
|
||||
is := db.getIndexSearch(noDeadline)
|
||||
defer db.putIndexSearch(is)
|
||||
|
||||
// Unconditionally search global index since a given day in per-day
|
||||
// index may not contain the full set of metricIDs that correspond
|
||||
// to the tfss.
|
||||
metricIDs, err := is.searchMetricIDs(qt, tfss, globalIndexTimeRange, maxMetrics)
|
||||
tr := globalIndexTimeRange
|
||||
if db.s.disableGlobalIndex {
|
||||
tr = db.tr
|
||||
}
|
||||
metricIDs, err := is.searchMetricIDs(qt, tfss, tr, maxMetrics)
|
||||
if err != nil {
|
||||
return nil, db.wrapError("delete series", err)
|
||||
}
|
||||
@@ -1967,6 +1966,7 @@ func (is *indexSearch) updateMetricIDsByMetricNameMatch(qt *querytracer.Tracer,
|
||||
qt.Printf("sort %d metric ids", len(sortedMetricIDs))
|
||||
|
||||
kb := &is.kb
|
||||
// TODO
|
||||
kb.B = is.marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
tfs = removeCompositeTagFilters(tfs, kb.B)
|
||||
|
||||
@@ -2077,6 +2077,7 @@ func hasCompositeTagFilters(tfs []*tagFilter, prefix []byte) bool {
|
||||
}
|
||||
|
||||
func matchTagFilters(mn *MetricName, tfs []*tagFilter, kb *bytesutil.ByteBuffer) (bool, error) {
|
||||
// TODO
|
||||
kb.B = marshalCommonPrefix(kb.B[:0], nsPrefixTagToMetricIDs)
|
||||
for i, tf := range tfs {
|
||||
if bytes.Equal(tf.key, graphiteReverseTagKey) {
|
||||
@@ -3236,6 +3237,7 @@ func (mp *tagToMetricIDsRowParser) GetMatchingSeriesCount(filter, negativeFilter
|
||||
}
|
||||
|
||||
func mergeTagToMetricIDsRows(data []byte, items []mergeset.Item) ([]byte, []mergeset.Item) {
|
||||
// TODO
|
||||
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixTagToMetricIDs)
|
||||
data, items = mergeTagToMetricIDsRowsInternal(data, items, nsPrefixDateTagToMetricIDs)
|
||||
return data, items
|
||||
|
||||
@@ -2,10 +2,10 @@ package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
|
||||
@@ -79,7 +79,7 @@ func mustOpenLegacyIndexDB(path string, s *Storage) *legacyIndexDB {
|
||||
|
||||
tr := TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: math.MaxInt64,
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
}
|
||||
idb := mustOpenIndexDB(id, tr, name, path, s, &s.isReadOnly, true)
|
||||
legacyIDB := &legacyIndexDB{idb: idb}
|
||||
|
||||
@@ -486,7 +486,7 @@ func TestIndexDBOpenClose(t *testing.T) {
|
||||
func TestIndexDB(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
f := func(t *testing.T, concurrency int, disablePerDayIndex bool) {
|
||||
f := func(t *testing.T, concurrency int, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const metricGroups = 10
|
||||
now := time.Now().UTC()
|
||||
timestamp := now.UnixMilli()
|
||||
@@ -500,6 +500,7 @@ func TestIndexDB(t *testing.T) {
|
||||
}
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -531,13 +532,19 @@ func TestIndexDB(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, concurrency := range []int{1, 4} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f(t, concurrency, disablePerDayIndex)
|
||||
// Repeat the same test on non-empty reopened storage.
|
||||
f(t, concurrency, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("concurrency=%d/disableGlobalIndex=%t/disablePerDayIndex=%t", concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f(t, concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
// Repeat the same test on non-empty reopened storage.
|
||||
f(t, concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1400,15 +1407,21 @@ func TestMatchTagFilters(t *testing.T) {
|
||||
|
||||
func TestIndexDBSearchTSIDs(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchTSIDs(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchTSIDs(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBSearchTSIDs(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
@@ -1417,6 +1430,7 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
allMetricIDs := &uint64set.Set{}
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1474,8 +1488,10 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
assertMetricIDs(date, metricsPerDay, perDayMetricIDs[date])
|
||||
}
|
||||
}
|
||||
// Check that all the metrics are found in global index
|
||||
assertMetricIDs(globalIndexDate, metricsPerDay*days, allMetricIDs)
|
||||
if !disableGlobalIndex {
|
||||
// Check that all the metrics are found in global index
|
||||
assertMetricIDs(globalIndexDate, metricsPerDay*days, allMetricIDs)
|
||||
}
|
||||
|
||||
db.putIndexSearch(is2)
|
||||
|
||||
@@ -1522,21 +1538,28 @@ func testIndexDBSearchTSIDs(t *testing.T, disablePerDayIndex bool) {
|
||||
|
||||
func TestIndexDBSearchLabelNames(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelNames(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelNames(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBSearchLabelNames(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBSearchLabelNames(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
baseDate := uint64(timestamp) / msecPerDay
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1633,16 +1656,21 @@ func testIndexDBSearchLabelNames(t *testing.T, disablePerDayIndex bool) {
|
||||
|
||||
func TestIndexDBSearchLabelValues(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelValues(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBSearchLabelValues(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBSearchLabelValues(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
@@ -1650,6 +1678,7 @@ func testIndexDBSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
|
||||
var allMetricNames []string
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1816,15 +1845,21 @@ func TestFilterLabelValues(t *testing.T) {
|
||||
|
||||
func TestIndexDBDeleteSeries(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBDeleteSeries(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBDeleteSeries(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBDeleteSeries(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
trAllDays := TimeRange{
|
||||
@@ -1837,6 +1872,7 @@ func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
|
||||
metricNamesByDate := make(map[uint64][]string)
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -1922,21 +1958,28 @@ func testIndexDBDeleteSeries(t *testing.T, disablePerDayIndex bool) {
|
||||
|
||||
func TestIndexDBGetTSDBStatus(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBGetTSDBStatus(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testIndexDBGetTSDBStatus(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testIndexDBGetTSDBStatus(t *testing.T, disablePerDayIndex bool) {
|
||||
func testIndexDBGetTSDBStatus(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
const days = 5
|
||||
const metricsPerDay = 1000
|
||||
timestamp := time.Date(2019, time.October, 15, 5, 1, 0, 0, time.UTC).UnixMilli()
|
||||
baseDate := uint64(timestamp) / msecPerDay
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
|
||||
@@ -80,6 +80,7 @@ type Storage struct {
|
||||
// compatibility with partition index.
|
||||
legacyIndexDBs atomic.Pointer[legacyIndexDBs]
|
||||
|
||||
disableGlobalIndex bool
|
||||
disablePerDayIndex bool
|
||||
|
||||
tb *table
|
||||
@@ -169,6 +170,7 @@ type OpenOptions struct {
|
||||
DenyQueriesOutsideRetention bool
|
||||
MaxHourlySeries int
|
||||
MaxDailySeries int
|
||||
DisableGlobalIndex bool
|
||||
DisablePerDayIndex bool
|
||||
TrackMetricNamesStats bool
|
||||
IDBPrefillStart time.Duration
|
||||
@@ -269,6 +271,10 @@ func MustOpenStorage(path string, opts OpenOptions) *Storage {
|
||||
fs.MustMkdirIfNotExist(metadataDir)
|
||||
s.minTimestampForCompositeIndex = mustGetMinTimestampForCompositeIndex(metadataDir, isEmptyDB)
|
||||
|
||||
if opts.DisableGlobalIndex && opts.DisablePerDayIndex {
|
||||
logger.Panicf("BUG: global and per-day indexes cannot be disabled at the same time")
|
||||
}
|
||||
s.disableGlobalIndex = opts.DisableGlobalIndex
|
||||
s.disablePerDayIndex = opts.DisablePerDayIndex
|
||||
|
||||
// Load legacy indexDBs.
|
||||
@@ -1732,6 +1738,12 @@ const maxDaysForPerDaySearch = 40
|
||||
// globalIndexTimeRange based on the time range length and -disablePerDayIndex
|
||||
// flag.
|
||||
func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
|
||||
// if the global index is disabled and the searchTR is the
|
||||
// globalIndexTimeRange, use idbTR.
|
||||
if s.disableGlobalIndex && searchTR == globalIndexTimeRange {
|
||||
return idbTR
|
||||
}
|
||||
|
||||
// If the per day index is disabled, unconditionally search global index.
|
||||
if s.disablePerDayIndex {
|
||||
return globalIndexTimeRange
|
||||
@@ -1748,7 +1760,7 @@ func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
|
||||
// For legacy IndexDBs only, partition indexDBs can't span more than a
|
||||
// month.
|
||||
minDate, maxDate := tr.DateRange()
|
||||
if maxDate-minDate > maxDaysForPerDaySearch {
|
||||
if !s.disableGlobalIndex && maxDate-minDate > maxDaysForPerDaySearch {
|
||||
return globalIndexTimeRange
|
||||
}
|
||||
|
||||
@@ -1756,7 +1768,7 @@ func (s *Storage) adjustTimeRange(searchTR, idbTR TimeRange) TimeRange {
|
||||
// the idb time range, then return globalIndexTimeRange to indicate that we
|
||||
// want to search the global index since the entire index db needs to be
|
||||
// searched anyway.
|
||||
if tr == idbTR {
|
||||
if !s.disableGlobalIndex && tr == idbTR {
|
||||
return globalIndexTimeRange
|
||||
}
|
||||
|
||||
|
||||
@@ -531,22 +531,29 @@ func TestStorageDeleteSeries(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
for _, concurrency := range []int{1, 4} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageDeleteSeries(t, concurrency, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("concurrency=%d/disableGlobalIndex=%t/disablePerDayIndex=%t", concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageDeleteSeries(t, concurrency, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testStorageDeleteSeries(t *testing.T, concurrency int, disablePerDayIndex bool) {
|
||||
func testStorageDeleteSeries(t *testing.T, concurrency int, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
tr := TimeRange{
|
||||
MinTimestamp: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC).UnixMilli(),
|
||||
MaxTimestamp: time.Date(2026, 1, 15, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
|
||||
}
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -2821,10 +2828,11 @@ func TestStorageGetTSDBStatus(t *testing.T) {
|
||||
func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
f := func(disablePerDayIndex bool, searchTR, idbTR, want TimeRange) {
|
||||
f := func(disableGlobalIndex, disablePerDayIndex bool, searchTR, idbTR, want TimeRange) {
|
||||
t.Helper()
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -2835,7 +2843,7 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
legacyIDBTimeRange := TimeRange{
|
||||
MinTimestamp: 0,
|
||||
MaxTimestamp: math.MaxInt64,
|
||||
MaxTimestamp: time.Now().UnixMilli(),
|
||||
}
|
||||
partitionIDBTimeRange := TimeRange{
|
||||
MinTimestamp: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
|
||||
@@ -2845,10 +2853,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
|
||||
// Search time range is the same as globalIndexTimeRange.
|
||||
searchTimeRange = globalIndexTimeRange
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, legacyIDBTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
|
||||
|
||||
// The search time range is smaller than a month (and therefore < 40 days)
|
||||
// and is fully included into the partition idb time range.
|
||||
@@ -2856,17 +2866,21 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
|
||||
// The search time range is the same as partition idb time range.
|
||||
searchTimeRange = partitionIDBTimeRange
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, searchTimeRange)
|
||||
|
||||
// The search time range is smaller than 40 days and fully includes the
|
||||
// partition idb time range.
|
||||
@@ -2874,10 +2888,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
|
||||
|
||||
// The search time range is 41 days and fully includes the partition idb
|
||||
// time range.
|
||||
@@ -2885,10 +2901,12 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + 41*msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, partitionIDBTimeRange)
|
||||
|
||||
// The search time range is smaller than 40 days and overlaps with partition
|
||||
// idb time range on the left.
|
||||
@@ -2896,13 +2914,18 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MinTimestamp + msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp,
|
||||
MaxTimestamp: searchTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: partitionIDBTimeRange.MinTimestamp,
|
||||
MaxTimestamp: searchTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
|
||||
// The search time range is smaller than 40 days and overlaps with partition
|
||||
// idb time range on the right.
|
||||
@@ -2910,13 +2933,18 @@ func TestStorageAdjustTimeRange(t *testing.T) {
|
||||
MinTimestamp: partitionIDBTimeRange.MaxTimestamp - msecPerDay,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp + msecPerDay,
|
||||
}
|
||||
f(false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
f(false, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(false, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: searchTimeRange.MinTimestamp,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(false, true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(false, true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, false, searchTimeRange, legacyIDBTimeRange, searchTimeRange)
|
||||
f(true, false, searchTimeRange, partitionIDBTimeRange, TimeRange{
|
||||
MinTimestamp: searchTimeRange.MinTimestamp,
|
||||
MaxTimestamp: partitionIDBTimeRange.MaxTimestamp,
|
||||
})
|
||||
f(true, searchTimeRange, legacyIDBTimeRange, globalIndexTimeRange)
|
||||
f(true, searchTimeRange, partitionIDBTimeRange, globalIndexTimeRange)
|
||||
}
|
||||
|
||||
type testStorageSearchWithoutIndexOptions struct {
|
||||
@@ -2935,9 +2963,11 @@ type testStorageSearchWithoutIndexOptions struct {
|
||||
func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutIndexOptions) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
// The data is inserted and the search is performed when per-day index is enabled.
|
||||
// The data is inserted and the search is performed when both global and
|
||||
// per-day indexes are enabled.
|
||||
t.Run("Add-Global-PerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -2948,9 +2978,11 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted and the search is performed when per-day index is disabled.
|
||||
// The data is inserted and the search is performed when global index is
|
||||
// enabled and per-day index is disabled.
|
||||
t.Run("Add-Global-noPerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -2964,10 +2996,12 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when per-day index are enabled.
|
||||
// The search is performed when per-day index is disabled.
|
||||
// The data is inserted when both global and per-day indexes are enabled.
|
||||
// The search is performed when the global index is enabled and per-day
|
||||
// index is disabled.
|
||||
t.Run("Add-Global-PerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -2975,6 +3009,7 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
@@ -2986,12 +3021,14 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when per-day index is disabled.
|
||||
// The search is performed when per-day index is enabled.
|
||||
// The data is inserted when global index is enabled and per-day index is
|
||||
// disabled.
|
||||
// The search is performed when both global and per-day index is enabled.
|
||||
// This case also shows that registering metric names recovers the per-day
|
||||
// index.
|
||||
t.Run("Add-Global-noPerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
@@ -2999,6 +3036,7 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
|
||||
@@ -3015,6 +3053,111 @@ func testStorageSearchWithoutIndex(t *testing.T, opts *testStorageSearchWithoutI
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted and the search is performed when global index is
|
||||
// disabled and per-day index is enabled.
|
||||
t.Run("Add-noGlobal-PerDay/Search-noGlobal-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, want)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when both global and per-day indexes are enabled.
|
||||
// The search is performed when the global index is disabled and per-day
|
||||
// index is enabled.
|
||||
t.Run("Add-Global-PerDay/Search-noGlobal-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, want)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when global index is disabled and per-day index is
|
||||
// enabled.
|
||||
// The search is performed when both global and per-day indexes are enabled.
|
||||
t.Run("Add-noGlobal-PerDay/Search-Global-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
for tr, want := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, want)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when global index is disabled and per-day index is
|
||||
// enabled.
|
||||
// The search is performed when global index is enabled and per-day index is
|
||||
// disabled.
|
||||
t.Run("Add-noGlobal-PerDay/Search-Global-noPerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
for tr := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, opts.wantEmpty)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
|
||||
// The data is inserted when global index is enabled and per-day index is
|
||||
// disabled.
|
||||
// The search is performed when the global index is disabled and per-day
|
||||
// index is enabled.
|
||||
t.Run("Add-Global-noPerDay/Search-noGlobal-PerDay", func(t *testing.T) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: false,
|
||||
DisablePerDayIndex: true,
|
||||
})
|
||||
s.AddRows(opts.mrs, defaultPrecisionBits)
|
||||
s.DebugFlush()
|
||||
s.MustClose()
|
||||
|
||||
s = MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: true,
|
||||
DisablePerDayIndex: false,
|
||||
})
|
||||
for tr := range opts.wantPerTimeRange {
|
||||
opts.assertSearchResult(t, s, tr, opts.wantEmpty)
|
||||
}
|
||||
s.MustClose()
|
||||
})
|
||||
}
|
||||
|
||||
func TestStorageGetTSDBStatusWithoutIndex(t *testing.T) {
|
||||
@@ -3388,16 +3531,23 @@ func TestStorageQueryWithoutIndex(t *testing.T) {
|
||||
func TestStorageAddRowsWithZeroDate(t *testing.T) {
|
||||
defer testRemoveAll(t)
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageAddRowsWithZeroDate(t, disablePerDayIndex)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testStorageAddRowsWithZeroDate(t, disableGlobalIndex, disablePerDayIndex)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testStorageAddRowsWithZeroDate(t *testing.T, disablePerDayIndex bool) {
|
||||
func testStorageAddRowsWithZeroDate(t *testing.T, disableGlobalIndex, disablePerDayIndex bool) {
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
defer s.MustClose()
|
||||
@@ -3626,17 +3776,23 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
|
||||
|
||||
const concurrency = 4
|
||||
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
prefix := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
|
||||
t.Run(prefix+"/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run(prefix+"/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
for _, disableGlobalIndex := range []bool{false, true} {
|
||||
for _, disablePerDayIndex := range []bool{false, true} {
|
||||
if disableGlobalIndex && disablePerDayIndex {
|
||||
// Both indexes cannot be disabled at the same time.
|
||||
continue
|
||||
}
|
||||
prefix := fmt.Sprintf("disableGlobalIndex=%t/disablePerDayIndex=%t", disableGlobalIndex, disablePerDayIndex)
|
||||
t.Run(prefix+"/serial", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, 1, false)
|
||||
})
|
||||
t.Run(prefix+"/concurrentRows", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, concurrency, true)
|
||||
})
|
||||
t.Run(prefix+"/concurrentBatches", func(t *testing.T) {
|
||||
testStorageVariousDataPatterns(t, disableGlobalIndex, disablePerDayIndex, registerOnly, op, concurrency, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3646,7 +3802,7 @@ func testStorageVariousDataPatternsConcurrently(t *testing.T, registerOnly bool,
|
||||
// The function is intended to be used by other tests that define the
|
||||
// concurrency, the per-day index setting, and the operation (AddRows or
|
||||
// RegisterMetricNames) under test.
|
||||
func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOnly bool, op func(s *Storage, mrs []MetricRow), concurrency int, splitBatches bool) {
|
||||
func testStorageVariousDataPatterns(t *testing.T, disableGlobalIndex, disablePerDayIndex, registerOnly bool, op func(s *Storage, mrs []MetricRow), concurrency int, splitBatches bool) {
|
||||
f := func(t *testing.T, sameBatchMetricNames, sameRowMetricNames, sameBatchDates, sameRowDates bool) {
|
||||
batches, wantCounts := testGenerateMetricRowBatches(&batchOptions{
|
||||
numBatches: 3,
|
||||
@@ -3663,6 +3819,7 @@ func testStorageVariousDataPatterns(t *testing.T, disablePerDayIndex, registerOn
|
||||
rowsAddedTotal := wantCounts.metrics.RowsAddedTotal
|
||||
|
||||
s := MustOpenStorage(t.Name(), OpenOptions{
|
||||
DisableGlobalIndex: disableGlobalIndex,
|
||||
DisablePerDayIndex: disablePerDayIndex,
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user