Compare commits

...

2 Commits

Author SHA1 Message Date
Artem Fetishev
8c496280f5 lib/storage: rewrite indexdb tests
Signed-off-by: Artem Fetishev <rtm@victoriametrics.com>
2026-07-24 17:32:45 +02:00
Zasda Yusuf Mikail
07b6070193 app/vmselect/promql: drop data points with missing samples on the right side of comparison operations
NaN values in non-scalar operands of comparison operations mean missing
samples, so they cannot match any sample on the other side. Previously
such data points were passed to the comparison function, so queries like
`foo != (bar > 100)` could return unexpected results, since `value !=
NaN` evaluates to `true`. This PR drops such data points in the same way
as Prometheus does.

Comparisons to scalars such as `foo != nan` keep the previous behavior,
since a scalar `NaN` is an explicit value rather than a missing sample
(see the existing `compare_to_nan_right` test case).

The check is applied only to the right side of the operation, since a
missing sample on the left side already results in `NaN` for every
comparison operation:
- `==`, `>`, `<`, `>=`, `<=`: `cf(NaN, b)` is `false`, so the data point
is dropped;
- `!=`: `cf(NaN, b)` is `true`, but the returned value is the left one,
which is `NaN`;
- `bool` modifier: the left `NaN` is handled explicitly in
`newBinaryOpCmpFunc`.

Verified with the reproduction steps from the linked issue: the query
`tsafdb0_value{table="t1", timeSeries="c0"} != ((272) <=
(tsafdb0_value{table="t1", timeSeries="c0"}))` now returns an empty
result, matching Prometheus behavior.

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018
2026-07-24 15:59:45 +02:00
4 changed files with 450 additions and 309 deletions

View File

@@ -164,11 +164,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
left := bfa.left
right := bfa.right
op := bfa.be.Op
switch true {
case metricsql.IsBinaryOpCmp(op):
// Do not remove empty series for comparison operations,
// since this may lead to missing result.
default:
isCmpOp := metricsql.IsBinaryOpCmp(op)
if !isCmpOp {
// Do not remove empty series for comparison operations, since NaN can be an
// explicitly present sample value. In particular, `value != NaN` is true.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
left = removeEmptySeries(left)
right = removeEmptySeries(right)
}
@@ -191,6 +191,14 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
isBool := bfa.be.Bool
fillLeft := bfa.be.FillLeft
fillRight := bfa.be.FillRight
// A NaN produced by a vector comparison denotes a filtered-out sample rather
// than an explicitly present NaN value. Drop it when this result is used as
// the right-hand side of another comparison.
// Note that filtered-out samples surviving as NaN inside other wrappers such as
// transform functions or arithmetic operations aren't detected, since such NaN
// is indistinguishable from an explicitly present NaN value there.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
for i, tsLeft := range left {
leftValues := tsLeft.Values
rightValues := right[i].Values
@@ -203,11 +211,16 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
b := rightValues[j]
leftIsNaN := math.IsNaN(a)
rightIsNaN := math.IsNaN(b)
// apply the fill value when either the left or right side is NaN, but not both.
// Both sides are NaN.
if leftIsNaN && rightIsNaN {
dstValues[j] = bf(a, b, isBool)
continue
}
if dropNaNRight && rightIsNaN && fillRight == nil {
dstValues[j] = nan
continue
}
// Apply the fill value when either the left or right side is NaN, but not both.
if leftIsNaN && fillLeft != nil {
a = fillLeft.N
}
@@ -223,6 +236,38 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
}
}
// isVectorComparisonExpr returns whether e is a comparison operation
// with at least one non-scalar operand.
func isVectorComparisonExpr(e metricsql.Expr) bool {
if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
// the shape of the result. Subqueries are left as is, since rollup
// functions skip NaN values on their own.
e = re.Expr
}
be, ok := e.(*metricsql.BinaryOpExpr)
if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
return false
}
return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
}
// isScalarLikeExpr returns whether e always evaluates to a scalar value.
func isScalarLikeExpr(e metricsql.Expr) bool {
switch e := e.(type) {
case *metricsql.NumberExpr, *metricsql.DurationExpr:
return true
case *metricsql.BinaryOpExpr:
return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
case *metricsql.FuncExpr:
switch strings.ToLower(e.Name) {
case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
return true
}
}
return false
}
func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
if isScalar(left) {

View File

@@ -2994,6 +2994,136 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != NaN`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != 1200`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, nan, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != (1 > 2)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
// A missing sample on the left side results in NaN for every comparison
// operation, so no special handling is needed there.
t.Parallel()
q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
// Prometheus drops the timestamps filtered out by the comparison on the right.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
t.Parallel()
q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
t.Parallel()
q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`-1 < 2`, func(t *testing.T) {
t.Parallel()
q := `-1 < 2`

View File

@@ -34,6 +34,7 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
@@ -122,6 +123,7 @@ Released at 2026-06-22
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
Released at 2026-06-08

View File

@@ -1400,10 +1400,22 @@ func TestMatchTagFilters(t *testing.T) {
}
func TestSearchTSIDWithTimeRange(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testSearchTSIDWithTimeRange(t, disablePerDayIndex)
})
}
}
func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
// TODO: @f41gh7 refactor this test:
// create a new test for LabelNames
// move exist LabelVales tests into TestSearchLabelValues
const path = "TestSearchTSIDWithTimeRange"
// Create a bunch of per-day time series
const days = 5
const metricsPerDay = 1000
@@ -1423,28 +1435,20 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
newMN := func(name string, day, metric int) MetricName {
var mn MetricName
mn.MetricGroup = []byte(name)
mn.AddTag(
"constant",
"const",
)
mn.AddTag(
"day",
fmt.Sprintf("%v", day),
)
mn.AddTag(
"UniqueId",
fmt.Sprintf("%v", metric),
)
mn.AddTag(
"some_unique_id",
fmt.Sprintf("%v", day),
)
mn.AddTag("constant", "const")
mn.AddTag("day", fmt.Sprintf("%v", day))
mn.AddTag("UniqueId", fmt.Sprintf("%v", metric))
mn.AddTag("some_unique_id", fmt.Sprintf("%v", day))
mn.sortTags()
return mn
}
s := MustOpenStorage(path, OpenOptions{})
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
ptw := s.tb.MustGetPartition(timestamp)
defer s.tb.PutPartition(ptw)
db := ptw.pt.idb
is := db.getIndexSearch(noDeadline)
@@ -1472,25 +1476,28 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
is2 := db.getIndexSearch(noDeadline)
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
if !disablePerDayIndex {
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
}
}
}
// Check that all the metrics are found in global index
metricIDs, err := is2.getMetricIDsForDate(0, metricsPerDay*days)
metricIDs, err := is2.getMetricIDsForDate(globalIndexDate, metricsPerDay*days)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !allMetricIDs.Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), allMetricIDs.AppendTo(nil))
}
db.putIndexSearch(is2)
// add a metric that will be deleted shortly
@@ -1498,10 +1505,7 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
day := days
date := baseDate - uint64(day)
mn := newMN("deletedMetric", day, 999)
mn.AddTag(
"labelToDelete",
fmt.Sprintf("%v", day),
)
mn.AddTag("labelToDelete", fmt.Sprintf("%v", day))
mn.sortTags()
metricNameBuf = mn.Marshal(metricNameBuf[:0])
var tsid TSID
@@ -1521,6 +1525,9 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
MinTimestamp: timestamp - msecPerDay,
MaxTimestamp: timestamp,
}
if disablePerDayIndex {
tr = globalIndexTimeRange
}
lns, err := db.SearchLabelNames(nil, nil, tr, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelNames(timeRange=%s): %s", &tr, err)
@@ -1563,22 +1570,17 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
MinTimestamp: timestamp - 2*msecPerHour - 1,
MaxTimestamp: timestamp,
}
wantNumMatchedTSIDs := metricsPerDay
if disablePerDayIndex {
tr = globalIndexTimeRange
wantNumMatchedTSIDs = days * metricsPerDay
}
matchedTSIDs, err := db.SearchTSIDs(nil, []*TagFilters{tfs}, tr, 1e5, noDeadline)
if err != nil {
t.Fatalf("error searching tsids: %v", err)
}
if len(matchedTSIDs) != metricsPerDay {
t.Fatalf("expected %d time series for current day, got %d time series", metricsPerDay, len(matchedTSIDs))
}
// Check SearchLabelNames with the specified filter.
lns, err = db.SearchLabelNames(nil, []*TagFilters{tfs}, TimeRange{}, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelNames(filters=%s): %s", tfs, err)
}
got = sortedSlice(lns)
if !reflect.DeepEqual(got, labelNames) {
t.Fatalf("unexpected labelNames; got\n%s\nwant\n%s", got, labelNames)
if len(matchedTSIDs) != wantNumMatchedTSIDs {
t.Fatalf("expected %d time series for current day, got %d time series", wantNumMatchedTSIDs, len(matchedTSIDs))
}
// Check SearchLabelNames with the specified filter and time range.
@@ -1611,16 +1613,6 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
t.Fatalf("unexpected labelNames; got\n%s\nwant\n%s", got, labelNames)
}
// Check SearchLabelValues with the specified filter.
lvs, err = db.SearchLabelValues(nil, "", []*TagFilters{tfs}, TimeRange{}, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelValues(filters=%s): %s", tfs, err)
}
got = sortedSlice(lvs)
if !reflect.DeepEqual(got, labelValues) {
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", got, labelValues)
}
// Check SearchLabelValues with the specified filter and time range.
lvs, err = db.SearchLabelValues(nil, "", []*TagFilters{tfs}, tr, 10000, 1e9, noDeadline)
if err != nil {
@@ -1657,6 +1649,9 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
MinTimestamp: timestamp - msecPerDay*days,
MaxTimestamp: timestamp,
}
if disablePerDayIndex {
tr = globalIndexTimeRange
}
matchedTSIDs, err = db.SearchTSIDs(nil, []*TagFilters{tfs}, tr, 1e5, noDeadline)
if err != nil {
@@ -1666,256 +1661,219 @@ func TestSearchTSIDWithTimeRange(t *testing.T) {
t.Fatalf("expected %d time series for all days, got %d time series", metricsPerDay*days, len(matchedTSIDs))
}
// Check GetTSDBStatus with nil filters.
status, err := db.GetTSDBStatus(nil, nil, baseDate, "day", 5, 1e6, noDeadline)
// Get TSDB status with nil filters.
tsdbStatusDate := baseDate
if disablePerDayIndex {
tsdbStatusDate = globalIndexDate
}
status, err := db.GetTSDBStatus(nil, nil, tsdbStatusDate, "day", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus with nil filters: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName := []TopHeapEntry{
{
Name: "testMetric",
Count: 1000,
wantStatus := &TSDBStatus{
TotalSeries: metricsPerDay,
TotalLabelValuePairs: 5 * metricsPerDay,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: metricsPerDay},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: metricsPerDay},
{Name: "constant", Count: metricsPerDay},
{Name: "day", Count: metricsPerDay},
{Name: "some_unique_id", Count: metricsPerDay},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{
{Name: "0", Count: metricsPerDay},
},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: metricsPerDay},
{Name: "constant=const", Count: metricsPerDay},
{Name: "day=0", Count: metricsPerDay},
{Name: "some_unique_id=0", Count: metricsPerDay},
{Name: "UniqueId=1", Count: 1},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
{Name: "day", Count: 1},
{Name: "some_unique_id", Count: 1},
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
if disablePerDayIndex {
wantStatus = &TSDBStatus{
TotalSeries: days * metricsPerDay,
TotalLabelValuePairs: days * 5 * metricsPerDay,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: days * metricsPerDay},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: days * metricsPerDay},
{Name: "__name__", Count: days * metricsPerDay},
{Name: "constant", Count: days * metricsPerDay},
{Name: "day", Count: days * metricsPerDay},
{Name: "some_unique_id", Count: days * metricsPerDay},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{
{Name: "0", Count: metricsPerDay},
{Name: "1", Count: metricsPerDay},
{Name: "2", Count: metricsPerDay},
{Name: "3", Count: metricsPerDay},
{Name: "4", Count: metricsPerDay},
},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: days * metricsPerDay},
{Name: "constant=const", Count: days * metricsPerDay},
{Name: "day=0", Count: metricsPerDay},
{Name: "day=1", Count: metricsPerDay},
{Name: "day=2", Count: metricsPerDay},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "day", Count: days},
{Name: "some_unique_id", Count: days},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
},
}
}
expectedSeriesCountByLabelName := []TopHeapEntry{
{
Name: "UniqueId",
Count: 1000,
},
{
Name: "__name__",
Count: 1000,
},
{
Name: "constant",
Count: 1000,
},
{
Name: "day",
Count: 1000,
},
{
Name: "some_unique_id",
Count: 1000,
},
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
}
if !reflect.DeepEqual(status.SeriesCountByLabelName, expectedSeriesCountByLabelName) {
t.Fatalf("unexpected SeriesCountByLabelName;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelName, expectedSeriesCountByLabelName)
// Get TSDB status with non-nil filter which matches all series.
tfs = NewTagFilters()
if err := tfs.Add([]byte("constant"), []byte("const"), false, false); err != nil {
t.Fatalf("cannot add filter: %s", err)
}
expectedSeriesCountByFocusLabelValue := []TopHeapEntry{
{
Name: "0",
Count: 1000,
},
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, tsdbStatusDate, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !reflect.DeepEqual(status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) {
t.Fatalf("unexpected SeriesCountByFocusLabelValue;\ngot\n%v\nwant\n%v", status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue)
}
expectedLabelValueCountByLabelName := []TopHeapEntry{
{
Name: "UniqueId",
Count: 1000,
},
{
Name: "__name__",
Count: 1,
},
{
Name: "constant",
Count: 1,
},
{
Name: "day",
Count: 1,
},
{
Name: "some_unique_id",
Count: 1,
},
}
if !reflect.DeepEqual(status.LabelValueCountByLabelName, expectedLabelValueCountByLabelName) {
t.Fatalf("unexpected LabelValueCountByLabelName;\ngot\n%v\nwant\n%v", status.LabelValueCountByLabelName, expectedLabelValueCountByLabelName)
}
expectedSeriesCountByLabelValuePair := []TopHeapEntry{
{
Name: "__name__=testMetric",
Count: 1000,
},
{
Name: "constant=const",
Count: 1000,
},
{
Name: "day=0",
Count: 1000,
},
{
Name: "some_unique_id=0",
Count: 1000,
},
{
Name: "UniqueId=1",
Count: 1,
},
}
if !reflect.DeepEqual(status.SeriesCountByLabelValuePair, expectedSeriesCountByLabelValuePair) {
t.Fatalf("unexpected SeriesCountByLabelValuePair;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelValuePair, expectedSeriesCountByLabelValuePair)
}
expectedTotalSeries := uint64(1000)
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs := uint64(5000)
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
// The result must be the same as with no filters (see above) except that
// the SeriesCountByFocusLabelValue must be empty.
wantStatus.SeriesCountByFocusLabelValue = []TopHeapEntry{}
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
}
// Check GetTSDBStatus with non-nil filter, which matches all the series
// Get TSDB status with non-nil filter that matches all the series on a
// given day.
tfs = NewTagFilters()
if err := tfs.Add([]byte("day"), []byte("0"), false, false); err != nil {
t.Fatalf("cannot add filter: %s", err)
}
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, baseDate, "", 5, 1e6, noDeadline)
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, tsdbStatusDate, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 1000,
wantStatus = &TSDBStatus{
TotalSeries: metricsPerDay,
TotalLabelValuePairs: 5 * metricsPerDay,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: metricsPerDay},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: metricsPerDay},
{Name: "constant", Count: metricsPerDay},
{Name: "day", Count: metricsPerDay},
{Name: "some_unique_id", Count: metricsPerDay},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: metricsPerDay},
{Name: "constant=const", Count: metricsPerDay},
{Name: "day=0", Count: metricsPerDay},
{Name: "some_unique_id=0", Count: metricsPerDay},
{Name: "UniqueId=1", Count: 1},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
{Name: "day", Count: 1},
{Name: "some_unique_id", Count: 1},
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 1000
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 5000
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
}
// Check GetTSDBStatus, which matches all the series on a global time range
status, err = db.GetTSDBStatus(nil, nil, 0, "day", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 5000,
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 5000
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 25000
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
expectedSeriesCountByFocusLabelValue = []TopHeapEntry{
{
Name: "0",
Count: 1000,
},
{
Name: "1",
Count: 1000,
},
{
Name: "2",
Count: 1000,
},
{
Name: "3",
Count: 1000,
},
{
Name: "4",
Count: 1000,
},
}
if !reflect.DeepEqual(status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) {
t.Fatalf("unexpected SeriesCountByFocusLabelValue;\ngot\n%v\nwant\n%v", status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue)
}
// Check GetTSDBStatus with non-nil filter, which matches only 3 series
// Get TSDB status with non-nil filter that matches only 3 series.
tfs = NewTagFilters()
if err := tfs.Add([]byte("UniqueId"), []byte("0|1|3"), false, true); err != nil {
t.Fatalf("cannot add filter: %s", err)
}
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, baseDate, "", 5, 1e6, noDeadline)
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, tsdbStatusDate, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
if !disablePerDayIndex {
wantStatus = &TSDBStatus{
TotalSeries: 3,
TotalLabelValuePairs: 5 * 3,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: 3},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: 3},
{Name: "__name__", Count: 3},
{Name: "constant", Count: 3},
{Name: "day", Count: 3},
{Name: "some_unique_id", Count: 3},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: 3},
{Name: "constant=const", Count: 3},
{Name: "day=0", Count: 3},
{Name: "some_unique_id=0", Count: 3},
{Name: "UniqueId=1", Count: 1},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: 3},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
{Name: "day", Count: 1},
{Name: "some_unique_id", Count: 1},
},
}
} else {
wantStatus = &TSDBStatus{
TotalSeries: days * 3,
TotalLabelValuePairs: days * 5 * 3,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: days * 3},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: days * 3},
{Name: "__name__", Count: days * 3},
{Name: "constant", Count: days * 3},
{Name: "day", Count: days * 3},
{Name: "some_unique_id", Count: days * 3},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: days * 3},
{Name: "constant=const", Count: days * 3},
{Name: "UniqueId=0", Count: days},
{Name: "UniqueId=1", Count: days},
{Name: "UniqueId=3", Count: days},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "day", Count: days},
{Name: "some_unique_id", Count: days},
{Name: "UniqueId", Count: 3},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
},
}
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 3,
},
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 3
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 15
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
// Check GetTSDBStatus with non-nil filter on global time range, which matches only 15 series
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, 0, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 15,
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 15
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 75
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
s.tb.PutPartition(ptw)
s.MustClose()
fs.MustRemoveDir(path)
}
func toTFPointers(tfs []tagFilter) []*tagFilter {
@@ -1990,7 +1948,19 @@ func TestIndexSearchLegacyContainsTimeRange_Concurrent(t *testing.T) {
}
func TestSearchLabelValues(t *testing.T) {
const path = "TestSearchLabelValues"
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testSearchLabelValues(t, disablePerDayIndex)
})
}
}
func testSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
// Create a bunch of per-day time series
const days = 5
const metricsPerDay = 1000
@@ -2008,28 +1978,20 @@ func TestSearchLabelValues(t *testing.T) {
uniqLabelNames[metricName] = struct{}{}
}
mn.MetricGroup = []byte(metricName)
mn.AddTag(
"constant",
"const",
)
mn.AddTag(
"day",
fmt.Sprintf("%v", day),
)
mn.AddTag(
"UniqueId",
fmt.Sprintf("%v", metric),
)
mn.AddTag(
"some_unique_id",
fmt.Sprintf("%v", day),
)
mn.AddTag("constant", "const")
mn.AddTag("day", fmt.Sprintf("%v", day))
mn.AddTag("UniqueId", fmt.Sprintf("%v", metric))
mn.AddTag("some_unique_id", fmt.Sprintf("%v", day))
mn.sortTags()
return mn
}
s := MustOpenStorage(path, OpenOptions{})
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
ptw := s.tb.MustGetPartition(timestamp)
defer s.tb.PutPartition(ptw)
db := ptw.pt.idb
is := db.getIndexSearch(noDeadline)
@@ -2059,25 +2021,28 @@ func TestSearchLabelValues(t *testing.T) {
is2 := db.getIndexSearch(noDeadline)
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
if !disablePerDayIndex {
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
}
}
}
// Check that all the metrics are found in global index
metricIDs, err := is2.getMetricIDsForDate(0, metricsPerDay*days)
metricIDs, err := is2.getMetricIDsForDate(globalIndexDate, metricsPerDay*days)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !allMetricIDs.Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), allMetricIDs.AppendTo(nil))
}
db.putIndexSearch(is2)
// Check SearchLabelNames with the specified time range.
@@ -2085,6 +2050,9 @@ func TestSearchLabelValues(t *testing.T) {
MinTimestamp: timestamp - msecPerDay,
MaxTimestamp: timestamp,
}
if disablePerDayIndex {
tr = globalIndexTimeRange
}
// Check SearchLabelValues with the specified time range.
lvs, err := db.SearchLabelValues(nil, "", nil, tr, 10000, 1e9, noDeadline)
@@ -2119,10 +2087,6 @@ func TestSearchLabelValues(t *testing.T) {
if !reflect.DeepEqual(got, labelValuesReMatch) {
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", got, labelValuesReMatch)
}
s.tb.PutPartition(ptw)
s.MustClose()
fs.MustRemoveDir(path)
}
func TestFilterLabelValues(t *testing.T) {