Compare commits

..

2 Commits

Author SHA1 Message Date
Vadim Rutkovsky
4a08ac7532 deployment/docker/rules: add RequestErrorsToUnknownPaths alert which focuses on errors without
path set (i.e. invalid authentication)

A new alert would help us separating two cases:
* API errors on known paths, indicating an application issue
* errors where path is not recorded - this indicates authentication problems
2026-07-24 09:57:54 +02:00
Vadim Rutkovsky
911c48fc92 deployment/docker/rules: update RequestErrorsToAPI alert definition 2026-07-24 09:57:48 +02:00
7 changed files with 324 additions and 453 deletions

View File

@@ -18,7 +18,7 @@ groups:
concurrency: 2
rules:
- alert: RequestErrorsToAPI
expr: increase(vm_http_request_errors_total[5m]) > 0
expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
for: 15m
labels:
severity: warning

View File

@@ -164,11 +164,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
left := bfa.left
right := bfa.right
op := bfa.be.Op
isCmpOp := metricsql.IsBinaryOpCmp(op)
if !isCmpOp {
// Do not remove empty series for comparison operations, since NaN can be an
// explicitly present sample value. In particular, `value != NaN` is true.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
switch true {
case metricsql.IsBinaryOpCmp(op):
// Do not remove empty series for comparison operations,
// since this may lead to missing result.
default:
left = removeEmptySeries(left)
right = removeEmptySeries(right)
}
@@ -191,14 +191,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
isBool := bfa.be.Bool
fillLeft := bfa.be.FillLeft
fillRight := bfa.be.FillRight
// A NaN produced by a vector comparison denotes a filtered-out sample rather
// than an explicitly present NaN value. Drop it when this result is used as
// the right-hand side of another comparison.
// Note that filtered-out samples surviving as NaN inside other wrappers such as
// transform functions or arithmetic operations aren't detected, since such NaN
// is indistinguishable from an explicitly present NaN value there.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
for i, tsLeft := range left {
leftValues := tsLeft.Values
rightValues := right[i].Values
@@ -211,16 +203,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
b := rightValues[j]
leftIsNaN := math.IsNaN(a)
rightIsNaN := math.IsNaN(b)
// Both sides are NaN.
// apply the fill value when either the left or right side is NaN, but not both.
if leftIsNaN && rightIsNaN {
dstValues[j] = bf(a, b, isBool)
continue
}
if dropNaNRight && rightIsNaN && fillRight == nil {
dstValues[j] = nan
continue
}
// Apply the fill value when either the left or right side is NaN, but not both.
if leftIsNaN && fillLeft != nil {
a = fillLeft.N
}
@@ -236,38 +223,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
}
}
// isVectorComparisonExpr returns whether e is a comparison operation
// with at least one non-scalar operand.
func isVectorComparisonExpr(e metricsql.Expr) bool {
if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
// the shape of the result. Subqueries are left as is, since rollup
// functions skip NaN values on their own.
e = re.Expr
}
be, ok := e.(*metricsql.BinaryOpExpr)
if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
return false
}
return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
}
// isScalarLikeExpr returns whether e always evaluates to a scalar value.
func isScalarLikeExpr(e metricsql.Expr) bool {
switch e := e.(type) {
case *metricsql.NumberExpr, *metricsql.DurationExpr:
return true
case *metricsql.BinaryOpExpr:
return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
case *metricsql.FuncExpr:
switch strings.ToLower(e.Name) {
case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
return true
}
}
return false
}
func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
if isScalar(left) {

View File

@@ -2994,136 +2994,6 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != NaN`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != 1200`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, nan, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != (1 > 2)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
// A missing sample on the left side results in NaN for every comparison
// operation, so no special handling is needed there.
t.Parallel()
q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
// Prometheus drops the timestamps filtered out by the comparison on the right.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
t.Parallel()
q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
t.Parallel()
q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`-1 < 2`, func(t *testing.T) {
t.Parallel()
q := `-1 < 2`

View File

@@ -75,7 +75,7 @@ groups:
Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
- alert: RequestErrorsToAPI
expr: increase(vm_http_request_errors_total[5m]) > 0
expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
for: 15m
labels:
severity: warning
@@ -86,6 +86,18 @@ groups:
description: "Requests to path {{ $labels.path }} are receiving errors.
Please verify if clients are sending correct requests."
- alert: RequestErrorsToUnknownPaths
expr: sum(increase(vm_http_request_errors_total{path="*"}[5m])) by(job, instance, reason) > 0
for: 15m
labels:
severity: warning
show_at: dashboard
annotations:
dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
description: "Requests are failing with reason {{ $labels.reason }}.
Please verify if clients are sending correct requests."
- alert: RPCErrors
expr: |
(

View File

@@ -75,7 +75,7 @@ groups:
Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
- alert: RequestErrorsToAPI
expr: increase(vm_http_request_errors_total[5m]) > 0
expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
for: 15m
labels:
severity: warning

View File

@@ -34,7 +34,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
@@ -123,7 +122,6 @@ Released at 2026-06-22
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
Released at 2026-06-08

View File

@@ -1400,22 +1400,10 @@ func TestMatchTagFilters(t *testing.T) {
}
func TestSearchTSIDWithTimeRange(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testSearchTSIDWithTimeRange(t, disablePerDayIndex)
})
}
}
func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
// TODO: @f41gh7 refactor this test:
// create a new test for LabelNames
// move exist LabelVales tests into TestSearchLabelValues
const path = "TestSearchTSIDWithTimeRange"
// Create a bunch of per-day time series
const days = 5
const metricsPerDay = 1000
@@ -1435,20 +1423,28 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
newMN := func(name string, day, metric int) MetricName {
var mn MetricName
mn.MetricGroup = []byte(name)
mn.AddTag("constant", "const")
mn.AddTag("day", fmt.Sprintf("%v", day))
mn.AddTag("UniqueId", fmt.Sprintf("%v", metric))
mn.AddTag("some_unique_id", fmt.Sprintf("%v", day))
mn.AddTag(
"constant",
"const",
)
mn.AddTag(
"day",
fmt.Sprintf("%v", day),
)
mn.AddTag(
"UniqueId",
fmt.Sprintf("%v", metric),
)
mn.AddTag(
"some_unique_id",
fmt.Sprintf("%v", day),
)
mn.sortTags()
return mn
}
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
s := MustOpenStorage(path, OpenOptions{})
ptw := s.tb.MustGetPartition(timestamp)
defer s.tb.PutPartition(ptw)
db := ptw.pt.idb
is := db.getIndexSearch(noDeadline)
@@ -1476,28 +1472,25 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
is2 := db.getIndexSearch(noDeadline)
if !disablePerDayIndex {
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
}
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
}
}
// Check that all the metrics are found in global index
metricIDs, err := is2.getMetricIDsForDate(globalIndexDate, metricsPerDay*days)
metricIDs, err := is2.getMetricIDsForDate(0, metricsPerDay*days)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !allMetricIDs.Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), allMetricIDs.AppendTo(nil))
}
db.putIndexSearch(is2)
// add a metric that will be deleted shortly
@@ -1505,7 +1498,10 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
day := days
date := baseDate - uint64(day)
mn := newMN("deletedMetric", day, 999)
mn.AddTag("labelToDelete", fmt.Sprintf("%v", day))
mn.AddTag(
"labelToDelete",
fmt.Sprintf("%v", day),
)
mn.sortTags()
metricNameBuf = mn.Marshal(metricNameBuf[:0])
var tsid TSID
@@ -1525,9 +1521,6 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
MinTimestamp: timestamp - msecPerDay,
MaxTimestamp: timestamp,
}
if disablePerDayIndex {
tr = globalIndexTimeRange
}
lns, err := db.SearchLabelNames(nil, nil, tr, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelNames(timeRange=%s): %s", &tr, err)
@@ -1570,17 +1563,22 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
MinTimestamp: timestamp - 2*msecPerHour - 1,
MaxTimestamp: timestamp,
}
wantNumMatchedTSIDs := metricsPerDay
if disablePerDayIndex {
tr = globalIndexTimeRange
wantNumMatchedTSIDs = days * metricsPerDay
}
matchedTSIDs, err := db.SearchTSIDs(nil, []*TagFilters{tfs}, tr, 1e5, noDeadline)
if err != nil {
t.Fatalf("error searching tsids: %v", err)
}
if len(matchedTSIDs) != wantNumMatchedTSIDs {
t.Fatalf("expected %d time series for current day, got %d time series", wantNumMatchedTSIDs, len(matchedTSIDs))
if len(matchedTSIDs) != metricsPerDay {
t.Fatalf("expected %d time series for current day, got %d time series", metricsPerDay, len(matchedTSIDs))
}
// Check SearchLabelNames with the specified filter.
lns, err = db.SearchLabelNames(nil, []*TagFilters{tfs}, TimeRange{}, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelNames(filters=%s): %s", tfs, err)
}
got = sortedSlice(lns)
if !reflect.DeepEqual(got, labelNames) {
t.Fatalf("unexpected labelNames; got\n%s\nwant\n%s", got, labelNames)
}
// Check SearchLabelNames with the specified filter and time range.
@@ -1613,6 +1611,16 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
t.Fatalf("unexpected labelNames; got\n%s\nwant\n%s", got, labelNames)
}
// Check SearchLabelValues with the specified filter.
lvs, err = db.SearchLabelValues(nil, "", []*TagFilters{tfs}, TimeRange{}, 10000, 1e9, noDeadline)
if err != nil {
t.Fatalf("unexpected error in SearchLabelValues(filters=%s): %s", tfs, err)
}
got = sortedSlice(lvs)
if !reflect.DeepEqual(got, labelValues) {
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", got, labelValues)
}
// Check SearchLabelValues with the specified filter and time range.
lvs, err = db.SearchLabelValues(nil, "", []*TagFilters{tfs}, tr, 10000, 1e9, noDeadline)
if err != nil {
@@ -1649,9 +1657,6 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
MinTimestamp: timestamp - msecPerDay*days,
MaxTimestamp: timestamp,
}
if disablePerDayIndex {
tr = globalIndexTimeRange
}
matchedTSIDs, err = db.SearchTSIDs(nil, []*TagFilters{tfs}, tr, 1e5, noDeadline)
if err != nil {
@@ -1661,219 +1666,256 @@ func testSearchTSIDWithTimeRange(t *testing.T, disablePerDayIndex bool) {
t.Fatalf("expected %d time series for all days, got %d time series", metricsPerDay*days, len(matchedTSIDs))
}
// Get TSDB status with nil filters.
tsdbStatusDate := baseDate
if disablePerDayIndex {
tsdbStatusDate = globalIndexDate
}
status, err := db.GetTSDBStatus(nil, nil, tsdbStatusDate, "day", 5, 1e6, noDeadline)
// Check GetTSDBStatus with nil filters.
status, err := db.GetTSDBStatus(nil, nil, baseDate, "day", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus with nil filters: %s", err)
}
wantStatus := &TSDBStatus{
TotalSeries: metricsPerDay,
TotalLabelValuePairs: 5 * metricsPerDay,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: metricsPerDay},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: metricsPerDay},
{Name: "constant", Count: metricsPerDay},
{Name: "day", Count: metricsPerDay},
{Name: "some_unique_id", Count: metricsPerDay},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{
{Name: "0", Count: metricsPerDay},
},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: metricsPerDay},
{Name: "constant=const", Count: metricsPerDay},
{Name: "day=0", Count: metricsPerDay},
{Name: "some_unique_id=0", Count: metricsPerDay},
{Name: "UniqueId=1", Count: 1},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
{Name: "day", Count: 1},
{Name: "some_unique_id", Count: 1},
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName := []TopHeapEntry{
{
Name: "testMetric",
Count: 1000,
},
}
if disablePerDayIndex {
wantStatus = &TSDBStatus{
TotalSeries: days * metricsPerDay,
TotalLabelValuePairs: days * 5 * metricsPerDay,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: days * metricsPerDay},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: days * metricsPerDay},
{Name: "__name__", Count: days * metricsPerDay},
{Name: "constant", Count: days * metricsPerDay},
{Name: "day", Count: days * metricsPerDay},
{Name: "some_unique_id", Count: days * metricsPerDay},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{
{Name: "0", Count: metricsPerDay},
{Name: "1", Count: metricsPerDay},
{Name: "2", Count: metricsPerDay},
{Name: "3", Count: metricsPerDay},
{Name: "4", Count: metricsPerDay},
},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: days * metricsPerDay},
{Name: "constant=const", Count: days * metricsPerDay},
{Name: "day=0", Count: metricsPerDay},
{Name: "day=1", Count: metricsPerDay},
{Name: "day=2", Count: metricsPerDay},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "day", Count: days},
{Name: "some_unique_id", Count: days},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
expectedSeriesCountByLabelName := []TopHeapEntry{
{
Name: "UniqueId",
Count: 1000,
},
{
Name: "__name__",
Count: 1000,
},
{
Name: "constant",
Count: 1000,
},
{
Name: "day",
Count: 1000,
},
{
Name: "some_unique_id",
Count: 1000,
},
}
// Get TSDB status with non-nil filter which matches all series.
tfs = NewTagFilters()
if err := tfs.Add([]byte("constant"), []byte("const"), false, false); err != nil {
t.Fatalf("cannot add filter: %s", err)
if !reflect.DeepEqual(status.SeriesCountByLabelName, expectedSeriesCountByLabelName) {
t.Fatalf("unexpected SeriesCountByLabelName;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelName, expectedSeriesCountByLabelName)
}
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, tsdbStatusDate, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
expectedSeriesCountByFocusLabelValue := []TopHeapEntry{
{
Name: "0",
Count: 1000,
},
}
// The result must be the same as with no filters (see above) except that
// the SeriesCountByFocusLabelValue must be empty.
wantStatus.SeriesCountByFocusLabelValue = []TopHeapEntry{}
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
if !reflect.DeepEqual(status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) {
t.Fatalf("unexpected SeriesCountByFocusLabelValue;\ngot\n%v\nwant\n%v", status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue)
}
expectedLabelValueCountByLabelName := []TopHeapEntry{
{
Name: "UniqueId",
Count: 1000,
},
{
Name: "__name__",
Count: 1,
},
{
Name: "constant",
Count: 1,
},
{
Name: "day",
Count: 1,
},
{
Name: "some_unique_id",
Count: 1,
},
}
if !reflect.DeepEqual(status.LabelValueCountByLabelName, expectedLabelValueCountByLabelName) {
t.Fatalf("unexpected LabelValueCountByLabelName;\ngot\n%v\nwant\n%v", status.LabelValueCountByLabelName, expectedLabelValueCountByLabelName)
}
expectedSeriesCountByLabelValuePair := []TopHeapEntry{
{
Name: "__name__=testMetric",
Count: 1000,
},
{
Name: "constant=const",
Count: 1000,
},
{
Name: "day=0",
Count: 1000,
},
{
Name: "some_unique_id=0",
Count: 1000,
},
{
Name: "UniqueId=1",
Count: 1,
},
}
if !reflect.DeepEqual(status.SeriesCountByLabelValuePair, expectedSeriesCountByLabelValuePair) {
t.Fatalf("unexpected SeriesCountByLabelValuePair;\ngot\n%v\nwant\n%v", status.SeriesCountByLabelValuePair, expectedSeriesCountByLabelValuePair)
}
expectedTotalSeries := uint64(1000)
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs := uint64(5000)
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
// Get TSDB status with non-nil filter that matches all the series on a
// given day.
// Check GetTSDBStatus with non-nil filter, which matches all the series
tfs = NewTagFilters()
if err := tfs.Add([]byte("day"), []byte("0"), false, false); err != nil {
t.Fatalf("cannot add filter: %s", err)
}
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, tsdbStatusDate, "", 5, 1e6, noDeadline)
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, baseDate, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
wantStatus = &TSDBStatus{
TotalSeries: metricsPerDay,
TotalLabelValuePairs: 5 * metricsPerDay,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: metricsPerDay},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: metricsPerDay},
{Name: "constant", Count: metricsPerDay},
{Name: "day", Count: metricsPerDay},
{Name: "some_unique_id", Count: metricsPerDay},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: metricsPerDay},
{Name: "constant=const", Count: metricsPerDay},
{Name: "day=0", Count: metricsPerDay},
{Name: "some_unique_id=0", Count: metricsPerDay},
{Name: "UniqueId=1", Count: 1},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: metricsPerDay},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
{Name: "day", Count: 1},
{Name: "some_unique_id", Count: 1},
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 1000,
},
}
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 1000
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 5000
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
// Get TSDB status with non-nil filter that matches only 3 series.
// Check GetTSDBStatus, which matches all the series on a global time range
status, err = db.GetTSDBStatus(nil, nil, 0, "day", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 5000,
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 5000
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 25000
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
expectedSeriesCountByFocusLabelValue = []TopHeapEntry{
{
Name: "0",
Count: 1000,
},
{
Name: "1",
Count: 1000,
},
{
Name: "2",
Count: 1000,
},
{
Name: "3",
Count: 1000,
},
{
Name: "4",
Count: 1000,
},
}
if !reflect.DeepEqual(status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue) {
t.Fatalf("unexpected SeriesCountByFocusLabelValue;\ngot\n%v\nwant\n%v", status.SeriesCountByFocusLabelValue, expectedSeriesCountByFocusLabelValue)
}
// Check GetTSDBStatus with non-nil filter, which matches only 3 series
tfs = NewTagFilters()
if err := tfs.Add([]byte("UniqueId"), []byte("0|1|3"), false, true); err != nil {
t.Fatalf("cannot add filter: %s", err)
}
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, tsdbStatusDate, "", 5, 1e6, noDeadline)
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, baseDate, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !disablePerDayIndex {
wantStatus = &TSDBStatus{
TotalSeries: 3,
TotalLabelValuePairs: 5 * 3,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: 3},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: 3},
{Name: "__name__", Count: 3},
{Name: "constant", Count: 3},
{Name: "day", Count: 3},
{Name: "some_unique_id", Count: 3},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: 3},
{Name: "constant=const", Count: 3},
{Name: "day=0", Count: 3},
{Name: "some_unique_id=0", Count: 3},
{Name: "UniqueId=1", Count: 1},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: 3},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
{Name: "day", Count: 1},
{Name: "some_unique_id", Count: 1},
},
}
} else {
wantStatus = &TSDBStatus{
TotalSeries: days * 3,
TotalLabelValuePairs: days * 5 * 3,
SeriesCountByMetricName: []TopHeapEntry{
{Name: "testMetric", Count: days * 3},
},
SeriesCountByLabelName: []TopHeapEntry{
{Name: "UniqueId", Count: days * 3},
{Name: "__name__", Count: days * 3},
{Name: "constant", Count: days * 3},
{Name: "day", Count: days * 3},
{Name: "some_unique_id", Count: days * 3},
},
SeriesCountByFocusLabelValue: []TopHeapEntry{},
SeriesCountByLabelValuePair: []TopHeapEntry{
{Name: "__name__=testMetric", Count: days * 3},
{Name: "constant=const", Count: days * 3},
{Name: "UniqueId=0", Count: days},
{Name: "UniqueId=1", Count: days},
{Name: "UniqueId=3", Count: days},
},
LabelValueCountByLabelName: []TopHeapEntry{
{Name: "day", Count: days},
{Name: "some_unique_id", Count: days},
{Name: "UniqueId", Count: 3},
{Name: "__name__", Count: 1},
{Name: "constant", Count: 1},
},
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
if diff := cmp.Diff(wantStatus, status); diff != "" {
t.Fatalf("unexpected TSDBStatus (-want, +got):\n%s", diff)
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 3,
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 3
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 15
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
// Check GetTSDBStatus with non-nil filter on global time range, which matches only 15 series
status, err = db.GetTSDBStatus(nil, []*TagFilters{tfs}, 0, "", 5, 1e6, noDeadline)
if err != nil {
t.Fatalf("error in GetTSDBStatus: %s", err)
}
if !status.hasEntries() {
t.Fatalf("expecting non-empty TSDB status")
}
expectedSeriesCountByMetricName = []TopHeapEntry{
{
Name: "testMetric",
Count: 15,
},
}
if !reflect.DeepEqual(status.SeriesCountByMetricName, expectedSeriesCountByMetricName) {
t.Fatalf("unexpected SeriesCountByMetricName;\ngot\n%v\nwant\n%v", status.SeriesCountByMetricName, expectedSeriesCountByMetricName)
}
expectedTotalSeries = 15
if status.TotalSeries != expectedTotalSeries {
t.Fatalf("unexpected TotalSeries; got %d; want %d", status.TotalSeries, expectedTotalSeries)
}
expectedLabelValuePairs = 75
if status.TotalLabelValuePairs != expectedLabelValuePairs {
t.Fatalf("unexpected TotalLabelValuePairs; got %d; want %d", status.TotalLabelValuePairs, expectedLabelValuePairs)
}
s.tb.PutPartition(ptw)
s.MustClose()
fs.MustRemoveDir(path)
}
func toTFPointers(tfs []tagFilter) []*tagFilter {
@@ -1948,19 +1990,7 @@ func TestIndexSearchLegacyContainsTimeRange_Concurrent(t *testing.T) {
}
func TestSearchLabelValues(t *testing.T) {
defer testRemoveAll(t)
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("disablePerDayIndex=%t", disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testSearchLabelValues(t, disablePerDayIndex)
})
}
}
func testSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
const path = "TestSearchLabelValues"
// Create a bunch of per-day time series
const days = 5
const metricsPerDay = 1000
@@ -1978,20 +2008,28 @@ func testSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
uniqLabelNames[metricName] = struct{}{}
}
mn.MetricGroup = []byte(metricName)
mn.AddTag("constant", "const")
mn.AddTag("day", fmt.Sprintf("%v", day))
mn.AddTag("UniqueId", fmt.Sprintf("%v", metric))
mn.AddTag("some_unique_id", fmt.Sprintf("%v", day))
mn.AddTag(
"constant",
"const",
)
mn.AddTag(
"day",
fmt.Sprintf("%v", day),
)
mn.AddTag(
"UniqueId",
fmt.Sprintf("%v", metric),
)
mn.AddTag(
"some_unique_id",
fmt.Sprintf("%v", day),
)
mn.sortTags()
return mn
}
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
s := MustOpenStorage(path, OpenOptions{})
ptw := s.tb.MustGetPartition(timestamp)
defer s.tb.PutPartition(ptw)
db := ptw.pt.idb
is := db.getIndexSearch(noDeadline)
@@ -2021,28 +2059,25 @@ func testSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
is2 := db.getIndexSearch(noDeadline)
if !disablePerDayIndex {
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
}
// Check that all the metrics are found for all the days.
for date := baseDate - days + 1; date <= baseDate; date++ {
metricIDs, err := is2.getMetricIDsForDate(date, metricsPerDay)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !perDayMetricIDs[date].Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), perDayMetricIDs[date].AppendTo(nil))
}
}
// Check that all the metrics are found in global index
metricIDs, err := is2.getMetricIDsForDate(globalIndexDate, metricsPerDay*days)
metricIDs, err := is2.getMetricIDsForDate(0, metricsPerDay*days)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !allMetricIDs.Equal(metricIDs) {
t.Fatalf("unexpected metricIDs found;\ngot\n%d\nwant\n%d", metricIDs.AppendTo(nil), allMetricIDs.AppendTo(nil))
}
db.putIndexSearch(is2)
// Check SearchLabelNames with the specified time range.
@@ -2050,9 +2085,6 @@ func testSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
MinTimestamp: timestamp - msecPerDay,
MaxTimestamp: timestamp,
}
if disablePerDayIndex {
tr = globalIndexTimeRange
}
// Check SearchLabelValues with the specified time range.
lvs, err := db.SearchLabelValues(nil, "", nil, tr, 10000, 1e9, noDeadline)
@@ -2087,6 +2119,10 @@ func testSearchLabelValues(t *testing.T, disablePerDayIndex bool) {
if !reflect.DeepEqual(got, labelValuesReMatch) {
t.Fatalf("unexpected labelValues; got\n%s\nwant\n%s", got, labelValuesReMatch)
}
s.tb.PutPartition(ptw)
s.MustClose()
fs.MustRemoveDir(path)
}
func TestFilterLabelValues(t *testing.T) {