Compare commits

..

2 Commits

Author SHA1 Message Date
Vadim Rutkovsky
4a08ac7532 deployment/docker/rules: add RequestErrorsToUnknownPaths alert which focuses on errors without
path set (i.e. invalid authentication)

A new alert would help us separating two cases:
* API errors on known paths, indicating an application issue
* errors where path is not recorded - this indicates authentication problems
2026-07-24 09:57:54 +02:00
Vadim Rutkovsky
911c48fc92 deployment/docker/rules: update RequestErrorsToAPI alert definition 2026-07-24 09:57:48 +02:00
10 changed files with 606 additions and 979 deletions

View File

@@ -18,7 +18,7 @@ groups:
concurrency: 2
rules:
- alert: RequestErrorsToAPI
expr: increase(vm_http_request_errors_total[5m]) > 0
expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
for: 15m
labels:
severity: warning

View File

@@ -164,11 +164,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
left := bfa.left
right := bfa.right
op := bfa.be.Op
isCmpOp := metricsql.IsBinaryOpCmp(op)
if !isCmpOp {
// Do not remove empty series for comparison operations, since NaN can be an
// explicitly present sample value. In particular, `value != NaN` is true.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150.
switch true {
case metricsql.IsBinaryOpCmp(op):
// Do not remove empty series for comparison operations,
// since this may lead to missing result.
default:
left = removeEmptySeries(left)
right = removeEmptySeries(right)
}
@@ -191,14 +191,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
isBool := bfa.be.Bool
fillLeft := bfa.be.FillLeft
fillRight := bfa.be.FillRight
// A NaN produced by a vector comparison denotes a filtered-out sample rather
// than an explicitly present NaN value. Drop it when this result is used as
// the right-hand side of another comparison.
// Note that filtered-out samples surviving as NaN inside other wrappers such as
// transform functions or arithmetic operations aren't detected, since such NaN
// is indistinguishable from an explicitly present NaN value there.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018.
dropNaNRight := isCmpOp && isVectorComparisonExpr(bfa.be.Right)
for i, tsLeft := range left {
leftValues := tsLeft.Values
rightValues := right[i].Values
@@ -211,16 +203,11 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
b := rightValues[j]
leftIsNaN := math.IsNaN(a)
rightIsNaN := math.IsNaN(b)
// Both sides are NaN.
// apply the fill value when either the left or right side is NaN, but not both.
if leftIsNaN && rightIsNaN {
dstValues[j] = bf(a, b, isBool)
continue
}
if dropNaNRight && rightIsNaN && fillRight == nil {
dstValues[j] = nan
continue
}
// Apply the fill value when either the left or right side is NaN, but not both.
if leftIsNaN && fillLeft != nil {
a = fillLeft.N
}
@@ -236,38 +223,6 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
}
}
// isVectorComparisonExpr returns whether e is a comparison operation
// with at least one non-scalar operand.
func isVectorComparisonExpr(e metricsql.Expr) bool {
if re, ok := e.(*metricsql.RollupExpr); ok && re.Window == nil {
// Unwrap `(...) offset <d>` and `(...) @ <t>`, which do not change
// the shape of the result. Subqueries are left as is, since rollup
// functions skip NaN values on their own.
e = re.Expr
}
be, ok := e.(*metricsql.BinaryOpExpr)
if !ok || !metricsql.IsBinaryOpCmp(be.Op) {
return false
}
return !isScalarLikeExpr(be.Left) || !isScalarLikeExpr(be.Right)
}
// isScalarLikeExpr returns whether e always evaluates to a scalar value.
func isScalarLikeExpr(e metricsql.Expr) bool {
switch e := e.(type) {
case *metricsql.NumberExpr, *metricsql.DurationExpr:
return true
case *metricsql.BinaryOpExpr:
return isScalarLikeExpr(e.Left) && isScalarLikeExpr(e.Right)
case *metricsql.FuncExpr:
switch strings.ToLower(e.Name) {
case "now", "pi", "scalar", "start", "end", "step", "time", "timezone_offset":
return true
}
}
return false
}
func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) ([]*timeseries, []*timeseries, []*timeseries, error) {
if len(be.GroupModifier.Op) == 0 && len(be.JoinModifier.Op) == 0 {
if isScalar(left) {

View File

@@ -2994,136 +2994,6 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_nan_left_vector_right_scalar`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != NaN`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_non_nan_scalar_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != 1200`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, nan, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_nan_vector_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != label_set(NaN, "foo", "bar")`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_nan_scalar_comparison_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != (1 > 2)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_right_offset`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != ((label_set(time(), "foo", "bar") > 100000) offset 0s)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_vector_left`, func(t *testing.T) {
// A missing sample on the left side results in NaN for every comparison
// operation, so no special handling is needed there.
t.Parallel()
q := `(label_set(time(), "foo", "bar") > 100000) != label_set(time(), "foo", "bar")`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_bool`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") == bool (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_partially_empty_series_right`, func(t *testing.T) {
// Prometheus drops the timestamps filtered out by the comparison on the right.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11100#issuecomment-4933952390.
t.Parallel()
q := `label_set(time(), "foo", "bar") != (label_set(time(), "foo", "bar") * 2 > 2800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`compare_to_empty_unlabeled_vector_right`, func(t *testing.T) {
t.Parallel()
q := `sum(label_set(time(), "foo", "bar")) != (sum(label_set(time(), "foo", "bar")) > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_with_fill_left`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != fill_left(0) (label_set(time(), "foo", "bar") > 100000)`
resultExpected := []netstorage.Result{}
f(q, resultExpected)
})
t.Run(`compare_to_empty_series_right_with_fill_right`, func(t *testing.T) {
t.Parallel()
q := `label_set(time(), "foo", "bar") != fill_right(0) (label_set(time(), "foo", "bar") > 100000)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
r.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("bar"),
}}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`-1 < 2`, func(t *testing.T) {
t.Parallel()
q := `-1 < 2`

View File

@@ -72,7 +72,7 @@ const Relabel: FC = () => {
<div className="vm-relabeling-header-configs">
<TextField
type="textarea"
label="Config"
label="Relabel configs"
value={config}
autofocus
onChange={handleChangeConfig}
@@ -82,9 +82,8 @@ const Relabel: FC = () => {
<div className="vm-relabeling-header__labels">
<TextField
type="textarea"
label="A time series"
label="Labels"
value={labels}
placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}"
onChange={handleChangeLabels}
onEnter={handleRunQuery}
/>
@@ -98,22 +97,22 @@ const Relabel: FC = () => {
<Button
variant="text"
color="gray"
startIcon={<WikiIcon/>}
startIcon={<InfoIcon/>}
>
Relabeling cookbook
</Button>
</a>
<a
target="_blank"
href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages"
href="https://docs.victoriametrics.com/victoriametrics/relabeling/"
rel="help noreferrer"
>
<Button
variant="text"
color="gray"
startIcon={<InfoIcon/>}
startIcon={<WikiIcon/>}
>
Relabeling Stages
Documentation
</Button>
</a>
<Button

View File

@@ -81,6 +81,7 @@ func AssertSeries(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID st
Status: "success",
Data: want,
},
Retries: 1000,
FailNow: true,
})
}

View File

@@ -1,9 +1,7 @@
package tests
import (
"path/filepath"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
@@ -182,123 +180,3 @@ func TestClusterMaxSeries(t *testing.T) {
}
}
func searchLimitsStartVmsingle(tc *apptest.TestCase, vmstorageFlags, vmselectFlags []string) apptest.PrometheusWriteQuerier {
vmsingleFlags := []string{
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
"-retentionPeriod=100y",
}
vmsingleFlags = append(vmsingleFlags, vmstorageFlags...)
vmsingleFlags = append(vmsingleFlags, vmselectFlags...)
return tc.MustStartVmsingle("vmsingle", vmsingleFlags)
}
func searchLimitsStopVmsingle(tc *apptest.TestCase) {
tc.StopApp("vmsingle")
}
func searchLimitsStartVmcluster(tc *apptest.TestCase, vmstorageFlags, vmselectFlags []string) apptest.PrometheusWriteQuerier {
vmstorageFlags = append(vmstorageFlags, []string{
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmstorage"),
"-retentionPeriod=100y",
}...)
vmstorage := tc.MustStartVmstorage("vmstorage", vmstorageFlags)
vminsert := tc.MustStartVminsert("vminsert", []string{
"-storageNode=" + vmstorage.VminsertAddr(),
})
vmselectFlags = append(vmselectFlags, []string{
"-storageNode=" + vmstorage.VmselectAddr(),
}...)
vmselect := tc.MustStartVmselect("vmselect", vmselectFlags)
return &apptest.Vmcluster{vminsert, vmselect, []*apptest.Vmstorage{vmstorage}}
}
func searchLimitsStopVmcluster(tc *apptest.TestCase) {
tc.StopApp("vminsert")
tc.StopApp("vmselect")
tc.StopApp("vmstorage")
}
type searchLimitsOpts struct {
startSUT func(tc *apptest.TestCase, vmstorageFlags, vmselectFlags []string) apptest.PrometheusWriteQuerier
stopSUT func(tc *apptest.TestCase)
}
func TestSingleSearchLimits_MetricNames(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
testSearchLimits_MetricNames(tc, searchLimitsOpts{
startSUT: searchLimitsStartVmsingle,
stopSUT: searchLimitsStopVmsingle,
})
}
func TestClusterSearchLimits_MetricNames(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
testSearchLimits_MetricNames(tc, searchLimitsOpts{
startSUT: searchLimitsStartVmcluster,
stopSUT: searchLimitsStopVmcluster,
})
}
func testSearchLimits_MetricNames(tc *apptest.TestCase, opts searchLimitsOpts) {
const numMetrics = 100
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
data := apptest.GenerateTestData("metric", numMetrics, start, end)
sut := opts.startSUT(tc, nil, nil)
sut.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
sut.ForceFlush(tc.T())
// Check that with default search limits all metrics can be retrieved.
apptest.AssertSeries(tc, sut, "metric.*", "", start, end, data.WantSeries)
// Restart cluster with explicit maxUniqueTimeseries on vmstorage side.
opts.stopSUT(tc)
sut = opts.startSUT(tc, []string{
"-search.maxUniqueTimeseries=10",
}, nil)
// Check that retrieving number of metrics that matches the
// maxUniqueTimeseries limit is ok but retrieving more than that causes an
// error.
apptest.AssertSeries(tc, sut, "metric_000[0-9]{1}", "", start, end, data.WantSeries[0:10])
apptest.AssertSeries(tc, sut, "metric.*", "", start, end, data.WantSeries) // Should fail but does not.
}
func _TestSingleSearchLimits(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
const numMetrics = 100
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
data := apptest.GenerateTestData("metric", numMetrics, start, end)
vmsingle := tc.MustStartVmsingle("vmsingle", []string{
"-storageDataPath=" + tc.Dir() + "/vmsingle",
"-retentionPeriod=100y",
"-search.maxUniqueTimeseries=10",
"-search.maxTagKeys=10",
"-search.maxTagValues=10",
"-search.maxTagValueSuffixesPerSearch=10",
"-search.maxFederateSeries=10",
"-search.maxExportSeries=10",
"-search.maxTSDBStatusSeries=10",
"-search.maxSeries=10",
"-search.maxDeleteSeries=10",
"-search.maxLabelsAPISeries=10",
})
vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
vmsingle.ForceFlush(t)
apptest.AssertSeries(tc, vmsingle, "metric_00[0-9]{2}", "", start, end, data.WantSeries)
return
apptest.AssertSeriesCount(tc, vmsingle, "", start, end, numMetrics)
apptest.AssertLabels(tc, vmsingle, "metric.*", "", start, end, data.WantLabels)
apptest.AssertLabelValues(tc, vmsingle, "metric.*", "label", "", start, end, data.WantLabelValues)
apptest.AssertQueryResults(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantQueryResults)
apptest.AssertMetadata(tc, vmsingle, "", "", data.WantMetadata)
}

View File

@@ -75,7 +75,7 @@ groups:
Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
- alert: RequestErrorsToAPI
expr: increase(vm_http_request_errors_total[5m]) > 0
expr: increase(vm_http_request_errors_total{path=~".+", path!="*"}[5m]) > 0
for: 15m
labels:
severity: warning
@@ -86,6 +86,18 @@ groups:
description: "Requests to path {{ $labels.path }} are receiving errors.
Please verify if clients are sending correct requests."
- alert: RequestErrorsToUnknownPaths
expr: sum(increase(vm_http_request_errors_total{path="*"}[5m])) by(job, instance, reason) > 0
for: 15m
labels:
severity: warning
show_at: dashboard
annotations:
dashboard: "{{ $externalURL }}/d/oS7Bi_0Wz?viewPanel=52&var-instance={{ $labels.instance }}"
summary: "Too many errors served for {{ $labels.job }} with reason {{ $labels.reason }} (instance {{ $labels.instance }})"
description: "Requests are failing with reason {{ $labels.reason }}.
Please verify if clients are sending correct requests."
- alert: RPCErrors
expr: |
(

View File

@@ -75,7 +75,7 @@ groups:
Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
- alert: RequestErrorsToAPI
expr: increase(vm_http_request_errors_total[5m]) > 0
expr: increase(vm_http_request_errors_total{path=~".+"}[5m]) > 0
for: 15m
labels:
severity: warning

View File

@@ -34,7 +34,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
@@ -123,7 +122,6 @@ Released at 2026-06-22
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
Released at 2026-06-08

File diff suppressed because it is too large Load Diff