mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-05-17 08:36:55 +03:00
feature: [or performance] optimize the performance of the or operation by reducing the marshaling call from O(m*n) to O(m+n) when large volumes of series are involved
This commit is contained in:
@@ -566,15 +566,24 @@ func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
|
||||
return
|
||||
}
|
||||
|
||||
nameLeft, nameRight := bbPool.Get(), bbPool.Get()
|
||||
// to avoid marshaling the same metric name multiple times, use a slice to store the results and an index to access them.
|
||||
// todo: this may allocate a lot of []byte and could be further optimized.
|
||||
nameRightSlice := make([][]byte, len(tssRight))
|
||||
getRightName := func(idx int) []byte {
|
||||
if nameRightSlice[idx] == nil {
|
||||
nameRightSlice[idx] = marshalMetricNameSorted(nameRightSlice[idx], &tssRight[idx].MetricName)
|
||||
}
|
||||
return nameRightSlice[idx]
|
||||
}
|
||||
|
||||
nameLeft := bbPool.Get()
|
||||
for _, tsLeft := range tssLeft {
|
||||
valuesLeft := tsLeft.Values
|
||||
nameLeft.B = marshalMetricNameSorted(nameLeft.B[:0], &tsLeft.MetricName)
|
||||
for i, v := range valuesLeft {
|
||||
leftIsNaN := math.IsNaN(v)
|
||||
for _, tsRight := range tssRight {
|
||||
nameRight.B = marshalMetricNameSorted(nameRight.B[:0], &tsRight.MetricName)
|
||||
canBeMerged := bytes.Equal(nameLeft.B, nameRight.B)
|
||||
for rIdx, tsRight := range tssRight {
|
||||
canBeMerged := bytes.Equal(nameLeft.B, getRightName(rIdx))
|
||||
valueRight := tsRight.Values[i]
|
||||
if leftIsNaN && canBeMerged {
|
||||
// fill NaNs with valueRight if labels match
|
||||
@@ -589,7 +598,6 @@ func fillLeftNaNsWithRightValuesOrMerge(tssLeft, tssRight []*timeseries) {
|
||||
}
|
||||
}
|
||||
bbPool.Put(nameLeft)
|
||||
bbPool.Put(nameRight)
|
||||
}
|
||||
|
||||
func binaryOpIfnot(bfa *binaryOpFuncArg) ([]*timeseries, error) {
|
||||
|
||||
@@ -92,6 +92,22 @@ func BenchmarkBinaryOpOr(b *testing.B) {
|
||||
}
|
||||
benchmarkBinaryOpOr(b, bfa)
|
||||
})
|
||||
|
||||
b.Run("tss:1000 or tss:40000: new", func(b *testing.B) {
|
||||
left, right := make([]*timeseries, 1000), make([]*timeseries, 40000)
|
||||
for i := range left {
|
||||
left[i] = ts(fmt.Sprintf(`a{foo="%d"}`, i))
|
||||
}
|
||||
for i := range right {
|
||||
right[i] = ts(fmt.Sprintf(`b{foo="%d"}`, i))
|
||||
}
|
||||
bfa := &binaryOpFuncArg{
|
||||
be: mustParseMetricsQL("a or b"),
|
||||
left: left,
|
||||
right: right,
|
||||
}
|
||||
benchmarkBinaryOpOr(b, bfa)
|
||||
})
|
||||
}
|
||||
|
||||
func benchmarkBinaryOpOr(b *testing.B, bfa *binaryOpFuncArg) {
|
||||
|
||||
@@ -26,6 +26,8 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): optimize the performance of the `or` operation by reducing the marshaling call from `O(m*n)` to `O(m+n)` when large volumes of series are involved. See [#10374](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10374).
|
||||
|
||||
## [v1.135.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.135.0)
|
||||
|
||||
Released at 2026-01-30
|
||||
|
||||
Reference in New Issue
Block a user