added compressor cleanup

app/vmalert-tool/unittest/recording.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/metricsql"
 )
@@ -49,7 +48,7 @@ Outer:
 		}
 		var expSamples []parsedSample
 		for _, s := range mt.ExpSamples {
-			expLb := []prompbmarshal.Label{}
+			expLb := datasource.Labels{}
 			if s.Labels != "" {
 				metricsqlExpr, err := metricsql.Parse(s.Labels)
 				if err != nil {
@@ -65,7 +64,7 @@ Outer:
 				}
 				if len(metricsqlMetricExpr.LabelFilterss) > 0 {
 					for _, l := range metricsqlMetricExpr.LabelFilterss[0] {
-						expLb = append(expLb, prompbmarshal.Label{
+						expLb = append(expLb, datasource.Label{
 							Name:  l.Label,
 							Value: l.Value,
 						})

app/vmalert/datasource/client_graphite.go

@@ -46,8 +46,8 @@ const (
 	graphitePrefix = "/graphite"
 )
 
-func (c *Client) setGraphiteReqParams(r *http.Request, query string) {
-	if c.appendTypePrefix {
+func (s *Client) setGraphiteReqParams(r *http.Request, query string) {
+	if s.appendTypePrefix {
 		r.URL.Path += graphitePrefix
 	}
 	r.URL.Path += graphitePath
@@ -58,7 +58,7 @@ func (c *Client) setGraphiteReqParams(r *http.Request, query string) {
 	q.Set("target", query)
 	q.Set("until", "now")
 
-	for k, vs := range c.extraParams {
+	for k, vs := range s.extraParams {
 		if q.Has(k) { // extraParams are prior to params in URL
 			q.Del(k)
 		}

app/vmalert/datasource/client_prom.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 
 	"github.com/valyala/fastjson"
 )
@@ -82,14 +81,14 @@ func (pi *promInstant) Unmarshal(b []byte) error {
 		labels := metric.GetObject()
 
 		r := &pi.ms[i]
-		r.Labels = make([]prompbmarshal.Label, 0, labels.Len())
+		r.Labels = make([]Label, 0, labels.Len())
 		labels.Visit(func(key []byte, v *fastjson.Value) {
 			lv, errLocal := v.StringBytes()
 			if errLocal != nil {
 				err = fmt.Errorf("error when parsing label value %q: %s", v, errLocal)
 				return
 			}
-			r.Labels = append(r.Labels, prompbmarshal.Label{
+			r.Labels = append(r.Labels, Label{
 				Name:  string(key),
 				Value: string(lv),
 			})
@@ -219,8 +218,8 @@ func parsePrometheusResponse(req *http.Request, resp *http.Response) (res Result
 	return res, nil
 }
 
-func (c *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
-	if c.appendTypePrefix {
+func (s *Client) setPrometheusInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+	if s.appendTypePrefix {
 		r.URL.Path += "/prometheus"
 	}
 	if !*disablePathAppend {
@@ -228,22 +227,22 @@ func (c *Client) setPrometheusInstantReqParams(r *http.Request, query string, ti
 	}
 	q := r.URL.Query()
 	q.Set("time", timestamp.Format(time.RFC3339))
-	if !*disableStepParam && c.evaluationInterval > 0 { // set step as evaluationInterval by default
+	if !*disableStepParam && s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
-	if !*disableStepParam && c.queryStep > 0 { // override step with user-specified value
+	if !*disableStepParam && s.queryStep > 0 { // override step with user-specified value
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(c.queryStep.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(s.queryStep.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	c.setReqParams(r, query)
+	s.setReqParams(r, query)
 }
 
-func (c *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
-	if c.appendTypePrefix {
+func (s *Client) setPrometheusRangeReqParams(r *http.Request, query string, start, end time.Time) {
+	if s.appendTypePrefix {
 		r.URL.Path += "/prometheus"
 	}
 	if !*disablePathAppend {
@@ -252,11 +251,11 @@ func (c *Client) setPrometheusRangeReqParams(r *http.Request, query string, star
 	q := r.URL.Query()
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
-	if c.evaluationInterval > 0 { // set step as evaluationInterval by default
+	if s.evaluationInterval > 0 { // set step as evaluationInterval by default
 		// always convert to seconds to keep compatibility with older
 		// Prometheus versions. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1943
-		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	c.setReqParams(r, query)
+	s.setReqParams(r, query)
 }

app/vmalert/datasource/client_test.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 var (
@@ -145,12 +144,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected := []Metric{
 		{
-			Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
+			Labels:     []Label{{Value: "vm_rows", Name: "__name__"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []prompbmarshal.Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
+			Labels:     []Label{{Value: "vm_requests", Name: "__name__"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -215,7 +214,7 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	exp := []Metric{
 		{
-			Labels:     []prompbmarshal.Label{{Value: "constantLine(10)", Name: "name"}},
+			Labels:     []Label{{Value: "constantLine(10)", Name: "name"}},
 			Timestamps: []int64{1611758403},
 			Values:     []float64{10},
 		},
@@ -237,12 +236,12 @@ func TestVMInstantQuery(t *testing.T) {
 	}
 	expected = []Metric{
 		{
-			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
+			Labels:     []Label{{Value: "total", Name: "stats_result"}, {Value: "bar", Name: "foo"}},
 			Timestamps: []int64{1583786142},
 			Values:     []float64{13763},
 		},
 		{
-			Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
+			Labels:     []Label{{Value: "total", Name: "stats_result"}, {Value: "baz", Name: "foo"}},
 			Timestamps: []int64{1583786140},
 			Values:     []float64{2000},
 		},
@@ -445,7 +444,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected := Metric{
-		Labels:     []prompbmarshal.Label{{Value: "vm_rows", Name: "__name__"}},
+		Labels:     []Label{{Value: "vm_rows", Name: "__name__"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{13763},
 	}
@@ -476,7 +475,7 @@ func TestVMRangeQuery(t *testing.T) {
 		t.Fatalf("expected 1 metric  got %d in %+v", len(m), m)
 	}
 	expected = Metric{
-		Labels:     []prompbmarshal.Label{{Value: "total", Name: "stats_result"}},
+		Labels:     []Label{{Value: "total", Name: "stats_result"}},
 		Timestamps: []int64{1583786142},
 		Values:     []float64{10},
 	}

app/vmalert/datasource/client_vlogs.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-func (c *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
+func (s *Client) setVLogsInstantReqParams(r *http.Request, query string, timestamp time.Time) {
 	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
 	if !*disablePathAppend {
 		r.URL.Path += "/select/logsql/stats_query"
@@ -16,15 +16,15 @@ func (c *Client) setVLogsInstantReqParams(r *http.Request, query string, timesta
 	q.Set("time", timestamp.Format(time.RFC3339))
 	// set the `start` and `end` params if applyIntervalAsTimeFilter is enabled(time filter is missing in the rule expr),
 	// so the query will be executed in time range [timestamp - evaluationInterval, timestamp].
-	if c.applyIntervalAsTimeFilter && c.evaluationInterval > 0 {
-		q.Set("start", timestamp.Add(-c.evaluationInterval).Format(time.RFC3339))
+	if s.applyIntervalAsTimeFilter && s.evaluationInterval > 0 {
+		q.Set("start", timestamp.Add(-s.evaluationInterval).Format(time.RFC3339))
 		q.Set("end", timestamp.Format(time.RFC3339))
 	}
 	r.URL.RawQuery = q.Encode()
-	c.setReqParams(r, query)
+	s.setReqParams(r, query)
 }
 
-func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
+func (s *Client) setVLogsRangeReqParams(r *http.Request, query string, start, end time.Time) {
 	// there is no type path prefix in victorialogs APIs right now, ignore appendTypePrefix.
 	if !*disablePathAppend {
 		r.URL.Path += "/select/logsql/stats_query_range"
@@ -33,11 +33,11 @@ func (c *Client) setVLogsRangeReqParams(r *http.Request, query string, start, en
 	q.Add("start", start.Format(time.RFC3339))
 	q.Add("end", end.Format(time.RFC3339))
 	// set step as evaluationInterval by default
-	if c.evaluationInterval > 0 {
-		q.Set("step", fmt.Sprintf("%ds", int(c.evaluationInterval.Seconds())))
+	if s.evaluationInterval > 0 {
+		q.Set("step", fmt.Sprintf("%ds", int(s.evaluationInterval.Seconds())))
 	}
 	r.URL.RawQuery = q.Encode()
-	c.setReqParams(r, query)
+	s.setReqParams(r, query)
 }
 
 func parseVLogsResponse(req *http.Request, resp *http.Response) (res Result, err error) {

app/vmalert/datasource/datasource.go

@@ -8,8 +8,6 @@ import (
 	"sort"
 	"strconv"
 	"time"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 // Querier interface wraps Query and QueryRange methods
@@ -57,7 +55,7 @@ type QuerierParams struct {
 
 // Metric is the basic entity which should be return by datasource
 type Metric struct {
-	Labels     []prompbmarshal.Label
+	Labels     []Label
 	Timestamps []int64
 	Values     []float64
 }
@@ -74,9 +72,22 @@ func (m *Metric) SetLabel(key, value string) {
 	m.AddLabel(key, value)
 }
 
+// SetLabels sets the given map as Metric labels
+func (m *Metric) SetLabels(ls map[string]string) {
+	var i int
+	m.Labels = make([]Label, len(ls))
+	for k, v := range ls {
+		m.Labels[i] = Label{
+			Name:  k,
+			Value: v,
+		}
+		i++
+	}
+}
+
 // AddLabel appends the given label to the label set
 func (m *Metric) AddLabel(key, value string) {
-	m.Labels = append(m.Labels, prompbmarshal.Label{Name: key, Value: value})
+	m.Labels = append(m.Labels, Label{Name: key, Value: value})
 }
 
 // DelLabel deletes the given label from the label set
@@ -99,8 +110,14 @@ func (m *Metric) Label(key string) string {
 	return ""
 }
 
+// Label represents metric's label
+type Label struct {
+	Name  string
+	Value string
+}
+
 // Labels is collection of Label
-type Labels []prompbmarshal.Label
+type Labels []Label
 
 func (ls Labels) Len() int           { return len(ls) }
 func (ls Labels) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
@@ -155,7 +172,7 @@ func LabelCompare(a, b Labels) int {
 // ConvertToLabels convert map to Labels
 func ConvertToLabels(m map[string]string) (labelset Labels) {
 	for k, v := range m {
-		labelset = append(labelset, prompbmarshal.Label{
+		labelset = append(labelset, Label{
 			Name:  k,
 			Value: v,
 		})

app/vmalert/datasource/vm_prom_api_test.go

@@ -3,8 +3,6 @@ package datasource
 import (
 	"reflect"
 	"testing"
-
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestPromInstant_UnmarshalPositive(t *testing.T) {
@@ -23,7 +21,7 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 
 	f(`[{"metric":{"__name__":"up"},"value":[1583780000,"42"]}]`, []Metric{
 		{
-			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
+			Labels:     []Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
@@ -33,17 +31,17 @@ func TestPromInstant_UnmarshalPositive(t *testing.T) {
 {"metric":{"__name__":"foo"},"value":[1583780001,"7"]},
 {"metric":{"__name__":"baz", "instance":"bar"},"value":[1583780002,"8"]}]`, []Metric{
 		{
-			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "up"}},
+			Labels:     []Label{{Name: "__name__", Value: "up"}},
 			Timestamps: []int64{1583780000},
 			Values:     []float64{42},
 		},
 		{
-			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "foo"}},
+			Labels:     []Label{{Name: "__name__", Value: "foo"}},
 			Timestamps: []int64{1583780001},
 			Values:     []float64{7},
 		},
 		{
-			Labels:     []prompbmarshal.Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
+			Labels:     []Label{{Name: "__name__", Value: "baz"}, {Name: "instance", Value: "bar"}},
 			Timestamps: []int64{1583780002},
 			Values:     []float64{8},
 		},

app/vmalert/notifier/alert.go

@@ -167,8 +167,14 @@ type tplData struct {
 	ExternalURL    string
 }
 
-func templateAnnotation(dst io.Writer, text string, data tplData, tpl *textTpl.Template, execute bool) error {
-	tpl, err := tpl.Parse(text)
+func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.Template, execute bool) error {
+	tpl, err := tmpl.Clone()
+	if err != nil {
+		return fmt.Errorf("error cloning template before parse annotation: %w", err)
+	}
+	// Clone() doesn't copy tpl Options, so we set them manually
+	tpl = tpl.Option("missingkey=zero")
+	tpl, err = tpl.Parse(text)
 	if err != nil {
 		return fmt.Errorf("error parsing annotation template: %w", err)
 	}

app/vmalert/notifier/alert_test.go

@@ -33,7 +33,7 @@ func TestAlertExecTemplate(t *testing.T) {
 		qFn := func(_ string) ([]datasource.Metric, error) {
 			return []datasource.Metric{
 				{
-					Labels: []prompbmarshal.Label{
+					Labels: []datasource.Label{
 						{Name: "foo", Value: "bar"},
 						{Name: "baz", Value: "qux"},
 					},
@@ -41,7 +41,7 @@ func TestAlertExecTemplate(t *testing.T) {
 					Timestamps: []int64{1},
 				},
 				{
-					Labels: []prompbmarshal.Label{
+					Labels: []datasource.Label{
 						{Name: "foo", Value: "garply"},
 						{Name: "baz", Value: "fred"},
 					},

app/vmalert/rule/alerting.go

@@ -14,10 +14,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // AlertingRule is basic alert entity
@@ -456,16 +454,13 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		ar.logDebugf(ts, a, "created in state PENDING")
 	}
 	var numActivePending int
-	var tss []prompbmarshal.TimeSeries
 	for h, a := range ar.alerts {
 		// if alert wasn't updated in this iteration
 		// means it is resolved already
 		if _, ok := updated[h]; !ok {
 			if a.State == notifier.StatePending {
-				// alert was in Pending state - it is not active anymore
-				// add stale time series
-				tss = append(tss, pendingAlertStaleTimeSeries(a.Labels, ts.Unix(), true)...)
-
+				// alert was in Pending state - it is not
+				// active anymore
 				delete(ar.alerts, h)
 				ar.logDebugf(ts, a, "PENDING => DELETED: is absent in current evaluation round")
 				continue
@@ -483,9 +478,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 				if ts.Sub(a.KeepFiringSince) >= ar.KeepFiringFor {
 					a.State = notifier.StateInactive
 					a.ResolvedAt = ts
-					// add stale time series
-					tss = append(tss, firingAlertStaleTimeSeries(a.Labels, ts.Unix())...)
-
 					ar.logDebugf(ts, a, "FIRING => INACTIVE: is absent in current evaluation round")
 					continue
 				}
@@ -497,10 +489,6 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 			a.State = notifier.StateFiring
 			a.Start = ts
 			alertsFired.Inc()
-			if ar.For > 0 {
-				// add stale time series
-				tss = append(tss, pendingAlertStaleTimeSeries(a.Labels, ts.Unix(), false)...)
-			}
 			ar.logDebugf(ts, a, "PENDING => FIRING: %s since becoming active at %v", ts.Sub(a.ActiveAt), a.ActiveAt)
 		}
 	}
@@ -509,7 +497,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
 		curState.Err = fmt.Errorf("exec exceeded limit of %d with %d alerts", limit, numActivePending)
 		return nil, curState.Err
 	}
-	return append(tss, ar.toTimeSeries(ts.Unix())...), nil
+	return ar.toTimeSeries(ts.Unix()), nil
 }
 
 func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.QueryFn, ts time.Time) (*labelSet, map[string]string, error) {
@@ -534,7 +522,6 @@ func (ar *AlertingRule) expandTemplates(m datasource.Metric, qFn templates.Query
 	return ls, as, nil
 }
 
-// toTimeSeries creates `ALERTS` and `ALERTS_FOR_STATE` for active alerts
 func (ar *AlertingRule) toTimeSeries(timestamp int64) []prompbmarshal.TimeSeries {
 	var tss []prompbmarshal.TimeSeries
 	for _, a := range ar.alerts {
@@ -614,83 +601,26 @@ func (ar *AlertingRule) alertToTimeSeries(a *notifier.Alert, timestamp int64) []
 }
 
 func alertToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
-	var labels []prompbmarshal.Label
+	labels := make(map[string]string)
 	for k, v := range a.Labels {
-		labels = append(labels, prompbmarshal.Label{
-			Name:  k,
-			Value: v,
-		})
-	}
-	// __name__ already been dropped, no need to check duplication
-	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
-	if ol := promrelabel.GetLabelByName(labels, alertStateLabel); ol != nil {
-		ol.Value = a.State.String()
-	} else {
-		labels = append(labels, prompbmarshal.Label{Name: alertStateLabel, Value: a.State.String()})
+		labels[k] = v
 	}
+	labels["__name__"] = alertMetricName
+	labels[alertStateLabel] = a.State.String()
 	return newTimeSeries([]float64{1}, []int64{timestamp}, labels)
 }
 
-// alertForToTimeSeries returns a time series that represents
+// alertForToTimeSeries returns a timeseries that represents
 // state of active alerts, where value is time when alert become active
 func alertForToTimeSeries(a *notifier.Alert, timestamp int64) prompbmarshal.TimeSeries {
-	var labels []prompbmarshal.Label
+	labels := make(map[string]string)
 	for k, v := range a.Labels {
-		labels = append(labels, prompbmarshal.Label{
-			Name:  k,
-			Value: v,
-		})
+		labels[k] = v
 	}
-	// __name__ already been dropped, no need to check duplication
-	labels = append(labels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
+	labels["__name__"] = alertForStateMetricName
 	return newTimeSeries([]float64{float64(a.ActiveAt.Unix())}, []int64{timestamp}, labels)
 }
 
-// pendingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
-// for alerts which changed their state from Pending to Inactive or Firing.
-func pendingAlertStaleTimeSeries(ls map[string]string, timestamp int64, includeAlertForState bool) []prompbmarshal.TimeSeries {
-	var result []prompbmarshal.TimeSeries
-	var baseLabels []prompbmarshal.Label
-	for k, v := range ls {
-		baseLabels = append(baseLabels, prompbmarshal.Label{
-			Name:  k,
-			Value: v,
-		})
-	}
-	// __name__ already been dropped, no need to check duplication
-	alertsLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
-	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StatePending.String()})
-	result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels))
-
-	if includeAlertForState {
-		alertsForStateLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
-		result = append(result, newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsForStateLabels))
-	}
-	return result
-}
-
-// firingAlertStaleTimeSeries returns stale `ALERTS` and `ALERTS_FOR_STATE` time series
-// for alerts which changed their state from Firing to Inactive.
-func firingAlertStaleTimeSeries(ls map[string]string, timestamp int64) []prompbmarshal.TimeSeries {
-	var baseLabels []prompbmarshal.Label
-	for k, v := range ls {
-		baseLabels = append(baseLabels, prompbmarshal.Label{
-			Name:  k,
-			Value: v,
-		})
-	}
-	// __name__ already been dropped, no need to check duplication
-	alertsLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertMetricName})
-	alertsLabels = append(alertsLabels, prompbmarshal.Label{Name: alertStateLabel, Value: notifier.StateFiring.String()})
-
-	alertsForStateLabels := append(baseLabels, prompbmarshal.Label{Name: "__name__", Value: alertForStateMetricName})
-
-	return []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsLabels),
-		newTimeSeries([]float64{decimal.StaleNaN}, []int64{timestamp}, alertsForStateLabels),
-	}
-}
-
 // restore restores the value of ActiveAt field for active alerts,
 // based on previously written time series `alertForStateMetricName`.
 // Only rules with For > 0 can be restored.

app/vmalert/rule/alerting_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
@@ -29,7 +28,7 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		rule.alerts[alert.ID] = alert
 		tss := rule.toTimeSeries(timestamp.Unix())
 		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
-			t.Fatalf("timeseries mismatch for rule %q: %s", rule.Name, err)
+			t.Fatalf("timeseries mismatch: %s", err)
 		}
 	}
 
@@ -37,23 +36,14 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: alertMetricName,
-			},
-			{
-				Name:  alertStateLabel,
-				Value: notifier.StateFiring.String(),
-			},
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":      alertMetricName,
+			alertStateLabel: notifier.StateFiring.String(),
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: alertForStateMetricName,
-				},
+			map[string]string{
+				"__name__": alertForStateMetricName,
 			}),
 	})
 
@@ -64,40 +54,18 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 			"instance": "bar",
 		},
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: alertMetricName,
-				},
-				{
-					Name:  alertStateLabel,
-					Value: notifier.StateFiring.String(),
-				},
-				{
-					Name:  "job",
-					Value: "foo",
-				},
-				{
-					Name:  "instance",
-					Value: "bar",
-				},
-			}),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":      alertMetricName,
+			alertStateLabel: notifier.StateFiring.String(),
+			"job":           "foo",
+			"instance":      "bar",
+		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: alertForStateMetricName,
-				},
-				{
-					Name:  "job",
-					Value: "foo",
-				},
-				{
-					Name:  "instance",
-					Value: "bar",
-				},
+			map[string]string{
+				"__name__": alertForStateMetricName,
+				"job":      "foo",
+				"instance": "bar",
 			}),
 	})
 
@@ -105,29 +73,18 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State: notifier.StateFiring, ActiveAt: timestamp.Add(time.Second),
 		Labels: map[string]string{
 			alertStateLabel: "foo",
+			"__name__":      "bar",
 		},
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: alertMetricName,
-			},
-			{
-				Name:  alertStateLabel,
-				Value: notifier.StateFiring.String(),
-			},
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":      alertMetricName,
+			alertStateLabel: notifier.StateFiring.String(),
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: alertForStateMetricName,
-				},
-				{
-					Name:  alertStateLabel,
-					Value: "foo",
-				},
+			map[string]string{
+				"__name__":      alertForStateMetricName,
+				alertStateLabel: "foo",
 			}),
 	})
 
@@ -135,23 +92,14 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StateFiring,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: alertMetricName,
-			},
-			{
-				Name:  alertStateLabel,
-				Value: notifier.StateFiring.String(),
-			},
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":      alertMetricName,
+			alertStateLabel: notifier.StateFiring.String(),
 		}),
 		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())},
 			[]int64{timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: alertForStateMetricName,
-				},
+			map[string]string{
+				"__name__": alertForStateMetricName,
 			}),
 	})
 
@@ -159,21 +107,12 @@ func TestAlertingRuleToTimeSeries(t *testing.T) {
 		State:    notifier.StatePending,
 		ActiveAt: timestamp.Add(time.Second),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: alertMetricName,
-			},
-			{
-				Name:  alertStateLabel,
-				Value: notifier.StatePending.String(),
-			},
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":      alertMetricName,
+			alertStateLabel: notifier.StatePending.String(),
 		}),
-		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: alertForStateMetricName,
-			},
+		newTimeSeries([]float64{float64(timestamp.Add(time.Second).Unix())}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": alertForStateMetricName,
 		}),
 	})
 }
@@ -185,9 +124,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		alert  *notifier.Alert
 	}
 
-	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
-
-	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert, tssExpected map[int][]prompbmarshal.TimeSeries) {
+	f := func(rule *AlertingRule, steps [][]datasource.Metric, alertsExpected map[int][]testAlert) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
@@ -197,19 +134,13 @@ func TestAlertingRule_Exec(t *testing.T) {
 			Name: "TestRule_Exec",
 		}
 		rule.GroupID = fakeGroup.ID()
+		ts := time.Now()
 		for i, step := range steps {
 			fq.Reset()
 			fq.Add(step...)
-			tss, err := rule.exec(context.TODO(), ts, 0)
-			if err != nil {
+			if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
 				t.Fatalf("unexpected error: %s", err)
 			}
-			// check generate time series
-			if _, ok := tssExpected[i]; ok {
-				if err := compareTimeSeries(t, tssExpected[i], tss); err != nil {
-					t.Fatalf("generated time series mismatch for rule %q in step %d: %s", rule.Name, i, err)
-				}
-			}
 
 			// shift the execution timestamp before the next iteration
 			ts = ts.Add(defaultStep)
@@ -243,21 +174,13 @@ func TestAlertingRule_Exec(t *testing.T) {
 		}
 	}
 
-	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil, nil)
+	f(newTestAlertingRule("empty", 0), [][]datasource.Metric{}, nil)
 
-	f(newTestAlertingRule("empty_labels", 0), [][]datasource.Metric{
+	f(newTestAlertingRule("empty labels", 0), [][]datasource.Metric{
 		{datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}},
 	}, map[int][]testAlert{
 		0: {{alert: &notifier.Alert{State: notifier.StateFiring}}},
-	},
-		map[int][]prompbmarshal.TimeSeries{
-			0: {
-				{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "empty_labels"}, {Name: "alertstate", Value: "firing"}},
-					Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
-				{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "empty_labels"}},
-					Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
-			},
-		})
+	})
 
 	f(newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive", 0), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -271,25 +194,6 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
-	}, map[int][]prompbmarshal.TimeSeries{
-		0: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
-		},
-		1: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-		},
-		2: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "single-firing=>inactive=>firing=>inactive=>inactive"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-		},
 	})
 
 	f(newTestAlertingRule("single-firing=>inactive=>firing=>inactive=>inactive=>firing", 0), [][]datasource.Metric{
@@ -306,7 +210,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	}, nil)
+	})
 
 	f(newTestAlertingRule("multiple-firing", 0), [][]datasource.Metric{
 		{
@@ -320,7 +224,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
 		},
-	}, nil)
+	})
 
 	// 1: fire first alert
 	// 2: fire second alert, set first inactive
@@ -329,57 +233,27 @@ func TestAlertingRule_Exec(t *testing.T) {
 		{metricWithLabels(t, "name", "foo")},
 		{metricWithLabels(t, "name", "foo1")},
 		{metricWithLabels(t, "name", "foo2")},
-	}, map[int][]testAlert{
-		0: {
-			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-		},
-		1: {
-			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-		},
-		2: {
-			{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-			{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
-			{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
-		},
-	}, map[int][]prompbmarshal.TimeSeries{
-		0: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
-		},
-		1: {
-			// stale time series for foo, `firing -> inactive`
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			// new time series for foo1
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-		},
-		2: {
-			// stale time series for foo1
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo1"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-			// new time series for foo2
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo2"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "multiple-steps-firing"}, {Name: "name", Value: "foo2"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(2 * defaultStep).Unix()), Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-		},
-	})
+	},
+		map[int][]testAlert{
+			0: {
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+			1: {
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+			2: {
+				{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo1"}, alert: &notifier.Alert{State: notifier.StateInactive}},
+				{labels: []string{"name", "foo2"}, alert: &notifier.Alert{State: notifier.StateFiring}},
+			},
+		})
 
 	f(newTestAlertingRule("for-pending", time.Minute), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
-	}, nil)
+	})
 
 	f(newTestAlertingRule("for-fired", defaultStep), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -387,22 +261,6 @@ func TestAlertingRule_Exec(t *testing.T) {
 	}, map[int][]testAlert{
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	}, map[int][]prompbmarshal.TimeSeries{
-		0: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
-		},
-		1: {
-			// stale time series for `pending -> firing`
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "alertstate", Value: "firing"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-fired"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Add(defaultStep).Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-		},
 	})
 
 	f(newTestAlertingRule("for-pending=>empty", time.Second), [][]datasource.Metric{
@@ -414,26 +272,6 @@ func TestAlertingRule_Exec(t *testing.T) {
 		0: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		2: {},
-	}, map[int][]prompbmarshal.TimeSeries{
-		0: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.UnixNano() / 1e6}}},
-		},
-		1: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: 1, Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: float64(ts.Unix()), Timestamp: ts.Add(defaultStep).UnixNano() / 1e6}}},
-		},
-		// stale time series for `pending -> inactive`
-		2: {
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "alertstate", Value: "pending"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-			{Labels: []prompbmarshal.Label{{Name: "__name__", Value: alertForStateMetricName}, {Name: "alertname", Value: "for-pending=>empty"}, {Name: "name", Value: "foo"}},
-				Samples: []prompbmarshal.Sample{{Value: decimal.StaleNaN, Timestamp: ts.Add(2*defaultStep).UnixNano() / 1e6}}},
-		},
 	})
 
 	f(newTestAlertingRule("for-pending=>firing=>inactive=>pending=>firing", defaultStep), [][]datasource.Metric{
@@ -449,7 +287,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	}, nil)
+	})
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>firing", defaultStep, 0, defaultStep, nil), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -462,7 +300,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		1: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		2: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
 		3: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	}, nil)
+	})
 
 	f(newTestAlertingRuleWithCustomFields("for-pending=>firing=>keepfiring=>keepfiring=>inactive=>pending=>firing", defaultStep, 0, 2*defaultStep, nil), [][]datasource.Metric{
 		{metricWithLabels(t, "name", "foo")},
@@ -483,7 +321,7 @@ func TestAlertingRule_Exec(t *testing.T) {
 		4: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateInactive}}},
 		5: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StatePending}}},
 		6: {{labels: []string{"name", "foo"}, alert: &notifier.Alert{State: notifier.StateFiring}}},
-	}, nil)
+	})
 }
 
 func TestAlertingRuleExecRange(t *testing.T) {
@@ -639,7 +477,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1, 1}, Timestamps: []int64{1, 3, 5}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
+			Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{State: notifier.StatePending, ActiveAt: time.Unix(1, 0)},
@@ -685,7 +523,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
 		{Values: []float64{1, 1}, Timestamps: []int64{1, 100}},
 		{
 			Values: []float64{1, 1}, Timestamps: []int64{1, 5},
-			Labels: []prompbmarshal.Label{{Name: "foo", Value: "bar"}},
+			Labels: []datasource.Label{{Name: "foo", Value: "bar"}},
 		},
 	}, []*notifier.Alert{
 		{
@@ -1209,7 +1047,7 @@ func newTestAlertingRuleWithCustomFields(name string, waitFor, evalInterval, kee
 
 func TestAlertingRule_ToLabels(t *testing.T) {
 	metric := datasource.Metric{
-		Labels: []prompbmarshal.Label{
+		Labels: []datasource.Label{
 			{Name: "instance", Value: "0.0.0.0:8800"},
 			{Name: "group", Value: "vmalert"},
 			{Name: "alertname", Value: "ConfigurationReloadFailure"},

app/vmalert/rule/group.go

@@ -8,9 +8,12 @@ import (
 	"fmt"
 	"hash/fnv"
 	"net/url"
+	"strconv"
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+
 	"github.com/cheggaaa/pb/v3"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
@@ -18,6 +21,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/metrics"
@@ -346,9 +350,10 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 	}
 
 	e := &executor{
-		Rw:              rw,
-		Notifiers:       nts,
-		notifierHeaders: g.NotifierHeaders,
+		Rw:                       rw,
+		Notifiers:                nts,
+		notifierHeaders:          g.NotifierHeaders,
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 
 	g.infof("started")
@@ -421,6 +426,8 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
 				continue
 			}
 
+			// ensure that staleness is tracked for existing rules only
+			e.purgeStaleSeries(g.Rules)
 			e.notifierHeaders = g.NotifierHeaders
 			g.mu.Unlock()
 
@@ -532,9 +539,10 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
 // ExecOnce evaluates all the rules under group for once with given timestamp.
 func (g *Group) ExecOnce(ctx context.Context, nts func() []notifier.Notifier, rw remotewrite.RWClient, evalTS time.Time) chan error {
 	e := &executor{
-		Rw:              rw,
-		Notifiers:       nts,
-		notifierHeaders: g.NotifierHeaders,
+		Rw:                       rw,
+		Notifiers:                nts,
+		notifierHeaders:          g.NotifierHeaders,
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 	if len(g.Rules) < 1 {
 		return nil
@@ -625,6 +633,13 @@ type executor struct {
 	notifierHeaders map[string]string
 
 	Rw remotewrite.RWClient
+
+	previouslySentSeriesToRWMu sync.Mutex
+	// previouslySentSeriesToRW stores series sent to RW on previous iteration
+	// map[ruleID]map[ruleLabels][]prompb.Label
+	// where `ruleID` is ID of the Rule within a Group
+	// and `ruleLabels` is []prompb.Label marshalled to a string
+	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
 }
 
 // execConcurrently executes rules concurrently if concurrency>1
@@ -691,6 +706,11 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 		if err := pushToRW(tss); err != nil {
 			return err
 		}
+
+		staleSeries := e.getStaleSeries(r, tss, ts)
+		if err := pushToRW(staleSeries); err != nil {
+			return err
+		}
 	}
 
 	ar, ok := r.(*AlertingRule)
@@ -717,3 +737,79 @@ func (e *executor) exec(ctx context.Context, r Rule, ts time.Time, resolveDurati
 	wg.Wait()
 	return errGr.Err()
 }
+
+var bbPool bytesutil.ByteBufferPool
+
+// getStaleSeries checks whether there are stale series from previously sent ones.
+func (e *executor) getStaleSeries(r Rule, tss []prompbmarshal.TimeSeries, timestamp time.Time) []prompbmarshal.TimeSeries {
+	bb := bbPool.Get()
+	defer bbPool.Put(bb)
+
+	ruleLabels := make(map[string][]prompbmarshal.Label, len(tss))
+	for _, ts := range tss {
+		// convert labels to strings, so we can compare with previously sent series
+		bb.B = labelsToString(bb.B, ts.Labels)
+		ruleLabels[string(bb.B)] = ts.Labels
+		bb.Reset()
+	}
+
+	rID := r.ID()
+	var staleS []prompbmarshal.TimeSeries
+	// check whether there are series which disappeared and need to be marked as stale
+	e.previouslySentSeriesToRWMu.Lock()
+	for key, labels := range e.previouslySentSeriesToRW[rID] {
+		if _, ok := ruleLabels[key]; ok {
+			continue
+		}
+		// previously sent series are missing in current series, so we mark them as stale
+		ss := newTimeSeriesPB([]float64{decimal.StaleNaN}, []int64{timestamp.Unix()}, labels)
+		staleS = append(staleS, ss)
+	}
+	// set previous series to current
+	e.previouslySentSeriesToRW[rID] = ruleLabels
+	e.previouslySentSeriesToRWMu.Unlock()
+
+	return staleS
+}
+
+// purgeStaleSeries deletes references in tracked
+// previouslySentSeriesToRW list to Rules which aren't present
+// in the given activeRules list. The method is used when the list
+// of loaded rules has changed and executor has to remove
+// references to non-existing rules.
+func (e *executor) purgeStaleSeries(activeRules []Rule) {
+	newPreviouslySentSeriesToRW := make(map[uint64]map[string][]prompbmarshal.Label)
+
+	e.previouslySentSeriesToRWMu.Lock()
+
+	for _, rule := range activeRules {
+		id := rule.ID()
+		prev, ok := e.previouslySentSeriesToRW[id]
+		if ok {
+			// keep previous series for staleness detection
+			newPreviouslySentSeriesToRW[id] = prev
+		}
+	}
+	e.previouslySentSeriesToRW = nil
+	e.previouslySentSeriesToRW = newPreviouslySentSeriesToRW
+
+	e.previouslySentSeriesToRWMu.Unlock()
+}
+
+func labelsToString(dst []byte, labels []prompbmarshal.Label) []byte {
+	dst = append(dst, '{')
+	for i, label := range labels {
+		if len(label.Name) == 0 {
+			dst = append(dst, "__name__"...)
+		} else {
+			dst = append(dst, label.Name...)
+		}
+		dst = append(dst, '=')
+		dst = strconv.AppendQuote(dst, label.Value)
+		if i < len(labels)-1 {
+			dst = append(dst, ',')
+		}
+	}
+	dst = append(dst, '}')
+	return dst
+}

app/vmalert/rule/group_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"os"
+	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -16,6 +17,8 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/templates"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
@@ -380,6 +383,153 @@ func TestGetResolveDuration(t *testing.T) {
 	f(2*time.Minute, 0, 1*time.Minute, 8*time.Minute)
 }
 
+func TestGetStaleSeries(t *testing.T) {
+	ts := time.Now()
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	f := func(r Rule, labels, expLabels [][]prompbmarshal.Label) {
+		t.Helper()
+
+		var tss []prompbmarshal.TimeSeries
+		for _, l := range labels {
+			tss = append(tss, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, l))
+		}
+		staleS := e.getStaleSeries(r, tss, ts)
+		if staleS == nil && expLabels == nil {
+			return
+		}
+		if len(staleS) != len(expLabels) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expLabels), len(staleS))
+		}
+		for i, exp := range expLabels {
+			got := staleS[i]
+			if !reflect.DeepEqual(exp, got.Labels) {
+				t.Fatalf("expected to get labels: \n%v;\ngot instead: \n%v",
+					exp, got.Labels)
+			}
+			if len(got.Samples) != 1 {
+				t.Fatalf("expected to have 1 sample; got %d", len(got.Samples))
+			}
+			if !decimal.IsStaleNaN(got.Samples[0].Value) {
+				t.Fatalf("expected sample value to be %v; got %v", decimal.StaleNaN, got.Samples[0].Value)
+			}
+		}
+	}
+
+	// warn: keep in mind, that executor holds the state, so sequence of f calls matters
+
+	// single series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		nil)
+
+	// multiple series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		nil,
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")})
+
+	// multiple rules and series
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 2},
+		[][]prompbmarshal.Label{
+			toPromLabels(t, "__name__", "job:foo", "job", "foo"),
+			toPromLabels(t, "__name__", "job:foo", "job", "bar"),
+		},
+		nil)
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "foo")})
+	f(&AlertingRule{RuleID: 1},
+		[][]prompbmarshal.Label{toPromLabels(t, "__name__", "job:foo", "job", "bar")},
+		nil)
+}
+
+func TestPurgeStaleSeries(t *testing.T) {
+	ts := time.Now()
+	labels := toPromLabels(t, "__name__", "job:foo", "job", "foo")
+	tss := []prompbmarshal.TimeSeries{newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels)}
+
+	f := func(curRules, newRules, expStaleRules []Rule) {
+		t.Helper()
+		e := &executor{
+			previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+		}
+		// seed executor with series for
+		// current rules
+		for _, rule := range curRules {
+			e.getStaleSeries(rule, tss, ts)
+		}
+
+		e.purgeStaleSeries(newRules)
+
+		if len(e.previouslySentSeriesToRW) != len(expStaleRules) {
+			t.Fatalf("expected to get %d stale series, got %d",
+				len(expStaleRules), len(e.previouslySentSeriesToRW))
+		}
+
+		for _, exp := range expStaleRules {
+			if _, ok := e.previouslySentSeriesToRW[exp.ID()]; !ok {
+				t.Fatalf("expected to have rule %d; got nil instead", exp.ID())
+			}
+		}
+	}
+
+	f(nil, nil, nil)
+	f(
+		nil,
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		nil,
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		nil,
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 2}},
+	)
+	f(
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+		[]Rule{&AlertingRule{RuleID: 1}, &AlertingRule{RuleID: 2}},
+	)
+}
+
 func TestFaultyNotifier(t *testing.T) {
 	fq := &datasource.FakeQuerier{}
 	fq.Add(metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "bar"))
@@ -430,7 +580,8 @@ func TestFaultyRW(t *testing.T) {
 	}
 
 	e := &executor{
-		Rw: &remotewrite.Client{},
+		Rw:                       &remotewrite.Client{},
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
 	}
 
 	err := e.exec(context.Background(), r, time.Now(), 0, 10)

app/vmalert/rule/group_timing_test.go

@@ -0,0 +1,36 @@
+package rule
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+)
+
+func BenchmarkGetStaleSeries(b *testing.B) {
+	ts := time.Now()
+	n := 100
+	payload := make([]prompbmarshal.TimeSeries, 0, n)
+	for i := 0; i < n; i++ {
+		s := fmt.Sprintf("%d", i)
+		labels := toPromLabels(b,
+			"__name__", "foo", ""+
+				"instance", s,
+			"job", s,
+			"state", s,
+		)
+		payload = append(payload, newTimeSeriesPB([]float64{1}, []int64{ts.Unix()}, labels))
+	}
+
+	e := &executor{
+		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
+	}
+	ar := &AlertingRule{RuleID: 1}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		e.getStaleSeries(ar, payload, ts)
+	}
+}

app/vmalert/rule/recording.go

@@ -3,17 +3,16 @@ package rule
 import (
 	"context"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
 // RecordingRule is a Rule that supposed
@@ -35,8 +34,6 @@ type RecordingRule struct {
 	// during evaluations
 	state *ruleState
 
-	lastEvaluation map[string]struct{}
-
 	metrics *recordingRuleMetrics
 }
 
@@ -116,7 +113,7 @@ func (rr *RecordingRule) execRange(ctx context.Context, start, end time.Time) ([
 	var tss []prompbmarshal.TimeSeries
 	for _, s := range res.Data {
 		ts := rr.toTimeSeries(s)
-		key := stringifyLabels(ts.Labels)
+		key := stringifyLabels(ts)
 		if _, ok := duplicates[key]; ok {
 			return nil, fmt.Errorf("original metric %v; resulting labels %q: %w", s.Labels, key, errDuplicate)
 		}
@@ -158,47 +155,28 @@ func (rr *RecordingRule) exec(ctx context.Context, ts time.Time, limit int) ([]p
 		return nil, curState.Err
 	}
 
-	curEvaluation := make(map[string]struct{}, len(qMetrics))
-	lastEvaluation := rr.lastEvaluation
+	duplicates := make(map[string]struct{}, len(qMetrics))
 	var tss []prompbmarshal.TimeSeries
 	for _, r := range qMetrics {
 		ts := rr.toTimeSeries(r)
-		key := stringifyLabels(ts.Labels)
-		if _, ok := curEvaluation[key]; ok {
+		key := stringifyLabels(ts)
+		if _, ok := duplicates[key]; ok {
 			curState.Err = fmt.Errorf("original metric %v; resulting labels %q: %w", r, key, errDuplicate)
 			return nil, curState.Err
 		}
-		curEvaluation[key] = struct{}{}
-		delete(lastEvaluation, key)
+		duplicates[key] = struct{}{}
 		tss = append(tss, ts)
 	}
-	// check for stale time series
-	for k := range lastEvaluation {
-		tss = append(tss, prompbmarshal.TimeSeries{
-			Labels: stringToLabels(k),
-			Samples: []prompbmarshal.Sample{
-				{Value: decimal.StaleNaN, Timestamp: ts.UnixNano() / 1e6},
-			}})
-	}
-	rr.lastEvaluation = curEvaluation
 	return tss, nil
 }
 
-func stringToLabels(s string) []prompbmarshal.Label {
-	labels := strings.Split(s, ",")
-	rLabels := make([]prompbmarshal.Label, 0, len(labels))
-	for i := range labels {
-		if label := strings.Split(labels[i], "="); len(label) == 2 {
-			rLabels = append(rLabels, prompbmarshal.Label{
-				Name:  label[0],
-				Value: label[1],
-			})
-		}
+func stringifyLabels(ts prompbmarshal.TimeSeries) string {
+	labels := ts.Labels
+	if len(labels) > 1 {
+		sort.Slice(labels, func(i, j int) bool {
+			return labels[i].Name < labels[j].Name
+		})
 	}
-	return rLabels
-}
-
-func stringifyLabels(labels []prompbmarshal.Label) string {
 	b := strings.Builder{}
 	for i, l := range labels {
 		b.WriteString(l.Name)
@@ -212,27 +190,19 @@ func stringifyLabels(labels []prompbmarshal.Label) string {
 }
 
 func (rr *RecordingRule) toTimeSeries(m datasource.Metric) prompbmarshal.TimeSeries {
-	if preN := promrelabel.GetLabelByName(m.Labels, "__name__"); preN != nil {
-		preN.Value = rr.Name
-	} else {
-		m.Labels = append(m.Labels, prompbmarshal.Label{
-			Name:  "__name__",
-			Value: rr.Name,
-		})
+	labels := make(map[string]string)
+	for _, l := range m.Labels {
+		labels[l.Name] = l.Value
 	}
-	for k := range rr.Labels {
-		prevLabel := promrelabel.GetLabelByName(m.Labels, k)
-		if prevLabel != nil && prevLabel.Value != rr.Labels[k] {
-			// Rename the prevLabel to "exported_" + label.Name
-			prevLabel.Name = fmt.Sprintf("exported_%s", prevLabel.Name)
+	labels["__name__"] = rr.Name
+	// override existing labels with configured ones
+	for k, v := range rr.Labels {
+		if _, ok := labels[k]; ok && labels[k] != v {
+			labels[fmt.Sprintf("exported_%s", k)] = labels[k]
 		}
-		m.Labels = append(m.Labels, prompbmarshal.Label{
-			Name:  k,
-			Value: rr.Labels[k],
-		})
+		labels[k] = v
 	}
-	ts := newTimeSeries(m.Values, m.Timestamps, m.Labels)
-	return ts
+	return newTimeSeries(m.Values, m.Timestamps, labels)
 }
 
 // updateWith copies all significant fields.

app/vmalert/rule/recording_test.go

@@ -9,151 +9,59 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
 func TestRecordingRule_Exec(t *testing.T) {
-	ts, _ := time.Parse(time.RFC3339, "2024-10-29T00:00:00Z")
-	const defaultStep = 5 * time.Millisecond
-
-	f := func(rule *RecordingRule, steps [][]datasource.Metric, tssExpected [][]prompbmarshal.TimeSeries) {
+	f := func(rule *RecordingRule, metrics []datasource.Metric, tssExpected []prompbmarshal.TimeSeries) {
 		t.Helper()
 
 		fq := &datasource.FakeQuerier{}
-		for i, step := range steps {
-			fq.Reset()
-			fq.Add(step...)
-			rule.q = fq
-			rule.state = &ruleState{
-				entries: make([]StateEntry, 10),
-			}
-			tss, err := rule.exec(context.TODO(), ts, 0)
-			if err != nil {
-				t.Fatalf("unexpected RecordingRule.exec error: %s", err)
-			}
-			if err := compareTimeSeries(t, tssExpected[i], tss); err != nil {
-				t.Fatalf("time series mismatch: %s", err)
-			}
-
-			ts = ts.Add(defaultStep)
+		fq.Add(metrics...)
+		rule.q = fq
+		rule.state = &ruleState{
+			entries: make([]StateEntry, 10),
+		}
+		tss, err := rule.exec(context.TODO(), time.Now(), 0)
+		if err != nil {
+			t.Fatalf("unexpected RecordingRule.exec error: %s", err)
+		}
+		if err := compareTimeSeries(t, tssExpected, tss); err != nil {
+			t.Fatalf("timeseries missmatch: %s", err)
 		}
 	}
 
+	timestamp := time.Now()
+
 	f(&RecordingRule{
 		Name: "foo",
-	}, [][]datasource.Metric{{
+	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 10, "__name__", "bar"),
-	}}, [][]prompbmarshal.TimeSeries{{
-		newTimeSeries([]float64{10}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foo",
-			},
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{10}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foo",
 		}),
-	}})
+	})
 
 	f(&RecordingRule{
 		Name: "foobarbaz",
-	}, [][]datasource.Metric{
-		{
-			metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
-			metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
-			metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
-		},
-		{
-			metricWithValueAndLabels(t, 10, "__name__", "foo", "job", "foo"),
-		},
-		{
-			metricWithValueAndLabels(t, 10, "__name__", "foo", "job", "bar"),
-		},
-	}, [][]prompbmarshal.TimeSeries{{
-		newTimeSeries([]float64{1}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foobarbaz",
-			},
-			{
-				Name:  "job",
-				Value: "foo",
-			},
+	}, []datasource.Metric{
+		metricWithValueAndLabels(t, 1, "__name__", "foo", "job", "foo"),
+		metricWithValueAndLabels(t, 2, "__name__", "bar", "job", "bar"),
+		metricWithValueAndLabels(t, 3, "__name__", "baz", "job", "baz"),
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "foo",
 		}),
-		newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foobarbaz",
-			},
-			{
-				Name:  "job",
-				Value: "bar",
-			},
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "bar",
 		}),
-		newTimeSeries([]float64{3}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foobarbaz",
-			},
-			{
-				Name:  "job",
-				Value: "baz",
-			},
+		newTimeSeries([]float64{3}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "baz",
 		}),
-	},
-		{
-			newTimeSeries([]float64{10}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foobarbaz",
-				},
-				{
-					Name:  "job",
-					Value: "foo",
-				},
-			}),
-			// other series are with NaN values
-			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foobarbaz",
-				},
-				{
-					Name:  "job",
-					Value: "bar",
-				},
-			}),
-			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(defaultStep).UnixNano()}, []prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foobarbaz",
-				},
-				{
-					Name:  "job",
-					Value: "baz",
-				},
-			}),
-		},
-		{
-			newTimeSeries([]float64{10}, []int64{ts.Add(2 * defaultStep).UnixNano()}, []prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foobarbaz",
-				},
-				{
-					Name:  "job",
-					Value: "bar",
-				},
-			}),
-			newTimeSeries([]float64{decimal.StaleNaN}, []int64{ts.Add(2 * defaultStep).UnixNano()}, []prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foobarbaz",
-				},
-				{
-					Name:  "job",
-					Value: "foo",
-				},
-			}),
-		},
 	})
 
 	f(&RecordingRule{
@@ -161,44 +69,22 @@ func TestRecordingRule_Exec(t *testing.T) {
 		Labels: map[string]string{
 			"source": "test",
 		},
-	}, [][]datasource.Metric{{
+	}, []datasource.Metric{
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar", "source", "origin"),
-	}}, [][]prompbmarshal.TimeSeries{{
-		newTimeSeries([]float64{2}, []int64{ts.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "job:foo",
-			},
-			{
-				Name:  "job",
-				Value: "foo",
-			},
-			{
-				Name:  "source",
-				Value: "test",
-			},
+	}, []prompbmarshal.TimeSeries{
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "job:foo",
+			"job":      "foo",
+			"source":   "test",
 		}),
-		newTimeSeries([]float64{1}, []int64{ts.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "job:foo",
-				},
-				{
-					Name:  "job",
-					Value: "bar",
-				},
-				{
-					Name:  "source",
-					Value: "test",
-				},
-				{
-					Name:  "exported_source",
-					Value: "origin",
-				},
-			}),
-	}})
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__":        "job:foo",
+			"job":             "bar",
+			"source":          "test",
+			"exported_source": "origin",
+		}),
+	})
 }
 
 func TestRecordingRule_ExecRange(t *testing.T) {
@@ -224,13 +110,9 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 	}, []datasource.Metric{
 		metricWithValuesAndLabels(t, []float64{10, 20, 30}, "__name__", "bar"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foo",
-				},
-			}),
+		newTimeSeries([]float64{10, 20, 30}, []int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+			"__name__": "foo",
+		}),
 	})
 
 	f(&RecordingRule{
@@ -240,36 +122,18 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 		metricWithValuesAndLabels(t, []float64{2, 3}, "__name__", "bar", "job", "bar"),
 		metricWithValuesAndLabels(t, []float64{4, 5, 6}, "__name__", "baz", "job", "baz"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foobarbaz",
-			},
-			{
-				Name:  "job",
-				Value: "foo",
-			},
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "foo",
 		}),
-		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "foobarbaz",
-			},
-			{
-				Name:  "job",
-				Value: "bar",
-			},
+		newTimeSeries([]float64{2, 3}, []int64{timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+			"__name__": "foobarbaz",
+			"job":      "bar",
 		}),
 		newTimeSeries([]float64{4, 5, 6},
-			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, []prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "foobarbaz",
-				},
-				{
-					Name:  "job",
-					Value: "baz",
-				},
+			[]int64{timestamp.UnixNano(), timestamp.UnixNano(), timestamp.UnixNano()}, map[string]string{
+				"__name__": "foobarbaz",
+				"job":      "baz",
 			}),
 	})
 
@@ -282,35 +146,16 @@ func TestRecordingRule_ExecRange(t *testing.T) {
 		metricWithValueAndLabels(t, 2, "__name__", "foo", "job", "foo"),
 		metricWithValueAndLabels(t, 1, "__name__", "bar", "job", "bar"),
 	}, []prompbmarshal.TimeSeries{
-		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, []prompbmarshal.Label{
-			{
-				Name:  "__name__",
-				Value: "job:foo",
-			},
-			{
-				Name:  "job",
-				Value: "foo",
-			},
-			{
-				Name:  "source",
-				Value: "test",
-			},
+		newTimeSeries([]float64{2}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "job:foo",
+			"job":      "foo",
+			"source":   "test",
+		}),
+		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()}, map[string]string{
+			"__name__": "job:foo",
+			"job":      "bar",
+			"source":   "test",
 		}),
-		newTimeSeries([]float64{1}, []int64{timestamp.UnixNano()},
-			[]prompbmarshal.Label{
-				{
-					Name:  "__name__",
-					Value: "job:foo",
-				},
-				{
-					Name:  "job",
-					Value: "bar",
-				},
-				{
-					Name:  "source",
-					Value: "test",
-				},
-			}),
 	})
 }
 

app/vmalert/rule/test_helpers.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
@@ -88,7 +87,7 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	}
 	m := datasource.Metric{Values: []float64{1}, Timestamps: []int64{1}}
 	for i := 0; i < len(labels); i += 2 {
-		m.Labels = append(m.Labels, prompbmarshal.Label{
+		m.Labels = append(m.Labels, datasource.Label{
 			Name:  labels[i],
 			Value: labels[i+1],
 		})
@@ -96,6 +95,21 @@ func metricWithLabels(t *testing.T, labels ...string) datasource.Metric {
 	return m
 }
 
+func toPromLabels(t testing.TB, labels ...string) []prompbmarshal.Label {
+	t.Helper()
+	if len(labels) == 0 || len(labels)%2 != 0 {
+		t.Fatalf("expected to get even number of labels")
+	}
+	var ls []prompbmarshal.Label
+	for i := 0; i < len(labels); i += 2 {
+		ls = append(ls, prompbmarshal.Label{
+			Name:  labels[i],
+			Value: labels[i+1],
+		})
+	}
+	return ls
+}
+
 func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 	t.Helper()
 	if len(a) != len(b) {
@@ -108,7 +122,7 @@ func compareTimeSeries(t *testing.T, a, b []prompbmarshal.TimeSeries) error {
 		}
 		for i, exp := range expTS.Samples {
 			got := gotTS.Samples[i]
-			if got.Value != exp.Value && (!decimal.IsStaleNaN(got.Value) || !decimal.IsStaleNaN(exp.Value)) {
+			if got.Value != exp.Value {
 				return fmt.Errorf("expected value %.2f; got %.2f", exp.Value, got.Value)
 			}
 			// timestamp validation isn't always correct for now.

app/vmalert/rule/utils.go

@@ -9,14 +9,10 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
 )
 
-// newTimeSeries first sorts given labels, then returns new time series.
-func newTimeSeries(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
-	promrelabel.SortLabels(labels)
+func newTimeSeries(values []float64, timestamps []int64, labels map[string]string) prompbmarshal.TimeSeries {
 	ts := prompbmarshal.TimeSeries{
-		Labels:  labels,
 		Samples: make([]prompbmarshal.Sample, len(values)),
 	}
 	for i := range values {
@@ -25,6 +21,34 @@ func newTimeSeries(values []float64, timestamps []int64, labels []prompbmarshal.
 			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
 		}
 	}
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys) // make order deterministic
+	for _, key := range keys {
+		ts.Labels = append(ts.Labels, prompbmarshal.Label{
+			Name:  key,
+			Value: labels[key],
+		})
+	}
+	return ts
+}
+
+// newTimeSeriesPB creates prompbmarshal.TimeSeries with given
+// values, timestamps and labels.
+// It expects that labels are already sorted.
+func newTimeSeriesPB(values []float64, timestamps []int64, labels []prompbmarshal.Label) prompbmarshal.TimeSeries {
+	ts := prompbmarshal.TimeSeries{
+		Samples: make([]prompbmarshal.Sample, len(values)),
+	}
+	for i := range values {
+		ts.Samples[i] = prompbmarshal.Sample{
+			Value:     values[i],
+			Timestamp: time.Unix(timestamps[i], 0).UnixNano() / 1e6,
+		}
+	}
+	ts.Labels = labels
 	return ts
 }
 

app/vmalert/templates/template.go

@@ -169,8 +169,6 @@ func GetWithFuncs(funcs textTpl.FuncMap) (*textTpl.Template, error) {
 	if err != nil {
 		return nil, err
 	}
-	// Clone() doesn't copy tpl Options, so we set them manually
-	tmpl = tmpl.Option("missingkey=zero")
 	return tmpl.Funcs(funcs), nil
 }
 

deployment/docker/vmanomaly/vmanomaly-integration/docker-compose.yml

@@ -72,7 +72,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.18.1
+    image: victoriametrics/vmanomaly:v1.18.0
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/CHANGELOG.md

@@ -11,25 +11,6 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
-## v1.18.1
-Released: 2024-11-12
-
-- IMPROVEMENT: Added a [reader-level](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) `data_range` argument, allowing users to define a default *valid* data range for all input queries in `queries`. Individual queries can still override this default with their own `data_range` if needed.
-- IMPROVEMENT: Added the `url` label to enhance labelset consistency across [self-monitoring metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#metrics-generated-by-vmanomaly) in both [reader](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#reader-behaviour-metrics) and [writer](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#writer-behaviour-metrics) components. Metrics affected:
-  - `vmanomaly_reader_received_bytes`
-  - `vmanomaly_reader_response_parsing_seconds`
-  - `vmanomaly_reader_timeseries_received`
-  - `vmanomaly_reader_datapoints_received`
-  - `vmanomaly_writer_request_serialize_seconds`
-  - `vmanomaly_writer_datapoints_sent`
-  - `vmanomaly_writer_timeseries_sent`
-
-- FIX: Resolved an issue where [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) incorrectly set their last seen `infer` timestamp during *first* `fit_infer` call, resulting in output being produced for *every datapoint* within the `fit_window` on its *first invocation*.
-- FIX: Resolved an issue in multi-[scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) configurations where [self-monitoring metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#metrics-generated-by-vmanomaly) values were overwriting each other.
-- FIX: Resolved an issue causing incorrect `query_key` label values in the `vmanomaly_model_datapoints_produced` [self-monitoring metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#models-behaviour-metrics) for [univariate models](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models).
-- FIX: Resolved an issue that caused the `vmanomaly_model_runs` [self-monitoring metric](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#models-behaviour-metrics) to miss increments for [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models).
-- FIX: Aligned the calculations of `vmanomaly_model_datapoints_accepted` and `vmanomaly_model_datapoints_produced` [self-monitoring model metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#models-behaviour-metrics) across all stages (`fit`, `infer`, and `fit_infer`) for consistency.
-
 ## v1.18.0
 Released: 2024-10-28
 

docs/anomaly-detection/FAQ.md

@@ -159,7 +159,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.18.1
+    image: victoriametrics/vmanomaly:v1.18.0
     # ...
     ports:
       - "8490:8490"

docs/anomaly-detection/Overview.md

@@ -229,7 +229,7 @@ This will expose metrics at `http://0.0.0.0:8080/metrics` page.
 To use *vmanomaly* you need to pull docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.18.1
+docker pull victoriametrics/vmanomaly:v1.18.0
 ```
 
 > Note: please check what is latest release in [CHANGELOG](https://docs.victoriametrics.com/anomaly-detection/changelog/)
@@ -239,7 +239,7 @@ docker pull victoriametrics/vmanomaly:v1.18.1
 You can put a tag on it for your convenience:
 
 ```sh
-docker image tag victoriametrics/vmanomaly:v1.18.1 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.18.0 vmanomaly
 ```
 Here is an example of how to run *vmanomaly* docker container with [license file](#licensing):
 

docs/anomaly-detection/QuickStart.md

@@ -58,13 +58,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.18.1
+docker pull victoriametrics/vmanomaly:v1.18.0
 ```
 
 2. (Optional step) tag the `vmanomaly` Docker image:
 
 ```sh
-docker image tag victoriametrics/vmanomaly:v1.18.1 vmanomaly
+docker image tag victoriametrics/vmanomaly:v1.18.0 vmanomaly
 ```
 
 3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
@@ -98,7 +98,7 @@ docker run -it --user 1000:1000 \
 services:
   # ...
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.18.1
+    image: victoriametrics/vmanomaly:v1.18.0
     volumes:
         $YOUR_LICENSE_FILE_PATH:/license
         $YOUR_CONFIG_FILE_PATH:/config.yml

docs/anomaly-detection/components/models.md

@@ -984,7 +984,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.18.1
+docker pull victoriametrics/vmanomaly:v1.18.0
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -998,7 +998,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.18.1 /config.yaml \
+victoriametrics/vmanomaly:v1.18.0 /config.yaml \
 --licenseFile=/license
 ```
 

docs/anomaly-detection/components/monitoring.md

@@ -319,7 +319,7 @@ Label names [description](#labelnames)
             <td>The total number of bytes received in responses for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
-`url`, `query_key`, `scheduler_alias`, `preset`
+`query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
         <tr>
@@ -331,7 +331,7 @@ Label names [description](#labelnames)
             <td>The total time (in seconds) taken for data parsing at each `step` (json, dataframe) for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
-`step`, `url`, `query_key`, `scheduler_alias`, `preset`
+`step`, `query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
         <tr>
@@ -343,7 +343,7 @@ Label names [description](#labelnames)
             <td>The total number of timeseries received from VictoriaMetrics for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
-`url`, `query_key`, `scheduler_alias`, `preset`
+`query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
         <tr>
@@ -355,7 +355,7 @@ Label names [description](#labelnames)
             <td>The total number of datapoints received from VictoriaMetrics for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
-`url`, `query_key`, `scheduler_alias`, `preset`
+`query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
     </tbody>
@@ -530,7 +530,7 @@ Label names [description](#labelnames)
             <td>The total time (in seconds) taken for serializing data for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
-`url`, `query_key`, `scheduler_alias`, `preset`
+`query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
         <tr>
@@ -542,7 +542,7 @@ Label names [description](#labelnames)
             <td>The total number of datapoints sent to VictoriaMetrics for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
 
-`url`, `query_key`, `scheduler_alias`, `preset`
+`query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
         <tr>
@@ -552,8 +552,7 @@ Label names [description](#labelnames)
             <td>`Counter`</td>
             <td>The total number of timeseries sent to VictoriaMetrics for the `query_key` query within the specified scheduler `scheduler_alias`, in the `vmanomaly` service running in `preset` mode.</td>
             <td>
-
-`url`, `query_key`, `scheduler_alias`, `preset`
+`query_key`, `scheduler_alias`, `preset`
             </td>
         </tr>
     </tbody>

docs/anomaly-detection/components/reader.md

@@ -68,8 +68,6 @@ Starting from [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/chang
   - **High anomaly scores** (>1) when the *data falls outside the expected range*, indicating a data constraint violation.
   - **Lowest anomaly scores** (=0) when the *model's predictions (`yhat`) fall outside the expected range*, meaning uncertain predictions.
 
-  > **Note**: if not set explicitly (or if older config style prior to [v1.13.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1130)) is used, then it is set to reader-level `data_range` arg (since [v1.18.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1181))
-
 - `max_points_per_query` (int): Introduced in [v1.17.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1170), optional arg overrides how `search.maxPointsPerTimeseries` flag (available since [v1.14.1](#v1141)) impacts `vmanomaly` on splitting long `fit_window` [queries](https://docs.victoriametrics.com/anomaly-detection/components/reader/?highlight=queries#vm-reader) into smaller sub-intervals. This helps users avoid hitting the `search.maxQueryDuration` limit for individual queries by distributing initial query across multiple subquery requests with minimal overhead. Set less than `search.maxPointsPerTimeseries` if hitting `maxQueryDuration` limits. If set on a query-level, it overrides the global `max_points_per_query` (reader-level).
 
 - `tz` (string): Introduced in [v1.18.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1180), this optional argument enables timezone specification per query, overriding the reader’s default `tz`. This setting helps to account for local timezone shifts, such as [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models that are sensitive to seasonal variations (e.g., [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)).
@@ -325,17 +323,6 @@ Introduced in [v1.17.0](https://docs.victoriametrics.com/anomaly-detection/chang
 Introduced in [v1.18.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1180), this optional argument specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
             </td>
         </tr>
-        <tr>
-            <td>
-`data_range`
-            </td>
-            <td>
-`["-inf", "inf"]`
-            </td>
-            <td>
-Added in [v1.18.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1180), this optional argument allows defining **valid** data ranges for input of all the queries in `queries`. Defaults to `["-inf", "inf"]` if not set and can be overridden on a [per-query basis](#per-query-parameters).
-            </td>
-        </tr>
     </tbody>
 </table>
 
@@ -347,13 +334,12 @@ reader:
   datasource_url: "https://play.victoriametrics.com/"
   tenant_id: "0:0"
   tz: 'America/New_York'
-  data_range: [1, 'inf']  # reader-level
   queries:
     ingestion_rate:
       expr: 'sum(rate(vm_rows_inserted_total[5m])) by (type) > 0'
-      step: '1m' # can override reader-level `sampling_period` on per-query level
-      data_range: [0, 'inf']  # if set, overrides reader-level data_range
-      tz: 'Australia/Sydney'  # if set, overrides reader-level tz
+      step: '1m' # can override global `sampling_period` on per-query level
+      data_range: [0, 'inf']
+      tz: 'Australia/Sydney'  # if set, overrides reader-wise tz
   sampling_period: '1m'
   query_from_last_seen_timestamp: True  # false by default
   latency_offset: '1ms'

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -385,7 +385,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.18.1
+    image: victoriametrics/vmanomaly:v1.18.0
     depends_on:
       - "victoriametrics"
     ports:

lib/prompbmarshal/util.go

@@ -27,7 +27,7 @@ func (wr *WriteRequest) Reset() {
 // ResetTimeSeries clears all the GC references from tss and returns an empty tss ready for further use.
 func ResetTimeSeries(tss []TimeSeries) []TimeSeries {
 	clear(tss)
-	return make([]TimeSeries, 0)
+	return tss[:0]
 }
 
 // MustParsePromMetrics parses metrics in Prometheus text exposition format from s and returns them.

lib/promutils/labelscompressor.go

@@ -11,16 +11,77 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
+type queue struct {
+	in      chan uint64
+	out     chan uint64
+	storage []uint64
+}
+
+func newQueue() *queue {
+	q := &queue{
+		out: make(chan uint64),
+		in:  make(chan uint64),
+	}
+	go func() {
+		defer close(q.out)
+		for {
+			if len(q.storage) == 0 {
+				item, ok := <-q.in
+				if !ok {
+					return
+				}
+				q.storage = append(q.storage, item)
+				continue
+			}
+
+			select {
+			case item, ok := <-q.in:
+				if ok {
+					q.storage = append(q.storage, item)
+				} else {
+					// unwind storage
+					for _, item := range q.storage {
+						q.out <- item
+					}
+
+					return
+				}
+			case q.out <- q.storage[0]:
+				if len(q.storage) == 1 {
+					q.storage = nil
+				} else {
+					q.storage = q.storage[1:]
+				}
+			}
+		}
+	}()
+	return q
+}
+
+type compressedLabel struct {
+	mu             sync.Mutex
+	code           uint64
+	deleted        bool
+	deleteDeadline int64
+}
+
 // LabelsCompressor compresses []prompbmarshal.Label into short binary strings
 type LabelsCompressor struct {
 	labelToIdx sync.Map
 	idxToLabel labelsMap
+	freeIdxs   *queue
 
 	nextIdx atomic.Uint64
 
 	totalSizeBytes atomic.Uint64
 }
 
+func NewLabelsCompressor() *LabelsCompressor {
+	return &LabelsCompressor{
+		freeIdxs: newQueue(),
+	}
+}
+
 // SizeBytes returns the size of lc data in bytes
 func (lc *LabelsCompressor) SizeBytes() uint64 {
 	return uint64(unsafe.Sizeof(*lc)) + lc.totalSizeBytes.Load()
@@ -28,13 +89,60 @@ func (lc *LabelsCompressor) SizeBytes() uint64 {
 
 // ItemsCount returns the number of items in lc
 func (lc *LabelsCompressor) ItemsCount() uint64 {
-	return lc.nextIdx.Load()
+	return lc.nextIdx.Load() - uint64(len(lc.freeIdxs.storage))
+}
+
+// Delete adds stale labels idx to lc.freeIdxs list
+func (lc *LabelsCompressor) Delete(src []byte, ts int64) {
+	labelsLen, nSize := encoding.UnmarshalVarUint64(src)
+	if nSize <= 0 {
+		logger.Panicf("BUG: cannot unmarshal labels length from uvarint")
+	}
+	tail := src[nSize:]
+	if labelsLen == 0 {
+		// fast path - nothing to decode
+		if len(tail) > 0 {
+			logger.Panicf("BUG: unexpected non-empty tail left; len(tail)=%d; tail=%X", len(tail), tail)
+		}
+		return
+	}
+
+	a := encoding.GetUint64s(int(labelsLen))
+	var err error
+	tail, err = encoding.UnmarshalVarUint64s(a.A, tail)
+	if err != nil {
+		logger.Panicf("BUG: cannot unmarshal label indexes: %s", err)
+	}
+	if len(tail) > 0 {
+		logger.Panicf("BUG: unexpected non-empty tail left: len(tail)=%d; tail=%X", len(tail), tail)
+	}
+	for _, idx := range a.A {
+		label, ok := lc.idxToLabel.Load(idx)
+		if !ok {
+			logger.Panicf("BUG: missing label for idx=%d", idx)
+		}
+		v, ok := lc.labelToIdx.Load(label)
+		if !ok {
+			continue
+		}
+		cl := v.(compressedLabel)
+		cl.mu.Lock()
+		if cl.deleteDeadline < ts {
+			cl.deleted = true
+			cl.mu.Unlock()
+			lc.freeIdxs.in <- idx
+		} else {
+			cl.mu.Unlock()
+		}
+
+	}
+	encoding.PutUint64s(a)
 }
 
 // Compress compresses labels, appends the compressed labels to dst and returns the result.
 //
 // It is safe calling Compress from concurrent goroutines.
-func (lc *LabelsCompressor) Compress(dst []byte, labels []prompbmarshal.Label) []byte {
+func (lc *LabelsCompressor) Compress(dst []byte, labels []prompbmarshal.Label, deleteDeadline int64) []byte {
 	if len(labels) == 0 {
 		// Fast path
 		return append(dst, 0)
@@ -42,22 +150,27 @@ func (lc *LabelsCompressor) Compress(dst []byte, labels []prompbmarshal.Label) [
 
 	a := encoding.GetUint64s(len(labels) + 1)
 	a.A[0] = uint64(len(labels))
-	lc.compress(a.A[1:], labels)
+	lc.compress(a.A[1:], labels, deleteDeadline)
 	dst = encoding.MarshalVarUint64s(dst, a.A)
 	encoding.PutUint64s(a)
 	return dst
 }
 
-func (lc *LabelsCompressor) compress(dst []uint64, labels []prompbmarshal.Label) {
+func (lc *LabelsCompressor) compress(dst []uint64, labels []prompbmarshal.Label, deleteDeadline int64) {
 	if len(labels) == 0 {
 		return
 	}
 	_ = dst[len(labels)-1]
 	for i, label := range labels {
+	again:
 		v, ok := lc.labelToIdx.Load(label)
 		if !ok {
-			idx := lc.nextIdx.Add(1)
-			v = idx
+			var idx uint64
+			select {
+			case idx = <-lc.freeIdxs.out:
+			default:
+				idx = lc.nextIdx.Add(1)
+			}
 			labelCopy := cloneLabel(label)
 
 			// Must store idxToLabel entry before labelToIdx,
@@ -66,6 +179,10 @@ func (lc *LabelsCompressor) compress(dst []uint64, labels []prompbmarshal.Label)
 			// We might store duplicated entries for single label with different indexes,
 			// and it's fine, see https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7118.
 			lc.idxToLabel.Store(idx, labelCopy)
+			v = &compressedLabel{
+				deleteDeadline: deleteDeadline,
+				code:           idx,
+			}
 			vNew, loaded := lc.labelToIdx.LoadOrStore(labelCopy, v)
 			if loaded {
 				// This label has been stored by a concurrent goroutine with different index,
@@ -78,7 +195,17 @@ func (lc *LabelsCompressor) compress(dst []uint64, labels []prompbmarshal.Label)
 			entrySizeBytes := labelSizeBytes + uint64(2*(unsafe.Sizeof(label)+unsafe.Sizeof(&label))+unsafe.Sizeof(v))
 			lc.totalSizeBytes.Add(entrySizeBytes)
 		}
-		dst[i] = v.(uint64)
+		cl := v.(*compressedLabel)
+		dst[i] = cl.code
+		cl.mu.Lock()
+		deleted := cl.deleted
+		if !deleted {
+			cl.deleteDeadline = deleteDeadline
+		}
+		cl.mu.Unlock()
+		if deleted {
+			goto again
+		}
 	}
 }
 

lib/promutils/labelscompressor_test.go

@@ -9,14 +9,14 @@ import (
 )
 
 func TestLabelsCompressorSerial(t *testing.T) {
-	var lc LabelsCompressor
+	lc := NewLabelsCompressor()
 
 	f := func(labels []prompbmarshal.Label) {
 		t.Helper()
 
 		sExpected := labelsToString(labels)
 
-		data := lc.Compress(nil, labels)
+		data := lc.Compress(nil, labels, 0)
 		labelsResult := lc.Decompress(nil, data)
 
 		sResult := labelsToString(labelsResult)
@@ -67,7 +67,7 @@ func TestLabelsCompressorSerial(t *testing.T) {
 
 func TestLabelsCompressorConcurrent(t *testing.T) {
 	const concurrency = 5
-	var lc LabelsCompressor
+	lc := NewLabelsCompressor()
 	var expectCompressedKeys sync.Map
 
 	var wg sync.WaitGroup
@@ -78,7 +78,7 @@ func TestLabelsCompressorConcurrent(t *testing.T) {
 			series := newTestSeries(100, 20)
 			for n, labels := range series {
 				sExpected := labelsToString(labels)
-				data := lc.Compress(nil, labels)
+				data := lc.Compress(nil, labels, 0)
 				if expectData, ok := expectCompressedKeys.LoadOrStore(n, data); ok {
 					if string(data) != string(expectData.([]byte)) {
 						panic(fmt.Errorf("unexpected compress result at series/%d in iteration %d ", n, i))

lib/promutils/labelscompressor_timing_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func BenchmarkLabelsCompressorCompress(b *testing.B) {
-	var lc LabelsCompressor
+	lc := NewLabelsCompressor()
 	series := newTestSeries(100, 10)
 
 	b.ReportAllocs()
@@ -19,7 +19,7 @@ func BenchmarkLabelsCompressorCompress(b *testing.B) {
 		for pb.Next() {
 			dst = dst[:0]
 			for _, labels := range series {
-				dst = lc.Compress(dst, labels)
+				dst = lc.Compress(dst, labels, 0)
 			}
 			Sink.Add(uint64(len(dst)))
 		}
@@ -27,13 +27,13 @@ func BenchmarkLabelsCompressorCompress(b *testing.B) {
 }
 
 func BenchmarkLabelsCompressorDecompress(b *testing.B) {
-	var lc LabelsCompressor
+	lc := NewLabelsCompressor()
 	series := newTestSeries(100, 10)
 	datas := make([][]byte, len(series))
 	var dst []byte
 	for i, labels := range series {
 		dstLen := len(dst)
-		dst = lc.Compress(dst, labels)
+		dst = lc.Compress(dst, labels, 0)
 		datas[i] = dst[dstLen:]
 	}
 

lib/streamaggr/avg.go

@@ -12,37 +12,33 @@ type avgAggrState struct {
 }
 
 type avgStateValue struct {
-	mu      sync.Mutex
-	sum     float64
-	count   int64
-	deleted bool
+	mu             sync.Mutex
+	sum            float64
+	count          int64
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newAvgAggrState() *avgAggrState {
 	return &avgAggrState{}
 }
 
-func (as *avgAggrState) pushSamples(samples []pushSample) {
+func (as *avgAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
-			v = &avgStateValue{
-				sum:   s.value,
-				count: 1,
-			}
+			v = &avgStateValue{}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The entry has been successfully stored
-				continue
+			if loaded {
+				// Update the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Update the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*avgStateValue)
 		sv.mu.Lock()
@@ -50,6 +46,7 @@ func (as *avgAggrState) pushSamples(samples []pushSample) {
 		if !deleted {
 			sv.sum += s.value
 			sv.count++
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -68,9 +65,22 @@ func (as *avgAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*avgStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
+		if sv.count == 0 {
+			sv.mu.Unlock()
+			return true
+		}
+
 		avg := sv.sum / float64(sv.count)
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.sum = 0
+		sv.count = 0
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/count_samples.go

@@ -12,41 +12,39 @@ type countSamplesAggrState struct {
 }
 
 type countSamplesStateValue struct {
-	mu      sync.Mutex
-	n       uint64
-	deleted bool
+	mu             sync.Mutex
+	n              uint64
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newCountSamplesAggrState() *countSamplesAggrState {
 	return &countSamplesAggrState{}
 }
 
-func (as *countSamplesAggrState) pushSamples(samples []pushSample) {
+func (as *countSamplesAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
-			v = &countSamplesStateValue{
-				n: 1,
-			}
+			v = &countSamplesStateValue{}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The new entry has been successfully created.
-				continue
+			if loaded {
+				// Use the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Use the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*countSamplesStateValue)
 		sv.mu.Lock()
 		deleted := sv.deleted
 		if !deleted {
 			sv.n++
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -65,9 +63,21 @@ func (as *countSamplesAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*countSamplesStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
 		n := sv.n
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		if n == 0 {
+			sv.mu.Unlock()
+			return true
+		}
+
+		sv.n = 0
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/count_series.go

@@ -13,16 +13,17 @@ type countSeriesAggrState struct {
 }
 
 type countSeriesStateValue struct {
-	mu      sync.Mutex
-	m       map[uint64]struct{}
-	deleted bool
+	mu             sync.Mutex
+	m              map[uint64]struct{}
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newCountSeriesAggrState() *countSeriesAggrState {
 	return &countSeriesAggrState{}
 }
 
-func (as *countSeriesAggrState) pushSamples(samples []pushSample) {
+func (as *countSeriesAggrState) pushSamples(samples []pushSample, deleteDeadline int64, _ bool) {
 	for i := range samples {
 		s := &samples[i]
 		inputKey, outputKey := getInputOutputKey(s.key)
@@ -36,18 +37,14 @@ func (as *countSeriesAggrState) pushSamples(samples []pushSample) {
 		if !ok {
 			// The entry is missing in the map. Try creating it.
 			v = &countSeriesStateValue{
-				m: map[uint64]struct{}{
-					h: {},
-				},
+				m: make(map[uint64]struct{}),
 			}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The entry has been added to the map.
-				continue
+			if loaded {
+				// Update the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Update the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*countSeriesStateValue)
 		sv.mu.Lock()
@@ -56,6 +53,7 @@ func (as *countSeriesAggrState) pushSamples(samples []pushSample) {
 			if _, ok := sv.m[h]; !ok {
 				sv.m[h] = struct{}{}
 			}
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -74,9 +72,21 @@ func (as *countSeriesAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*countSeriesStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
 		n := len(sv.m)
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		if n == 0 {
+			sv.mu.Unlock()
+			return true
+		}
+
+		sv.m = make(map[uint64]struct{})
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/dedup.go

@@ -8,12 +8,14 @@ import (
 	"github.com/cespare/xxhash/v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 )
 
 const dedupAggrShardsCount = 128
 
 type dedupAggr struct {
+	lc     *promutils.LabelsCompressor
 	shards []dedupAggrShard
 }
 
@@ -26,7 +28,7 @@ type dedupAggrShard struct {
 }
 
 type dedupAggrShardNopad struct {
-	mu sync.Mutex
+	mu sync.RWMutex
 	m  map[string]*dedupAggrSample
 
 	samplesBuf []dedupAggrSample
@@ -36,14 +38,16 @@ type dedupAggrShardNopad struct {
 }
 
 type dedupAggrSample struct {
-	value     float64
-	timestamp int64
+	value          float64
+	timestamp      int64
+	deleteDeadline int64
 }
 
-func newDedupAggr() *dedupAggr {
+func newDedupAggr(lc *promutils.LabelsCompressor) *dedupAggr {
 	shards := make([]dedupAggrShard, dedupAggrShardsCount)
 	return &dedupAggr{
 		shards: shards,
+		lc:     lc,
 	}
 }
 
@@ -63,7 +67,7 @@ func (da *dedupAggr) itemsCount() uint64 {
 	return n
 }
 
-func (da *dedupAggr) pushSamples(samples []pushSample) {
+func (da *dedupAggr) pushSamples(samples []pushSample, deleteDeadline int64) {
 	pss := getPerShardSamples()
 	shards := pss.shards
 	for _, sample := range samples {
@@ -75,17 +79,21 @@ func (da *dedupAggr) pushSamples(samples []pushSample) {
 		if len(shardSamples) == 0 {
 			continue
 		}
-		da.shards[i].pushSamples(shardSamples)
+		da.shards[i].pushSamples(shardSamples, deleteDeadline)
 	}
 	putPerShardSamples(pss)
 }
 
-func getDedupFlushCtx() *dedupFlushCtx {
+func getDedupFlushCtx(flushTimestamp, deleteDeadline int64, lc *promutils.LabelsCompressor) *dedupFlushCtx {
 	v := dedupFlushCtxPool.Get()
 	if v == nil {
-		return &dedupFlushCtx{}
+		v = &dedupFlushCtx{}
 	}
-	return v.(*dedupFlushCtx)
+	ctx := v.(*dedupFlushCtx)
+	ctx.lc = lc
+	ctx.deleteDeadline = deleteDeadline
+	ctx.flushTimestamp = flushTimestamp
+	return ctx
 }
 
 func putDedupFlushCtx(ctx *dedupFlushCtx) {
@@ -96,15 +104,24 @@ func putDedupFlushCtx(ctx *dedupFlushCtx) {
 var dedupFlushCtxPool sync.Pool
 
 type dedupFlushCtx struct {
-	samples []pushSample
+	keysToDelete   []string
+	samples        []pushSample
+	deleteDeadline int64
+	flushTimestamp int64
+	lc             *promutils.LabelsCompressor
 }
 
 func (ctx *dedupFlushCtx) reset() {
+	ctx.deleteDeadline = 0
+	ctx.flushTimestamp = 0
+	ctx.lc = nil
+	clear(ctx.keysToDelete)
+	ctx.keysToDelete = ctx.keysToDelete[:0]
 	clear(ctx.samples)
 	ctx.samples = ctx.samples[:0]
 }
 
-func (da *dedupAggr) flush(f func(samples []pushSample)) {
+func (da *dedupAggr) flush(f func(samples []pushSample, deleteDeadline int64), flushTimestamp, deleteDeadline int64) {
 	var wg sync.WaitGroup
 	for i := range da.shards {
 		flushConcurrencyCh <- struct{}{}
@@ -115,7 +132,7 @@ func (da *dedupAggr) flush(f func(samples []pushSample)) {
 				wg.Done()
 			}()
 
-			ctx := getDedupFlushCtx()
+			ctx := getDedupFlushCtx(flushTimestamp, deleteDeadline, da.lc)
 			shard.flush(ctx, f)
 			putDedupFlushCtx(ctx)
 		}(&da.shards[i])
@@ -154,7 +171,7 @@ func putPerShardSamples(pss *perShardSamples) {
 
 var perShardSamplesPool sync.Pool
 
-func (das *dedupAggrShard) pushSamples(samples []pushSample) {
+func (das *dedupAggrShard) pushSamples(samples []pushSample, deleteDeadline int64) {
 	das.mu.Lock()
 	defer das.mu.Unlock()
 
@@ -171,6 +188,7 @@ func (das *dedupAggrShard) pushSamples(samples []pushSample) {
 			s = &samplesBuf[len(samplesBuf)-1]
 			s.value = sample.value
 			s.timestamp = sample.timestamp
+			s.deleteDeadline = deleteDeadline
 
 			key := bytesutil.InternString(sample.key)
 			m[key] = s
@@ -183,30 +201,28 @@ func (das *dedupAggrShard) pushSamples(samples []pushSample) {
 		if sample.timestamp > s.timestamp || (sample.timestamp == s.timestamp && sample.value > s.value) {
 			s.value = sample.value
 			s.timestamp = sample.timestamp
+			s.deleteDeadline = deleteDeadline
 		}
 	}
 	das.samplesBuf = samplesBuf
 }
 
-func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f func(samples []pushSample)) {
-	das.mu.Lock()
-
-	m := das.m
-	if len(m) > 0 {
-		das.m = make(map[string]*dedupAggrSample, len(m))
-		das.sizeBytes.Store(0)
-		das.itemsCount.Store(0)
-		das.samplesBuf = make([]dedupAggrSample, 0, len(das.samplesBuf))
-	}
-
-	das.mu.Unlock()
-
-	if len(m) == 0 {
+func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f func(samples []pushSample, deleteDeadline int64)) {
+	if len(das.m) == 0 {
 		return
 	}
 
+	keysToDelete := ctx.keysToDelete
 	dstSamples := ctx.samples
-	for key, s := range m {
+	das.mu.RLock()
+	for key, s := range das.m {
+		if ctx.flushTimestamp > s.deleteDeadline {
+			das.itemsCount.Add(^uint64(0))
+			//ctx.lc.Delete(key)
+			das.sizeBytes.Add(^(uint64(len(key)) + uint64(unsafe.Sizeof(key)+unsafe.Sizeof(s)+unsafe.Sizeof(*s)) - 1))
+			keysToDelete = append(keysToDelete, key)
+			continue
+		}
 		dstSamples = append(dstSamples, pushSample{
 			key:       key,
 			value:     s.value,
@@ -215,11 +231,17 @@ func (das *dedupAggrShard) flush(ctx *dedupFlushCtx, f func(samples []pushSample
 
 		// Limit the number of samples per each flush in order to limit memory usage.
 		if len(dstSamples) >= 10_000 {
-			f(dstSamples)
+			f(dstSamples, ctx.deleteDeadline)
 			clear(dstSamples)
 			dstSamples = dstSamples[:0]
 		}
 	}
-	f(dstSamples)
+	das.mu.RUnlock()
+	das.mu.Lock()
+	for _, key := range keysToDelete {
+		delete(das.m, key)
+	}
+	das.mu.Unlock()
+	f(dstSamples, ctx.deleteDeadline)
 	ctx.samples = dstSamples
 }

lib/streamaggr/dedup_test.go

@@ -5,10 +5,13 @@ import (
 	"reflect"
 	"sync"
 	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func TestDedupAggrSerial(t *testing.T) {
-	da := newDedupAggr()
+	var lc promutils.LabelsCompressor
+	da := newDedupAggr(&lc)
 
 	const seriesCount = 100_000
 	expectedSamplesMap := make(map[string]pushSample)
@@ -20,11 +23,11 @@ func TestDedupAggrSerial(t *testing.T) {
 			sample.value = float64(i + j)
 			expectedSamplesMap[sample.key] = *sample
 		}
-		da.pushSamples(samples)
+		da.pushSamples(samples, 0)
 	}
 
-	if n := da.sizeBytes(); n > 5_000_000 {
-		t.Fatalf("too big dedupAggr state before flush: %d bytes; it shouldn't exceed 5_000_000 bytes", n)
+	if n := da.sizeBytes(); n > 6_000_000 {
+		t.Fatalf("too big dedupAggr state before flush: %d bytes; it shouldn't exceed 6_000_000 bytes", n)
 	}
 	if n := da.itemsCount(); n != seriesCount {
 		t.Fatalf("unexpected itemsCount; got %d; want %d", n, seriesCount)
@@ -32,19 +35,21 @@ func TestDedupAggrSerial(t *testing.T) {
 
 	flushedSamplesMap := make(map[string]pushSample)
 	var mu sync.Mutex
-	flushSamples := func(samples []pushSample) {
+	flushSamples := func(samples []pushSample, _ int64) {
 		mu.Lock()
 		for _, sample := range samples {
 			flushedSamplesMap[sample.key] = sample
 		}
 		mu.Unlock()
 	}
-	da.flush(flushSamples)
+	da.flush(flushSamples, 0, 0)
 
 	if !reflect.DeepEqual(expectedSamplesMap, flushedSamplesMap) {
 		t.Fatalf("unexpected samples;\ngot\n%v\nwant\n%v", flushedSamplesMap, expectedSamplesMap)
 	}
 
+	da.flush(flushSamples, 1, 0)
+
 	if n := da.sizeBytes(); n > 17_000 {
 		t.Fatalf("too big dedupAggr state after flush; %d bytes; it shouldn't exceed 17_000 bytes", n)
 	}
@@ -54,9 +59,10 @@ func TestDedupAggrSerial(t *testing.T) {
 }
 
 func TestDedupAggrConcurrent(_ *testing.T) {
+	var lc promutils.LabelsCompressor
 	const concurrency = 5
 	const seriesCount = 10_000
-	da := newDedupAggr()
+	da := newDedupAggr(&lc)
 
 	var wg sync.WaitGroup
 	for i := 0; i < concurrency; i++ {
@@ -70,7 +76,7 @@ func TestDedupAggrConcurrent(_ *testing.T) {
 					sample.key = fmt.Sprintf("key_%d", j)
 					sample.value = float64(i + j)
 				}
-				da.pushSamples(samples)
+				da.pushSamples(samples, 0)
 			}
 		}()
 	}

lib/streamaggr/dedup_timing_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
 )
 
 func BenchmarkDedupAggr(b *testing.B) {
@@ -17,9 +18,11 @@ func BenchmarkDedupAggr(b *testing.B) {
 }
 
 func benchmarkDedupAggr(b *testing.B, samplesPerPush int) {
+	var lc promutils.LabelsCompressor
 	const loops = 2
 	benchSamples := newBenchSamples(samplesPerPush)
-	da := newDedupAggr()
+
+	da := newDedupAggr(&lc)
 
 	b.ResetTimer()
 	b.ReportAllocs()
@@ -27,13 +30,14 @@ func benchmarkDedupAggr(b *testing.B, samplesPerPush int) {
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			for i := 0; i < loops; i++ {
-				da.pushSamples(benchSamples)
+				da.pushSamples(benchSamples, 0)
 			}
 		}
 	})
 }
 
 func newBenchSamples(count int) []pushSample {
+	var lc promutils.LabelsCompressor
 	labels := []prompbmarshal.Label{
 		{
 			Name:  "app",
@@ -65,7 +69,7 @@ func newBenchSamples(count int) []pushSample {
 			Name:  "app",
 			Value: fmt.Sprintf("instance-%d", i),
 		})
-		keyBuf = compressLabels(keyBuf[:0], labels[:labelsLen], labels[labelsLen:])
+		keyBuf = compressLabels(keyBuf[:0], &lc, labels[:labelsLen], labels[labelsLen:], false, 0)
 		sample.key = string(keyBuf)
 		sample.value = float64(i)
 	}

lib/streamaggr/deduplicator.go

@@ -16,8 +16,10 @@ import (
 // Deduplicator deduplicates samples per each time series.
 type Deduplicator struct {
 	da *dedupAggr
+	lc *promutils.LabelsCompressor
 
-	dropLabels []string
+	dropLabels        []string
+	stalenessInterval int64
 
 	wg     sync.WaitGroup
 	stopCh chan struct{}
@@ -40,13 +42,16 @@ type Deduplicator struct {
 // MustStop must be called on the returned deduplicator in order to free up occupied resources.
 func NewDeduplicator(pushFunc PushFunc, dedupInterval time.Duration, dropLabels []string, alias string) *Deduplicator {
 	d := &Deduplicator{
-		da:         newDedupAggr(),
-		dropLabels: dropLabels,
+		dropLabels:        dropLabels,
+		stalenessInterval: 2 * dedupInterval.Milliseconds(),
+		lc:                promutils.NewLabelsCompressor(),
 
 		stopCh: make(chan struct{}),
 		ms:     metrics.NewSet(),
 	}
 
+	d.da = newDedupAggr(d.lc)
+
 	ms := d.ms
 
 	metricLabels := fmt.Sprintf(`name="dedup",url=%q`, alias)
@@ -57,6 +62,12 @@ func NewDeduplicator(pushFunc PushFunc, dedupInterval time.Duration, dropLabels
 	_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_dedup_state_items_count{%s}`, metricLabels), func() float64 {
 		return float64(d.da.itemsCount())
 	})
+	_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_labels_compressor_size_bytes{%s}`, metricLabels), func() float64 {
+		return float64(d.lc.SizeBytes())
+	})
+	_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_labels_compressor_items_count{%s}`, metricLabels), func() float64 {
+		return float64(d.lc.ItemsCount())
+	})
 
 	d.dedupFlushDuration = ms.NewHistogram(fmt.Sprintf(`vm_streamaggr_dedup_flush_duration_seconds{%s}`, metricLabels))
 	d.dedupFlushTimeouts = ms.NewCounter(fmt.Sprintf(`vm_streamaggr_dedup_flush_timeouts_total{%s}`, metricLabels))
@@ -87,6 +98,7 @@ func (d *Deduplicator) Push(tss []prompbmarshal.TimeSeries) {
 	pss := ctx.pss
 	labels := &ctx.labels
 	buf := ctx.buf
+	deleteDeadline := time.Now().UnixMilli() + d.stalenessInterval
 
 	dropLabels := d.dropLabels
 	for _, ts := range tss {
@@ -101,7 +113,7 @@ func (d *Deduplicator) Push(tss []prompbmarshal.TimeSeries) {
 		labels.Sort()
 
 		bufLen := len(buf)
-		buf = lc.Compress(buf, labels.Labels)
+		buf = d.lc.Compress(buf, labels.Labels, deleteDeadline)
 		key := bytesutil.ToUnsafeString(buf[bufLen:])
 		for _, s := range ts.Samples {
 			pss = append(pss, pushSample{
@@ -112,7 +124,7 @@ func (d *Deduplicator) Push(tss []prompbmarshal.TimeSeries) {
 		}
 	}
 
-	d.da.pushSamples(pss)
+	d.da.pushSamples(pss, deleteDeadline)
 
 	ctx.pss = pss
 	ctx.buf = buf
@@ -145,7 +157,8 @@ func (d *Deduplicator) flush(pushFunc PushFunc, dedupInterval time.Duration) {
 	startTime := time.Now()
 
 	timestamp := startTime.UnixMilli()
-	d.da.flush(func(pss []pushSample) {
+	deleteDeadline := timestamp + d.stalenessInterval
+	d.da.flush(func(pss []pushSample, deleteDeadline int64) {
 		ctx := getDeduplicatorFlushCtx()
 
 		tss := ctx.tss
@@ -153,7 +166,7 @@ func (d *Deduplicator) flush(pushFunc PushFunc, dedupInterval time.Duration) {
 		samples := ctx.samples
 		for _, ps := range pss {
 			labelsLen := len(labels)
-			labels = decompressLabels(labels, ps.key)
+			labels = decompressLabels(labels, d.lc, ps.key)
 
 			samplesLen := len(samples)
 			samples = append(samples, prompbmarshal.Sample{
@@ -172,7 +185,7 @@ func (d *Deduplicator) flush(pushFunc PushFunc, dedupInterval time.Duration) {
 		ctx.labels = labels
 		ctx.samples = samples
 		putDeduplicatorFlushCtx(ctx)
-	})
+	}, timestamp, deleteDeadline)
 
 	duration := time.Since(startTime)
 	d.dedupFlushDuration.Update(duration.Seconds())

lib/streamaggr/histogram_bucket.go

@@ -1,42 +1,32 @@
 package streamaggr
 
 import (
-	"math"
 	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/metrics"
 )
 
 // histogramBucketAggrState calculates output=histogram_bucket, e.g. VictoriaMetrics histogram over input samples.
 type histogramBucketAggrState struct {
 	m sync.Map
-
-	stalenessSecs uint64
 }
 
 type histogramBucketStateValue struct {
 	mu             sync.Mutex
 	h              metrics.Histogram
-	deleteDeadline uint64
+	deleteDeadline int64
 	deleted        bool
 }
 
-func newHistogramBucketAggrState(stalenessInterval time.Duration) *histogramBucketAggrState {
-	stalenessSecs := roundDurationToSecs(stalenessInterval)
-	return &histogramBucketAggrState{
-		stalenessSecs: stalenessSecs,
-	}
+func newHistogramBucketAggrState() *histogramBucketAggrState {
+	return &histogramBucketAggrState{}
 }
 
-func (as *histogramBucketAggrState) pushSamples(samples []pushSample) {
-	currentTime := fasttime.UnixTimestamp()
-	deleteDeadline := currentTime + as.stalenessSecs
+func (as *histogramBucketAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
@@ -66,13 +56,13 @@ func (as *histogramBucketAggrState) pushSamples(samples []pushSample) {
 	}
 }
 
-func (as *histogramBucketAggrState) removeOldEntries(currentTime uint64) {
+func (as *histogramBucketAggrState) removeOldEntries(ctx *flushCtx) {
 	m := &as.m
 	m.Range(func(k, v any) bool {
 		sv := v.(*histogramBucketStateValue)
 
 		sv.mu.Lock()
-		deleted := currentTime > sv.deleteDeadline
+		deleted := ctx.flushTimestamp > sv.deleteDeadline
 		if deleted {
 			// Mark the current entry as deleted
 			sv.deleted = deleted
@@ -80,6 +70,8 @@ func (as *histogramBucketAggrState) removeOldEntries(currentTime uint64) {
 		sv.mu.Unlock()
 
 		if deleted {
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
 			m.Delete(k)
 		}
 		return true
@@ -87,9 +79,7 @@ func (as *histogramBucketAggrState) removeOldEntries(currentTime uint64) {
 }
 
 func (as *histogramBucketAggrState) flushState(ctx *flushCtx) {
-	currentTime := fasttime.UnixTimestamp()
-
-	as.removeOldEntries(currentTime)
+	as.removeOldEntries(ctx)
 
 	m := &as.m
 	m.Range(func(k, v any) bool {
@@ -105,11 +95,3 @@ func (as *histogramBucketAggrState) flushState(ctx *flushCtx) {
 		return true
 	})
 }
-
-func roundDurationToSecs(d time.Duration) uint64 {
-	if d < 0 {
-		return 0
-	}
-	secs := d.Seconds()
-	return uint64(math.Ceil(secs))
-}

lib/streamaggr/last.go

@@ -12,46 +12,45 @@ type lastAggrState struct {
 }
 
 type lastStateValue struct {
-	mu        sync.Mutex
-	last      float64
-	timestamp int64
-	deleted   bool
+	mu             sync.Mutex
+	last           float64
+	timestamp      int64
+	deleted        bool
+	defined        bool
+	deleteDeadline int64
 }
 
 func newLastAggrState() *lastAggrState {
 	return &lastAggrState{}
 }
 
-func (as *lastAggrState) pushSamples(samples []pushSample) {
+func (as *lastAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
-			v = &lastStateValue{
-				last:      s.value,
-				timestamp: s.timestamp,
-			}
+			v = &lastStateValue{}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The new entry has been successfully created.
-				continue
+			if loaded {
+				// Use the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Use the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*lastStateValue)
 		sv.mu.Lock()
 		deleted := sv.deleted
 		if !deleted {
-			if s.timestamp >= sv.timestamp {
+			if !sv.defined || s.timestamp >= sv.timestamp {
 				sv.last = s.value
 				sv.timestamp = s.timestamp
+				sv.deleteDeadline = deleteDeadline
 			}
+			sv.defined = true
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -70,6 +69,14 @@ func (as *lastAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*lastStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
 		last := sv.last
 		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
 		sv.deleted = true

lib/streamaggr/max.go

@@ -12,35 +12,33 @@ type maxAggrState struct {
 }
 
 type maxStateValue struct {
-	mu      sync.Mutex
-	max     float64
-	deleted bool
+	mu             sync.Mutex
+	max            float64
+	deleted        bool
+	defined        bool
+	deleteDeadline int64
 }
 
 func newMaxAggrState() *maxAggrState {
 	return &maxAggrState{}
 }
 
-func (as *maxAggrState) pushSamples(samples []pushSample) {
+func (as *maxAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
-			v = &maxStateValue{
-				max: s.value,
-			}
+			v = &maxStateValue{}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The new entry has been successfully created.
-				continue
+			if loaded {
+				// Use the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Use the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*maxStateValue)
 		sv.mu.Lock()
@@ -49,6 +47,10 @@ func (as *maxAggrState) pushSamples(samples []pushSample) {
 			if s.value > sv.max {
 				sv.max = s.value
 			}
+			if !sv.defined {
+				sv.defined = true
+			}
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -67,9 +69,21 @@ func (as *maxAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*maxStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
+		if !sv.defined {
+			sv.mu.Unlock()
+			return true
+		}
 		max := sv.max
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.max = 0
+		sv.defined = false
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/min.go

@@ -12,43 +12,45 @@ type minAggrState struct {
 }
 
 type minStateValue struct {
-	mu      sync.Mutex
-	min     float64
-	deleted bool
+	mu             sync.Mutex
+	min            float64
+	deleted        bool
+	defined        bool
+	deleteDeadline int64
 }
 
 func newMinAggrState() *minAggrState {
 	return &minAggrState{}
 }
 
-func (as *minAggrState) pushSamples(samples []pushSample) {
+func (as *minAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
-			v = &minStateValue{
-				min: s.value,
-			}
+			v = &minStateValue{}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The new entry has been successfully created.
-				continue
+			if loaded {
+				// Use the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Use the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*minStateValue)
 		sv.mu.Lock()
 		deleted := sv.deleted
 		if !deleted {
-			if s.value < sv.min {
+			if !sv.defined || s.value < sv.min {
 				sv.min = s.value
 			}
+			sv.deleteDeadline = deleteDeadline
+			if !sv.defined {
+				sv.defined = true
+			}
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -67,10 +69,23 @@ func (as *minAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*minStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
+		if !sv.defined {
+			sv.mu.Unlock()
+			return true
+		}
 		min := sv.min
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.min = 0
+		sv.defined = false
 		sv.mu.Unlock()
+
 		key := k.(string)
 		ctx.appendSeries(key, "min", min)
 		return true

lib/streamaggr/quantiles.go

@@ -16,9 +16,10 @@ type quantilesAggrState struct {
 }
 
 type quantilesStateValue struct {
-	mu      sync.Mutex
-	h       *histogram.Fast
-	deleted bool
+	mu             sync.Mutex
+	h              *histogram.Fast
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newQuantilesAggrState(phis []float64) *quantilesAggrState {
@@ -27,10 +28,10 @@ func newQuantilesAggrState(phis []float64) *quantilesAggrState {
 	}
 }
 
-func (as *quantilesAggrState) pushSamples(samples []pushSample) {
+func (as *quantilesAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
@@ -53,6 +54,7 @@ func (as *quantilesAggrState) pushSamples(samples []pushSample) {
 		deleted := sv.deleted
 		if !deleted {
 			sv.h.Update(s.value)
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -74,10 +76,16 @@ func (as *quantilesAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*quantilesStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
 		quantiles = sv.h.Quantiles(quantiles[:0], phis)
-		histogram.PutFast(sv.h)
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.h.Reset()
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/rate.go

@@ -2,10 +2,8 @@ package streamaggr
 
 import (
 	"sync"
-	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 )
 
 // rateAggrState calculates output=rate_avg and rate_sum, e.g. the average per-second increase rate for counter metrics.
@@ -14,22 +12,19 @@ type rateAggrState struct {
 
 	// isAvg is set to true if rate_avg() must be calculated instead of rate_sum().
 	isAvg bool
-
-	// Time series state is dropped if no new samples are received during stalenessSecs.
-	stalenessSecs uint64
 }
 
 type rateStateValue struct {
 	mu             sync.Mutex
 	lastValues     map[string]rateLastValueState
-	deleteDeadline uint64
+	deleteDeadline int64
 	deleted        bool
 }
 
 type rateLastValueState struct {
 	value          float64
 	timestamp      int64
-	deleteDeadline uint64
+	deleteDeadline int64
 
 	// increase stores cumulative increase for the current time series on the current aggregation interval
 	increase float64
@@ -38,17 +33,13 @@ type rateLastValueState struct {
 	prevTimestamp int64
 }
 
-func newRateAggrState(stalenessInterval time.Duration, isAvg bool) *rateAggrState {
-	stalenessSecs := roundDurationToSecs(stalenessInterval)
+func newRateAggrState(isAvg bool) *rateAggrState {
 	return &rateAggrState{
-		isAvg:         isAvg,
-		stalenessSecs: stalenessSecs,
+		isAvg: isAvg,
 	}
 }
 
-func (as *rateAggrState) pushSamples(samples []pushSample) {
-	currentTime := fasttime.UnixTimestamp()
-	deleteDeadline := currentTime + as.stalenessSecs
+func (as *rateAggrState) pushSamples(samples []pushSample, deleteDeadline int64, _ bool) {
 	for i := range samples {
 		s := &samples[i]
 		inputKey, outputKey := getInputOutputKey(s.key)
@@ -106,11 +97,9 @@ func (as *rateAggrState) pushSamples(samples []pushSample) {
 }
 
 func (as *rateAggrState) flushState(ctx *flushCtx) {
-	currentTime := fasttime.UnixTimestamp()
-
 	suffix := as.getSuffix()
 
-	as.removeOldEntries(currentTime)
+	as.removeOldEntries(ctx)
 
 	m := &as.m
 	m.Range(func(k, v any) bool {
@@ -156,16 +145,18 @@ func (as *rateAggrState) getSuffix() string {
 	return "rate_sum"
 }
 
-func (as *rateAggrState) removeOldEntries(currentTime uint64) {
+func (as *rateAggrState) removeOldEntries(ctx *flushCtx) {
 	m := &as.m
 	m.Range(func(k, v any) bool {
 		sv := v.(*rateStateValue)
 
 		sv.mu.Lock()
-		if currentTime > sv.deleteDeadline {
+		if ctx.flushTimestamp > sv.deleteDeadline {
 			// Mark the current entry as deleted
 			sv.deleted = true
 			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
 			m.Delete(k)
 			return true
 		}
@@ -173,7 +164,7 @@ func (as *rateAggrState) removeOldEntries(currentTime uint64) {
 		// Delete outdated entries in sv.lastValues
 		lvs := sv.lastValues
 		for k1, lv := range lvs {
-			if currentTime > lv.deleteDeadline {
+			if ctx.flushTimestamp > lv.deleteDeadline {
 				delete(lvs, k1)
 			}
 		}

lib/streamaggr/stddev.go

@@ -13,21 +13,22 @@ type stddevAggrState struct {
 }
 
 type stddevStateValue struct {
-	mu      sync.Mutex
-	count   float64
-	avg     float64
-	q       float64
-	deleted bool
+	mu             sync.Mutex
+	count          float64
+	avg            float64
+	q              float64
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newStddevAggrState() *stddevAggrState {
 	return &stddevAggrState{}
 }
 
-func (as *stddevAggrState) pushSamples(samples []pushSample) {
+func (as *stddevAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
@@ -50,6 +51,7 @@ func (as *stddevAggrState) pushSamples(samples []pushSample) {
 			avg := sv.avg + (s.value-sv.avg)/sv.count
 			sv.q += (s.value - sv.avg) * (s.value - avg)
 			sv.avg = avg
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -68,9 +70,23 @@ func (as *stddevAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*stddevStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
+		if sv.count == 0 {
+			sv.mu.Unlock()
+			return true
+		}
+
 		stddev := math.Sqrt(sv.q / sv.count)
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.q = 0
+		sv.count = 0
+		sv.avg = 0
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/stdvar.go

@@ -12,21 +12,22 @@ type stdvarAggrState struct {
 }
 
 type stdvarStateValue struct {
-	mu      sync.Mutex
-	count   float64
-	avg     float64
-	q       float64
-	deleted bool
+	mu             sync.Mutex
+	count          float64
+	avg            float64
+	q              float64
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newStdvarAggrState() *stdvarAggrState {
 	return &stdvarAggrState{}
 }
 
-func (as *stdvarAggrState) pushSamples(samples []pushSample) {
+func (as *stdvarAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
@@ -49,6 +50,7 @@ func (as *stdvarAggrState) pushSamples(samples []pushSample) {
 			avg := sv.avg + (s.value-sv.avg)/sv.count
 			sv.q += (s.value - sv.avg) * (s.value - avg)
 			sv.avg = avg
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -67,9 +69,22 @@ func (as *stdvarAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*stdvarStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
+		if sv.count == 0 {
+			sv.mu.Unlock()
+			return true
+		}
 		stdvar := sv.q / sv.count
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.q = 0
+		sv.count = 0
+		sv.avg = 0
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/streamaggr.go

@@ -47,19 +47,6 @@ var supportedOutputs = []string{
 	"unique_samples",
 }
 
-var (
-	// lc contains information about all compressed labels for streaming aggregation
-	lc promutils.LabelsCompressor
-
-	_ = metrics.NewGauge(`vm_streamaggr_labels_compressor_size_bytes`, func() float64 {
-		return float64(lc.SizeBytes())
-	})
-
-	_ = metrics.NewGauge(`vm_streamaggr_labels_compressor_items_count`, func() float64 {
-		return float64(lc.ItemsCount())
-	})
-)
-
 // LoadFromFile loads Aggregators from the given path and uses the given pushFunc for pushing the aggregated data.
 //
 // opts can contain additional options. If opts is nil, then default options are used.
@@ -373,13 +360,15 @@ type aggregator struct {
 
 	keepMetricNames  bool
 	ignoreOldSamples bool
+	includeInputKey  bool
 
 	by                  []string
 	without             []string
 	aggregateOnlyByTime bool
 
 	// interval is the interval between flushes
-	interval time.Duration
+	interval          time.Duration
+	stalenessInterval int64
 
 	// dedupInterval is optional deduplication interval for incoming samples
 	dedupInterval time.Duration
@@ -390,6 +379,9 @@ type aggregator struct {
 	// aggrOutputs contains aggregate states for the given outputs
 	aggrOutputs []aggrOutput
 
+	// lc is used for compressing series keys before passing them to dedupAggr and aggrState
+	lc *promutils.LabelsCompressor
+
 	// minTimestamp is used for ignoring old samples when ignoreOldSamples is set
 	minTimestamp atomic.Int64
 
@@ -424,7 +416,7 @@ type aggrState interface {
 	// pushSamples must push samples to the aggrState.
 	//
 	// samples[].key must be cloned by aggrState, since it may change after returning from pushSamples.
-	pushSamples(samples []pushSample)
+	pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool)
 
 	// flushState must flush aggrState data to ctx.
 	flushState(ctx *flushCtx)
@@ -556,11 +548,15 @@ func newAggregator(cfg *Config, path string, pushFunc PushFunc, ms *metrics.Set,
 	}
 	aggrOutputs := make([]aggrOutput, len(cfg.Outputs))
 	outputsSeen := make(map[string]struct{}, len(cfg.Outputs))
+	includeInputKey := false
 	for i, output := range cfg.Outputs {
-		as, err := newAggrState(output, outputsSeen, stalenessInterval)
+		as, ik, err := newAggrState(output, outputsSeen)
 		if err != nil {
 			return nil, err
 		}
+		if ik {
+			includeInputKey = true
+		}
 		aggrOutputs[i] = aggrOutput{
 			as: as,
 
@@ -592,9 +588,11 @@ func newAggregator(cfg *Config, path string, pushFunc PushFunc, ms *metrics.Set,
 		by:                  by,
 		without:             without,
 		aggregateOnlyByTime: aggregateOnlyByTime,
+		lc:                  promutils.NewLabelsCompressor(),
 
-		interval:      interval,
-		dedupInterval: dedupInterval,
+		interval:          interval,
+		dedupInterval:     dedupInterval,
+		stalenessInterval: stalenessInterval.Milliseconds(),
 
 		aggrOutputs: aggrOutputs,
 
@@ -614,18 +612,25 @@ func newAggregator(cfg *Config, path string, pushFunc PushFunc, ms *metrics.Set,
 	}
 
 	if dedupInterval > 0 {
-		a.da = newDedupAggr()
+		includeInputKey = true
+		a.da = newDedupAggr(a.lc)
 
 		_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_dedup_state_size_bytes{%s}`, metricLabels), func() float64 {
-			n := a.da.sizeBytes()
-			return float64(n)
+			return float64(a.da.sizeBytes())
 		})
 		_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_dedup_state_items_count{%s}`, metricLabels), func() float64 {
-			n := a.da.itemsCount()
-			return float64(n)
+			return float64(a.da.itemsCount())
+		})
+		_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_labels_compressor_size_bytes{%s}`, metricLabels), func() float64 {
+			return float64(a.lc.SizeBytes())
+		})
+		_ = ms.NewGauge(fmt.Sprintf(`vm_streamaggr_labels_compressor_items_count{%s}`, metricLabels), func() float64 {
+			return float64(a.lc.ItemsCount())
 		})
 	}
 
+	a.includeInputKey = includeInputKey
+
 	alignFlushToInterval := !opts.NoAlignFlushToInterval
 	if v := cfg.NoAlignFlushToInterval; v != nil {
 		alignFlushToInterval = !*v
@@ -645,20 +650,20 @@ func newAggregator(cfg *Config, path string, pushFunc PushFunc, ms *metrics.Set,
 	return a, nil
 }
 
-func newAggrState(output string, outputsSeen map[string]struct{}, stalenessInterval time.Duration) (aggrState, error) {
+func newAggrState(output string, outputsSeen map[string]struct{}) (aggrState, bool, error) {
 	// check for duplicated output
 	if _, ok := outputsSeen[output]; ok {
-		return nil, fmt.Errorf("`outputs` list contains duplicate aggregation function: %s", output)
+		return nil, true, fmt.Errorf("`outputs` list contains duplicate aggregation function: %s", output)
 	}
 	outputsSeen[output] = struct{}{}
 
 	if strings.HasPrefix(output, "quantiles(") {
 		if !strings.HasSuffix(output, ")") {
-			return nil, fmt.Errorf("missing closing brace for `quantiles()` output")
+			return nil, false, fmt.Errorf("missing closing brace for `quantiles()` output")
 		}
 		argsStr := output[len("quantiles(") : len(output)-1]
 		if len(argsStr) == 0 {
-			return nil, fmt.Errorf("`quantiles()` must contain at least one phi")
+			return nil, false, fmt.Errorf("`quantiles()` must contain at least one phi")
 		}
 		args := strings.Split(argsStr, ",")
 		phis := make([]float64, len(args))
@@ -666,57 +671,57 @@ func newAggrState(output string, outputsSeen map[string]struct{}, stalenessInter
 			arg = strings.TrimSpace(arg)
 			phi, err := strconv.ParseFloat(arg, 64)
 			if err != nil {
-				return nil, fmt.Errorf("cannot parse phi=%q for quantiles(%s): %w", arg, argsStr, err)
+				return nil, false, fmt.Errorf("cannot parse phi=%q for quantiles(%s): %w", arg, argsStr, err)
 			}
 			if phi < 0 || phi > 1 {
-				return nil, fmt.Errorf("phi inside quantiles(%s) must be in the range [0..1]; got %v", argsStr, phi)
+				return nil, false, fmt.Errorf("phi inside quantiles(%s) must be in the range [0..1]; got %v", argsStr, phi)
 			}
 			phis[i] = phi
 		}
 		if _, ok := outputsSeen["quantiles"]; ok {
-			return nil, fmt.Errorf("`outputs` list contains duplicated `quantiles()` function, please combine multiple phi* like `quantiles(0.5, 0.9)`")
+			return nil, false, fmt.Errorf("`outputs` list contains duplicated `quantiles()` function, please combine multiple phi* like `quantiles(0.5, 0.9)`")
 		}
 		outputsSeen["quantiles"] = struct{}{}
-		return newQuantilesAggrState(phis), nil
+		return newQuantilesAggrState(phis), false, nil
 	}
 
 	switch output {
 	case "avg":
-		return newAvgAggrState(), nil
+		return newAvgAggrState(), false, nil
 	case "count_samples":
-		return newCountSamplesAggrState(), nil
+		return newCountSamplesAggrState(), false, nil
 	case "count_series":
-		return newCountSeriesAggrState(), nil
+		return newCountSeriesAggrState(), true, nil
 	case "histogram_bucket":
-		return newHistogramBucketAggrState(stalenessInterval), nil
+		return newHistogramBucketAggrState(), false, nil
 	case "increase":
-		return newTotalAggrState(stalenessInterval, true, true), nil
+		return newTotalAggrState(true, true), true, nil
 	case "increase_prometheus":
-		return newTotalAggrState(stalenessInterval, true, false), nil
+		return newTotalAggrState(true, false), true, nil
 	case "last":
-		return newLastAggrState(), nil
+		return newLastAggrState(), false, nil
 	case "max":
-		return newMaxAggrState(), nil
+		return newMaxAggrState(), false, nil
 	case "min":
-		return newMinAggrState(), nil
+		return newMinAggrState(), false, nil
 	case "rate_avg":
-		return newRateAggrState(stalenessInterval, true), nil
+		return newRateAggrState(true), true, nil
 	case "rate_sum":
-		return newRateAggrState(stalenessInterval, false), nil
+		return newRateAggrState(false), true, nil
 	case "stddev":
-		return newStddevAggrState(), nil
+		return newStddevAggrState(), false, nil
 	case "stdvar":
-		return newStdvarAggrState(), nil
+		return newStdvarAggrState(), false, nil
 	case "sum_samples":
-		return newSumSamplesAggrState(), nil
+		return newSumSamplesAggrState(), false, nil
 	case "total":
-		return newTotalAggrState(stalenessInterval, false, true), nil
+		return newTotalAggrState(false, true), true, nil
 	case "total_prometheus":
-		return newTotalAggrState(stalenessInterval, false, false), nil
+		return newTotalAggrState(false, false), true, nil
 	case "unique_samples":
-		return newUniqueSamplesAggrState(), nil
+		return newUniqueSamplesAggrState(), false, nil
 	default:
-		return nil, fmt.Errorf("unsupported output=%q; supported values: %s; see https://docs.victoriametrics.com/stream-aggregation/", output, supportedOutputs)
+		return nil, false, fmt.Errorf("unsupported output=%q; supported values: %s; see https://docs.victoriametrics.com/stream-aggregation/", output, supportedOutputs)
 	}
 }
 
@@ -823,8 +828,10 @@ func (a *aggregator) dedupFlush() {
 	}
 
 	startTime := time.Now()
+	timestamp := startTime.UnixMilli()
+	deleteDeadline := timestamp + a.stalenessInterval
 
-	a.da.flush(a.pushSamples)
+	a.da.flush(a.pushSamples, timestamp, deleteDeadline)
 
 	d := time.Since(startTime)
 	a.dedupFlushDuration.Update(d.Seconds())
@@ -902,6 +909,7 @@ func (a *aggregator) Push(tss []prompbmarshal.TimeSeries, matchIdxs []byte) {
 	minTimestamp := a.minTimestamp.Load()
 
 	nowMsec := time.Now().UnixMilli()
+	deleteDeadline := nowMsec + a.stalenessInterval
 	var maxLagMsec int64
 	for idx, ts := range tss {
 		if !a.match.Match(ts.Labels) {
@@ -930,7 +938,7 @@ func (a *aggregator) Push(tss []prompbmarshal.TimeSeries, matchIdxs []byte) {
 		}
 
 		bufLen := len(buf)
-		buf = compressLabels(buf, inputLabels.Labels, outputLabels.Labels)
+		buf = compressLabels(buf, a.lc, inputLabels.Labels, outputLabels.Labels, a.includeInputKey, deleteDeadline)
 		// key remains valid only by the end of this function and can't be reused after
 		// do not intern key because number of unique keys could be too high
 		key := bytesutil.ToUnsafeString(buf[bufLen:])
@@ -964,52 +972,60 @@ func (a *aggregator) Push(tss []prompbmarshal.TimeSeries, matchIdxs []byte) {
 	ctx.buf = buf
 
 	if a.da != nil {
-		a.da.pushSamples(samples)
+		a.da.pushSamples(samples, deleteDeadline)
 	} else {
-		a.pushSamples(samples)
+		a.pushSamples(samples, deleteDeadline)
 	}
 }
 
-func compressLabels(dst []byte, inputLabels, outputLabels []prompbmarshal.Label) []byte {
+func compressLabels(dst []byte, lc *promutils.LabelsCompressor, inputLabels, outputLabels []prompbmarshal.Label, includeInputKey bool, deleteDeadline int64) []byte {
 	bb := bbPool.Get()
-	bb.B = lc.Compress(bb.B, inputLabels)
-	dst = encoding.MarshalVarUint64(dst, uint64(len(bb.B)))
-	dst = append(dst, bb.B...)
-	bbPool.Put(bb)
-	dst = lc.Compress(dst, outputLabels)
+	bb.B = lc.Compress(bb.B, outputLabels, deleteDeadline)
+	if includeInputKey {
+		dst = encoding.MarshalVarUint64(dst, uint64(len(bb.B)))
+		dst = append(dst, bb.B...)
+		bbPool.Put(bb)
+		dst = lc.Compress(dst, inputLabels, deleteDeadline)
+	} else {
+		dst = append(dst, bb.B...)
+		bbPool.Put(bb)
+	}
 	return dst
 }
 
-func decompressLabels(dst []prompbmarshal.Label, key string) []prompbmarshal.Label {
+func decompressLabels(dst []prompbmarshal.Label, lc *promutils.LabelsCompressor, key string) []prompbmarshal.Label {
 	return lc.Decompress(dst, bytesutil.ToUnsafeBytes(key))
 }
 
-func getOutputKey(key string) string {
+func getOutputKey(key string, includeInputKey bool) string {
 	src := bytesutil.ToUnsafeBytes(key)
-	inputKeyLen, nSize := encoding.UnmarshalVarUint64(src)
-	if nSize <= 0 {
-		logger.Panicf("BUG: cannot unmarshal inputKeyLen from uvarint")
+	outputKey := src
+	if includeInputKey {
+		outputKeyLen, nSize := encoding.UnmarshalVarUint64(src)
+		if nSize <= 0 {
+			logger.Panicf("BUG: cannot unmarshal outputKeyLen from uvarint")
+		}
+		src = src[nSize:]
+		outputKey = src[:outputKeyLen]
 	}
-	src = src[nSize:]
-	outputKey := src[inputKeyLen:]
 	return bytesutil.ToUnsafeString(outputKey)
 }
 
 func getInputOutputKey(key string) (string, string) {
 	src := bytesutil.ToUnsafeBytes(key)
-	inputKeyLen, nSize := encoding.UnmarshalVarUint64(src)
+	outputKeyLen, nSize := encoding.UnmarshalVarUint64(src)
 	if nSize <= 0 {
-		logger.Panicf("BUG: cannot unmarshal inputKeyLen from uvarint")
+		logger.Panicf("BUG: cannot unmarshal outputKeyLen from uvarint")
 	}
 	src = src[nSize:]
-	inputKey := src[:inputKeyLen]
-	outputKey := src[inputKeyLen:]
+	outputKey := src[:outputKeyLen]
+	inputKey := src[outputKeyLen:]
 	return bytesutil.ToUnsafeString(inputKey), bytesutil.ToUnsafeString(outputKey)
 }
 
-func (a *aggregator) pushSamples(samples []pushSample) {
+func (a *aggregator) pushSamples(samples []pushSample, deleteDeadline int64) {
 	for _, ao := range a.aggrOutputs {
-		ao.as.pushSamples(samples)
+		ao.as.pushSamples(samples, deleteDeadline, a.includeInputKey)
 	}
 }
 
@@ -1169,7 +1185,7 @@ func (ctx *flushCtx) flushSeries() {
 func (ctx *flushCtx) appendSeries(key, suffix string, value float64) {
 	labelsLen := len(ctx.labels)
 	samplesLen := len(ctx.samples)
-	ctx.labels = decompressLabels(ctx.labels, key)
+	ctx.labels = decompressLabels(ctx.labels, ctx.a.lc, key)
 	if !ctx.a.keepMetricNames {
 		ctx.labels = addMetricSuffix(ctx.labels, labelsLen, ctx.a.suffix, suffix)
 	}
@@ -1191,7 +1207,7 @@ func (ctx *flushCtx) appendSeries(key, suffix string, value float64) {
 func (ctx *flushCtx) appendSeriesWithExtraLabel(key, suffix string, value float64, extraName, extraValue string) {
 	labelsLen := len(ctx.labels)
 	samplesLen := len(ctx.samples)
-	ctx.labels = decompressLabels(ctx.labels, key)
+	ctx.labels = decompressLabels(ctx.labels, ctx.a.lc, key)
 	if !ctx.a.keepMetricNames {
 		ctx.labels = addMetricSuffix(ctx.labels, labelsLen, ctx.a.suffix, suffix)
 	}

lib/streamaggr/sum_samples.go

@@ -12,41 +12,46 @@ type sumSamplesAggrState struct {
 }
 
 type sumSamplesStateValue struct {
-	mu      sync.Mutex
-	sum     float64
-	deleted bool
+	mu             sync.Mutex
+	sum            float64
+	deleted        bool
+	defined        bool
+	deleteDeadline int64
 }
 
 func newSumSamplesAggrState() *sumSamplesAggrState {
 	return &sumSamplesAggrState{}
 }
 
-func (as *sumSamplesAggrState) pushSamples(samples []pushSample) {
+func (as *sumSamplesAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
-			v = &sumSamplesStateValue{
-				sum: s.value,
-			}
+			v = &sumSamplesStateValue{}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The new entry has been successfully created.
-				continue
+			if loaded {
+				// Use the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Use the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*sumSamplesStateValue)
 		sv.mu.Lock()
+		if !sv.defined {
+			sv.defined = true
+		}
 		deleted := sv.deleted
 		if !deleted {
 			sv.sum += s.value
+			sv.deleteDeadline = deleteDeadline
+			if !sv.defined {
+				sv.defined = true
+			}
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -65,9 +70,22 @@ func (as *sumSamplesAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*sumSamplesStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
+		if !sv.defined {
+			sv.mu.Unlock()
+			return true
+		}
+
 		sum := sv.sum
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.defined = false
+		sv.sum = 0
 		sv.mu.Unlock()
 
 		key := k.(string)

lib/streamaggr/total.go

@@ -3,10 +3,9 @@ package streamaggr
 import (
 	"math"
 	"sync"
-	"time"
+	"sync/atomic"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 )
 
 // totalAggrState calculates output=total, total_prometheus, increase and increase_prometheus.
@@ -19,48 +18,37 @@ type totalAggrState struct {
 	// Whether to take into account the first sample in new time series when calculating the output value.
 	keepFirstSample bool
 
-	// Time series state is dropped if no new samples are received during stalenessSecs.
-	//
-	// Aslo, the first sample per each new series is ignored during stalenessSecs even if keepFirstSample is set.
-	// see ignoreFirstSampleDeadline for more details.
-	stalenessSecs uint64
-
-	// The first sample per each new series is ignored until this unix timestamp deadline in seconds even if keepFirstSample is set.
+	// The first sample per each new series is ignored first two intervals
 	// This allows avoiding an initial spike of the output values at startup when new time series
-	// cannot be distinguished from already existing series. This is tracked with ignoreFirstSampleDeadline.
-	ignoreFirstSampleDeadline uint64
+	// cannot be distinguished from already existing series. This is tracked with ignoreFirstSamples.
+	ignoreFirstSamples atomic.Int32
 }
 
 type totalStateValue struct {
 	mu             sync.Mutex
 	lastValues     map[string]totalLastValueState
 	total          float64
-	deleteDeadline uint64
+	deleteDeadline int64
 	deleted        bool
 }
 
 type totalLastValueState struct {
 	value          float64
 	timestamp      int64
-	deleteDeadline uint64
+	deleteDeadline int64
 }
 
-func newTotalAggrState(stalenessInterval time.Duration, resetTotalOnFlush, keepFirstSample bool) *totalAggrState {
-	stalenessSecs := roundDurationToSecs(stalenessInterval)
-	ignoreFirstSampleDeadline := fasttime.UnixTimestamp() + stalenessSecs
-
-	return &totalAggrState{
-		resetTotalOnFlush:         resetTotalOnFlush,
-		keepFirstSample:           keepFirstSample,
-		stalenessSecs:             stalenessSecs,
-		ignoreFirstSampleDeadline: ignoreFirstSampleDeadline,
+func newTotalAggrState(resetTotalOnFlush, keepFirstSample bool) *totalAggrState {
+	as := &totalAggrState{
+		resetTotalOnFlush: resetTotalOnFlush,
+		keepFirstSample:   keepFirstSample,
 	}
+	as.ignoreFirstSamples.Store(2)
+	return as
 }
 
-func (as *totalAggrState) pushSamples(samples []pushSample) {
-	currentTime := fasttime.UnixTimestamp()
-	deleteDeadline := currentTime + as.stalenessSecs
-	keepFirstSample := as.keepFirstSample && currentTime > as.ignoreFirstSampleDeadline
+func (as *totalAggrState) pushSamples(samples []pushSample, deleteDeadline int64, _ bool) {
+	keepFirstSample := as.keepFirstSample && as.ignoreFirstSamples.Load() <= 0
 	for i := range samples {
 		s := &samples[i]
 		inputKey, outputKey := getInputOutputKey(s.key)
@@ -116,11 +104,9 @@ func (as *totalAggrState) pushSamples(samples []pushSample) {
 }
 
 func (as *totalAggrState) flushState(ctx *flushCtx) {
-	currentTime := fasttime.UnixTimestamp()
-
 	suffix := as.getSuffix()
 
-	as.removeOldEntries(currentTime)
+	as.removeOldEntries(ctx)
 
 	m := &as.m
 	m.Range(func(k, v any) bool {
@@ -143,6 +129,10 @@ func (as *totalAggrState) flushState(ctx *flushCtx) {
 		}
 		return true
 	})
+	ignoreFirstSamples := as.ignoreFirstSamples.Load()
+	if ignoreFirstSamples > 0 {
+		as.ignoreFirstSamples.Add(-1)
+	}
 }
 
 func (as *totalAggrState) getSuffix() string {
@@ -159,16 +149,18 @@ func (as *totalAggrState) getSuffix() string {
 	return "total_prometheus"
 }
 
-func (as *totalAggrState) removeOldEntries(currentTime uint64) {
+func (as *totalAggrState) removeOldEntries(ctx *flushCtx) {
 	m := &as.m
 	m.Range(func(k, v any) bool {
 		sv := v.(*totalStateValue)
 
 		sv.mu.Lock()
-		if currentTime > sv.deleteDeadline {
+		if ctx.flushTimestamp > sv.deleteDeadline {
 			// Mark the current entry as deleted
 			sv.deleted = true
 			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
 			m.Delete(k)
 			return true
 		}
@@ -176,7 +168,7 @@ func (as *totalAggrState) removeOldEntries(currentTime uint64) {
 		// Delete outdated entries in sv.lastValues
 		lvs := sv.lastValues
 		for k1, lv := range lvs {
-			if currentTime > lv.deleteDeadline {
+			if ctx.flushTimestamp > lv.deleteDeadline {
 				delete(lvs, k1)
 			}
 		}

lib/streamaggr/unique_samples.go

@@ -12,37 +12,34 @@ type uniqueSamplesAggrState struct {
 }
 
 type uniqueSamplesStateValue struct {
-	mu      sync.Mutex
-	m       map[float64]struct{}
-	deleted bool
+	mu             sync.Mutex
+	m              map[float64]struct{}
+	deleted        bool
+	deleteDeadline int64
 }
 
 func newUniqueSamplesAggrState() *uniqueSamplesAggrState {
 	return &uniqueSamplesAggrState{}
 }
 
-func (as *uniqueSamplesAggrState) pushSamples(samples []pushSample) {
+func (as *uniqueSamplesAggrState) pushSamples(samples []pushSample, deleteDeadline int64, includeInputKey bool) {
 	for i := range samples {
 		s := &samples[i]
-		outputKey := getOutputKey(s.key)
+		outputKey := getOutputKey(s.key, includeInputKey)
 
 	again:
 		v, ok := as.m.Load(outputKey)
 		if !ok {
 			// The entry is missing in the map. Try creating it.
 			v = &uniqueSamplesStateValue{
-				m: map[float64]struct{}{
-					s.value: {},
-				},
+				m: make(map[float64]struct{}),
 			}
 			outputKey = bytesutil.InternString(outputKey)
 			vNew, loaded := as.m.LoadOrStore(outputKey, v)
-			if !loaded {
-				// The new entry has been successfully created.
-				continue
+			if loaded {
+				// Use the entry created by a concurrent goroutine.
+				v = vNew
 			}
-			// Use the entry created by a concurrent goroutine.
-			v = vNew
 		}
 		sv := v.(*uniqueSamplesStateValue)
 		sv.mu.Lock()
@@ -51,6 +48,7 @@ func (as *uniqueSamplesAggrState) pushSamples(samples []pushSample) {
 			if _, ok := sv.m[s.value]; !ok {
 				sv.m[s.value] = struct{}{}
 			}
+			sv.deleteDeadline = deleteDeadline
 		}
 		sv.mu.Unlock()
 		if deleted {
@@ -69,13 +67,21 @@ func (as *uniqueSamplesAggrState) flushState(ctx *flushCtx) {
 
 		sv := v.(*uniqueSamplesStateValue)
 		sv.mu.Lock()
+		if ctx.flushTimestamp > sv.deleteDeadline {
+			sv.deleted = true
+			sv.mu.Unlock()
+			key := k.(string)
+			ctx.a.lc.Delete(bytesutil.ToUnsafeBytes(key), ctx.flushTimestamp)
+			m.Delete(k)
+			return true
+		}
 		n := len(sv.m)
-		// Mark the entry as deleted, so it won't be updated anymore by concurrent pushSample() calls.
-		sv.deleted = true
+		sv.m = make(map[float64]struct{})
 		sv.mu.Unlock()
-
-		key := k.(string)
-		ctx.appendSeries(key, "unique_samples", float64(n))
+		if n > 0 {
+			key := k.(string)
+			ctx.appendSeries(key, "unique_samples", float64(n))
+		}
 		return true
 	})
 }