Merge tag 'v1.82.1' into HEAD

v1.82.1

app/victoria-metrics/main.go

@@ -29,8 +29,16 @@ var (
 		"equal to -dedup.minScrapeInterval > 0. See https://docs.victoriametrics.com/#deduplication and https://docs.victoriametrics.com/#downsampling")
 	dryRun = flag.Bool("dryRun", false, "Whether to check only -promscrape.config and then exit. "+
 		"Unknown config entries aren't allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse=false command-line flag")
+	downsamplingPeriods = flagutil.NewArrayString("downsampling.period", "Comma-separated downsampling periods in the format 'offset:period'. For example, '30d:10m' instructs "+
+		"to leave a single sample per 10 minutes for samples older than 30 days. See https://docs.victoriametrics.com/#downsampling for details")
 )
 
+// custom api help links [["/api","doc"]] without http.pathPrefix.
+var customAPIPathList = [][]string{
+	{"/graph/explore", "explore metrics grafana page"},
+	{"/graph/d/prometheus-advanced/advanced-data-exploration", "PMM grafana dashboard"},
+}
+
 func main() {
 	// Write flags and help message to stdout, since it is easier to grep or pipe.
 	flag.CommandLine.SetOutput(os.Stdout)
@@ -53,7 +61,10 @@ func main() {
 
 	logger.Infof("starting VictoriaMetrics at %q...", *httpListenAddr)
 	startTime := time.Now()
-	storage.SetDedupInterval(*minScrapeInterval)
+	err := storage.SetDownsamplingPeriods(*downsamplingPeriods, *minScrapeInterval)
+	if err != nil {
+		logger.Fatalf("cannot parse -downsampling.period: %s", err)
+	}
 	vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
 	vmselect.Init()
 	vminsert.Init()
@@ -104,6 +115,10 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 			{"api/v1/status/top_queries", "top queries"},
 			{"api/v1/status/active_queries", "active queries"},
 		})
+		for _, p := range customAPIPathList {
+			p, doc := p[0], p[1]
+			fmt.Fprintf(w, "<a href=%q>%s</a> - %s<br/>", p, p, doc)
+		}
 		return true
 	}
 	if vminsert.RequestHandler(w, r) {

app/vmalert/notifier/alertmanager_request.qtpl

@@ -15,10 +15,10 @@
     "endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
     {% endif %}
     "labels": {
-        "alertname":{%q= alert.Name %}
         {% code lbls := alert.toPromLabels(relabelCfg) %}
-        {% for _, l := range lbls %}
-            ,{%q= l.Name %}:{%q= l.Value %}
+        {% code ll := len(lbls) %}
+        {% for idx, l := range lbls %}
+            {%q= l.Name %}:{%q= l.Value %}{% if idx != ll-1 %}, {% endif %}
         {% endfor %}
     },
     "annotations": {

app/vmalert/notifier/alertmanager_request.qtpl.go

@@ -51,22 +51,27 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
 		}
 //line app/vmalert/notifier/alertmanager_request.qtpl:16
-		qw422016.N().S(`"labels": {"alertname":`)
+		qw422016.N().S(`"labels": {`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:18
-		qw422016.N().Q(alert.Name)
-//line app/vmalert/notifier/alertmanager_request.qtpl:19
 		lbls := alert.toPromLabels(relabelCfg)
 
+//line app/vmalert/notifier/alertmanager_request.qtpl:19
+		ll := len(lbls)
+
 //line app/vmalert/notifier/alertmanager_request.qtpl:20
-		for _, l := range lbls {
-//line app/vmalert/notifier/alertmanager_request.qtpl:20
-			qw422016.N().S(`,`)
+		for idx, l := range lbls {
 //line app/vmalert/notifier/alertmanager_request.qtpl:21
 			qw422016.N().Q(l.Name)
 //line app/vmalert/notifier/alertmanager_request.qtpl:21
 			qw422016.N().S(`:`)
 //line app/vmalert/notifier/alertmanager_request.qtpl:21
 			qw422016.N().Q(l.Value)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			if idx != ll-1 {
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+				qw422016.N().S(`,`)
+//line app/vmalert/notifier/alertmanager_request.qtpl:21
+			}
 //line app/vmalert/notifier/alertmanager_request.qtpl:22
 		}
 //line app/vmalert/notifier/alertmanager_request.qtpl:22

app/vmalert/notifier/alertmanager_test.go

@@ -67,9 +67,6 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].GeneratorURL != "0/0" {
 				t.Errorf("expected 0/0 as generatorURL got %s", a[0].GeneratorURL)
 			}
-			if a[0].Labels["alertname"] != "alert0" {
-				t.Errorf("expected alert0 as alert name got %s", a[0].Labels["alertname"])
-			}
 			if a[0].StartsAt.IsZero() {
 				t.Errorf("expected non-zero start time")
 			}

app/vmalert/web_types.go

@@ -5,11 +5,11 @@ import (
 	"time"
 )
 
-// APIAlert represents a notifier.AlertingRule ruleState
+// APIAlert represents a notifier.AlertingRule state
 // for WEB view
 // https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIAlert struct {
-	State       string            `json:"ruleState"`
+	State       string            `json:"state"`
 	Name        string            `json:"name"`
 	Value       string            `json:"value"`
 	Labels      map[string]string `json:"labels,omitempty"`
@@ -30,7 +30,7 @@ type APIAlert struct {
 	// SourceLink contains a link to a system which should show
 	// why Alert was generated
 	SourceLink string `json:"source"`
-	// Restored shows whether Alert's ruleState was restored on restart
+	// Restored shows whether Alert's state was restored on restart
 	Restored bool `json:"restored"`
 }
 
@@ -86,10 +86,10 @@ type GroupAlerts struct {
 // see https://github.com/prometheus/compliance/blob/main/alert_generator/specification.md#get-apiv1rules
 type APIRule struct {
 	// State must be one of these under following scenarios
-	//  "pending": at least 1 alert in the rule in pending ruleState and no other alert in firing ruleState.
-	//  "firing": at least 1 alert in the rule in firing ruleState.
-	//  "inactive": no alert in the rule in firing or pending ruleState.
-	State string `json:"ruleState"`
+	//  "pending": at least 1 alert in the rule in pending state and no other alert in firing ruleState.
+	//  "firing": at least 1 alert in the rule in firing state.
+	//  "inactive": no alert in the rule in firing or pending state.
+	State string `json:"state"`
 	Name  string `json:"name"`
 	// Query represents Rule's `expression` field
 	Query string `json:"query"`
@@ -121,7 +121,7 @@ type APIRule struct {
 	// GroupID is an unique Group's ID
 	GroupID string `json:"group_id"`
 
-	// TODO:
+	// Updates contains the ordered list of recorded ruleStateEntry objects
 	Updates []ruleStateEntry `json:"updates"`
 }
 

app/vmselect/netstorage/netstorage.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/promdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
@@ -264,6 +265,12 @@ var gomaxprocs = cgroup.AvailableCPUs()
 type packedTimeseries struct {
 	metricName string
 	brs        []blockRef
+	pd         *promData
+}
+
+type promData struct {
+	values     []float64
+	timestamps []int64
 }
 
 type unpackWorkItem struct {
@@ -461,11 +468,44 @@ func (pts *packedTimeseries) Unpack(dst *Result, tbf *tmpBlocksFile, tr storage.
 	if firstErr != nil {
 		return firstErr
 	}
-	dedupInterval := storage.GetDedupInterval()
+	if pts.pd != nil {
+		// Add data from Prometheus to dst.
+		// It usually has smaller timestamps than the data from sbs, so put it first.
+		dst.Values = append(dst.Values, pts.pd.values...)
+		dst.Timestamps = append(dst.Timestamps, pts.pd.timestamps...)
+	}
+	dedupInterval := storage.GetDedupInterval(tr.MinTimestamp)
 	mergeSortBlocks(dst, sbs, dedupInterval)
+	if pts.pd != nil {
+		if !sort.IsSorted(dst) {
+			sort.Sort(dst)
+		}
+		pts.pd = nil
+	}
 	return nil
 }
 
+// sort.Interface implementation for Result
+
+// Len implements sort.Interface
+func (r *Result) Len() int {
+	return len(r.Timestamps)
+}
+
+// Less implements sort.Interface
+func (r *Result) Less(i, j int) bool {
+	timestamps := r.Timestamps
+	return timestamps[i] < timestamps[j]
+}
+
+// Swap implements sort.Interface
+func (r *Result) Swap(i, j int) {
+	timestamps := r.Timestamps
+	values := r.Values
+	timestamps[i], timestamps[j] = timestamps[j], timestamps[i]
+	values[i], values[j] = values[j], values[i]
+}
+
 func getSortBlock() *sortBlock {
 	v := sbPool.Get()
 	if v == nil {
@@ -661,6 +701,15 @@ func LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames i
 	if err != nil {
 		return nil, fmt.Errorf("error during labels search on time range: %w", err)
 	}
+
+	// Merge labels obtained from Prometheus storage.
+	promLabels, err := promdb.GetLabelNamesOnTimeRange(tr, deadline)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain labels from Prometheus storage: %w", err)
+	}
+	qt.Printf("get %d label names from Prometheus storage", len(promLabels))
+	labels = mergeStrings(labels, promLabels)
+
 	// Sort labels like Prometheus does
 	sort.Strings(labels)
 	qt.Printf("sort %d labels", len(labels))
@@ -732,14 +781,44 @@ func LabelValues(qt *querytracer.Tracer, labelName string, sq *storage.SearchQue
 	}
 	labelValues, err := vmstorage.SearchLabelValuesWithFiltersOnTimeRange(qt, labelName, tfss, tr, maxLabelValues, sq.MaxMetrics, deadline.Deadline())
 	if err != nil {
-		return nil, fmt.Errorf("error during label values search on time range for labelName=%q: %w", labelName, err)
+		return nil, fmt.Errorf("error during label values search on time range: %w", err)
 	}
+
+	// Merge label values obtained from Prometheus storage.
+	promLabelValues, err := promdb.GetLabelValuesOnTimeRange(labelName, tr, deadline)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain label values on time range for %q from Prometheus storage: %w", labelName, err)
+	}
+	qt.Printf("get %d label values from Prometheus storage", len(promLabelValues))
+	labelValues = mergeStrings(labelValues, promLabelValues)
+
 	// Sort labelValues like Prometheus does
 	sort.Strings(labelValues)
 	qt.Printf("sort %d label values", len(labelValues))
 	return labelValues, nil
 }
 
+func mergeStrings(a, b []string) []string {
+	if len(a) == 0 {
+		return b
+	}
+	if len(b) == 0 {
+		return a
+	}
+	m := make(map[string]struct{}, len(a)+len(b))
+	for _, s := range a {
+		m[s] = struct{}{}
+	}
+	for _, s := range b {
+		m[s] = struct{}{}
+	}
+	result := make([]string, 0, len(m))
+	for s := range m {
+		result = append(result, s)
+	}
+	return result
+}
+
 // GraphiteTagValues returns tag values for the given tagName until the given deadline.
 func GraphiteTagValues(qt *querytracer.Tracer, tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, error) {
 	qt = qt.NewChild("get graphite tag values for tagName=%s, filter=%s, limit=%d", tagName, filter, limit)
@@ -1068,6 +1147,26 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 	}
 	qt.Printf("fetch unique series=%d, blocks=%d, samples=%d, bytes=%d", len(m), blocksRead, samples, tbf.Len())
 
+	// Fetch data from promdb.
+	pm := make(map[string]*promData)
+	err = promdb.VisitSeries(sq, deadline, func(metricName []byte, values []float64, timestamps []int64) {
+		pd := pm[string(metricName)]
+		if pd == nil {
+			if _, ok := m[string(metricName)]; !ok {
+				orderedMetricNames = append(orderedMetricNames, string(metricName))
+			}
+			pd = &promData{}
+			pm[string(metricName)] = pd
+		}
+		pd.values = append(pd.values, values...)
+		pd.timestamps = append(pd.timestamps, timestamps...)
+	})
+	if err != nil {
+		putTmpBlocksFile(tbf)
+		putStorageSearch(sr)
+		return nil, fmt.Errorf("error when searching in Prometheus data: %w", err)
+	}
+
 	var rss Results
 	rss.tr = tr
 	rss.deadline = deadline
@@ -1076,6 +1175,7 @@ func ProcessSearchQuery(qt *querytracer.Tracer, sq *storage.SearchQuery, deadlin
 		pts[i] = packedTimeseries{
 			metricName: metricName,
 			brs:        m[metricName],
+			pd:         pm[metricName],
 		}
 	}
 	rss.packedTimeseries = pts

app/vmselect/vmui/asset-manifest.json

@@ -1,12 +1,12 @@
 {
   "files": {
     "main.css": "./static/css/main.ba692000.css",
-    "main.js": "./static/js/main.623b88d4.js",
+    "main.js": "./static/js/main.dd4b1276.js",
     "static/js/27.939f971b.chunk.js": "./static/js/27.939f971b.chunk.js",
     "index.html": "./index.html"
   },
   "entrypoints": [
     "static/css/main.ba692000.css",
-    "static/js/main.623b88d4.js"
+    "static/js/main.dd4b1276.js"
   ]
 }

app/vmselect/vmui/index.html

@@ -1 +1 @@
-<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.623b88d4.js"></script><link href="./static/css/main.ba692000.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
+<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.ico"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="VM-UI is a metric explorer for Victoria Metrics"/><link rel="apple-touch-icon" href="./apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="./favicon-32x32.png"><link rel="manifest" href="./manifest.json"/><title>VM UI</title><link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"/><script src="./dashboards/index.js" type="module"></script><script defer="defer" src="./static/js/main.dd4b1276.js"></script><link href="./static/css/main.ba692000.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

app/vmstorage/main.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage/promdb"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
@@ -126,6 +127,8 @@ func InitWithoutMetrics(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
 	sizeBytes := tm.SmallSizeBytes + tm.BigSizeBytes
 	logger.Infof("successfully opened storage %q in %.3f seconds; partsCount: %d; blocksCount: %d; rowsCount: %d; sizeBytes: %d",
 		*DataPath, time.Since(startTime).Seconds(), partsCount, blocksCount, rowsCount, sizeBytes)
+
+	promdb.Init(retentionPeriod.Msecs)
 }
 
 // Storage is a storage.
@@ -238,6 +241,7 @@ func Stop() {
 	logger.Infof("gracefully closing the storage at %s", *DataPath)
 	startTime := time.Now()
 	WG.WaitAndBlock()
+	promdb.MustClose()
 	stopStaleSnapshotsRemover()
 	Storage.MustClose()
 	logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())

app/vmstorage/promdb/promdb.go

@@ -0,0 +1,254 @@
+package promdb
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
+	"github.com/go-kit/kit/log"
+	"github.com/oklog/ulid"
+	"github.com/prometheus/prometheus/pkg/labels"
+	promstorage "github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/tsdb"
+)
+
+var prometheusDataPath = flag.String("prometheusDataPath", "", "Optional path to readonly historical Prometheus data")
+
+var prometheusRetentionMsecs int64
+
+// Init must be called after flag.Parse and before using the package.
+//
+// See also MustClose.
+func Init(retentionMsecs int64) {
+	if promDB != nil {
+		logger.Fatalf("BUG: it looks like MustOpenPromDB is called multiple times without MustClosePromDB call")
+	}
+	prometheusRetentionMsecs = retentionMsecs
+	if *prometheusDataPath == "" {
+		return
+	}
+	l := log.LoggerFunc(func(a ...interface{}) error {
+		logger.Infof("%v", a)
+		return nil
+	})
+	opts := tsdb.DefaultOptions()
+	opts.RetentionDuration = retentionMsecs
+
+	// Set max block duration to 10% of retention period or 31 days
+	// according to https://prometheus.io/docs/prometheus/latest/storage/#compaction
+	maxBlockDuration := int64((31 * 24 * time.Hour) / time.Millisecond)
+	if maxBlockDuration > retentionMsecs/10 {
+		maxBlockDuration = retentionMsecs / 10
+	}
+	if maxBlockDuration < opts.MinBlockDuration {
+		maxBlockDuration = opts.MinBlockDuration
+	}
+	opts.MaxBlockDuration = maxBlockDuration
+
+	// Custom delete function is needed, because Prometheus by default doesn't delete
+	// blocks outside the retention if no new blocks are created with samples with the current timestamps.
+	// See https://github.com/prometheus/prometheus/blob/997bb7134fcfd7279f250e183e78681e48a56aff/tsdb/db.go#L1116
+	opts.BlocksToDelete = func(blocks []*tsdb.Block) map[ulid.ULID]struct{} {
+		m := make(map[ulid.ULID]struct{})
+		minRetentionTime := time.Now().Unix()*1000 - retentionMsecs
+		for _, block := range blocks {
+			meta := block.Meta()
+			// delete block marked for deletion by compaction code.
+			if meta.Compaction.Deletable {
+				m[meta.ULID] = struct{}{}
+				continue
+			}
+			if block.MaxTime() < minRetentionTime {
+				m[meta.ULID] = struct{}{}
+			}
+		}
+		return m
+	}
+	pdb, err := tsdb.Open(*prometheusDataPath, l, nil, opts)
+	if err != nil {
+		logger.Panicf("FATAL: cannot open Prometheus data at -prometheusDataPath=%q: %s", *prometheusDataPath, err)
+	}
+	promDB = pdb
+	logger.Infof("successfully opened historical Prometheus data at -prometheusDataPath=%q with retentionMsecs=%d", *prometheusDataPath, retentionMsecs)
+}
+
+// MustClose must be called on graceful shutdown.
+//
+// Package functionality cannot be used after this call.
+func MustClose() {
+	if *prometheusDataPath == "" {
+		return
+	}
+	if promDB == nil {
+		logger.Panicf("BUG: it looks like MustClosePromDB is called without MustOpenPromDB call")
+	}
+	if err := promDB.Close(); err != nil {
+		logger.Panicf("FATAL: cannot close promDB: %s", err)
+	}
+	promDB = nil
+	logger.Infof("successfully closed historical Prometheus data at -prometheusDataPath=%q", *prometheusDataPath)
+}
+
+var promDB *tsdb.DB
+
+// GetLabelNamesOnTimeRange returns label names.
+func GetLabelNamesOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) ([]string, error) {
+	d := time.Unix(int64(deadline.Deadline()), 0)
+	ctx, cancel := context.WithDeadline(context.Background(), d)
+	defer cancel()
+	q, err := promDB.Querier(ctx, tr.MinTimestamp, tr.MaxTimestamp)
+	if err != nil {
+		return nil, err
+	}
+	defer mustCloseQuerier(q)
+
+	names, _, err := q.LabelNames()
+	// Make full copy of names, since they cannot be used after q is closed.
+	names = copyStringsWithMemory(names)
+	return names, err
+}
+
+// GetLabelValuesOnTimeRange returns values for the given labelName on the given tr.
+func GetLabelValuesOnTimeRange(labelName string, tr storage.TimeRange, deadline searchutils.Deadline) ([]string, error) {
+	d := time.Unix(int64(deadline.Deadline()), 0)
+	ctx, cancel := context.WithDeadline(context.Background(), d)
+	defer cancel()
+	q, err := promDB.Querier(ctx, tr.MinTimestamp, tr.MaxTimestamp)
+	if err != nil {
+		return nil, err
+	}
+	defer mustCloseQuerier(q)
+
+	values, _, err := q.LabelValues(labelName)
+	// Make full copy of values, since they cannot be used after q is closed.
+	values = copyStringsWithMemory(values)
+	return values, err
+}
+
+func copyStringsWithMemory(a []string) []string {
+	result := make([]string, len(a))
+	for i, s := range a {
+		result[i] = string(append([]byte{}, s...))
+	}
+	return result
+}
+
+// SeriesVisitor is called by VisitSeries for each matching time series.
+//
+// The caller shouldn't hold references to metricName, values and timestamps after returning.
+type SeriesVisitor func(metricName []byte, values []float64, timestamps []int64)
+
+// VisitSeries calls f for each series found in the pdb.
+func VisitSeries(sq *storage.SearchQuery, deadline searchutils.Deadline, f SeriesVisitor) error {
+	if *prometheusDataPath == "" {
+		return nil
+	}
+	d := time.Unix(int64(deadline.Deadline()), 0)
+	ctx, cancel := context.WithDeadline(context.Background(), d)
+	defer cancel()
+	minTime, maxTime := getSearchTimeRange(sq)
+	q, err := promDB.Querier(ctx, minTime, maxTime)
+	if err != nil {
+		return err
+	}
+	defer mustCloseQuerier(q)
+	var seriesSet []promstorage.SeriesSet
+	for _, tf := range sq.TagFilterss {
+		ms, err := convertTagFiltersToMatchers(tf)
+		if err != nil {
+			return fmt.Errorf("cannot convert tag filters to matchers: %w", err)
+		}
+		s := q.Select(false, nil, ms...)
+		seriesSet = append(seriesSet, s)
+	}
+	ss := promstorage.NewMergeSeriesSet(seriesSet, promstorage.ChainedSeriesMerge)
+	var (
+		mn         storage.MetricName
+		metricName []byte
+		values     []float64
+		timestamps []int64
+	)
+	for ss.Next() {
+		s := ss.At()
+		convertPromLabelsToMetricName(&mn, s.Labels())
+		metricName = mn.SortAndMarshal(metricName[:0])
+		values = values[:0]
+		timestamps = timestamps[:0]
+		it := s.Iterator()
+		for it.Next() {
+			ts, v := it.At()
+			values = append(values, v)
+			timestamps = append(timestamps, ts)
+		}
+		if err := it.Err(); err != nil {
+			return fmt.Errorf("error when iterating Prometheus series: %w", err)
+		}
+		f(metricName, values, timestamps)
+	}
+	return ss.Err()
+}
+
+func getSearchTimeRange(sq *storage.SearchQuery) (int64, int64) {
+	maxTime := sq.MaxTimestamp
+	minTime := sq.MinTimestamp
+	minRetentionTime := time.Now().Unix()*1000 - prometheusRetentionMsecs
+	if maxTime < minRetentionTime {
+		maxTime = minRetentionTime
+	}
+	if minTime < minRetentionTime {
+		minTime = minRetentionTime
+	}
+	return minTime, maxTime
+}
+
+func convertPromLabelsToMetricName(dst *storage.MetricName, labels []labels.Label) {
+	dst.Reset()
+	for _, label := range labels {
+		if label.Name == "__name__" {
+			dst.MetricGroup = append(dst.MetricGroup[:0], label.Value...)
+		} else {
+			dst.AddTag(label.Name, label.Value)
+		}
+	}
+}
+
+func convertTagFiltersToMatchers(tfs []storage.TagFilter) ([]*labels.Matcher, error) {
+	ms := make([]*labels.Matcher, 0, len(tfs))
+	for _, tf := range tfs {
+		var mt labels.MatchType
+		if tf.IsNegative {
+			if tf.IsRegexp {
+				mt = labels.MatchNotRegexp
+			} else {
+				mt = labels.MatchNotEqual
+			}
+		} else {
+			if tf.IsRegexp {
+				mt = labels.MatchRegexp
+			} else {
+				mt = labels.MatchEqual
+			}
+		}
+		key := string(tf.Key)
+		if key == "" {
+			key = "__name__"
+		}
+		value := string(tf.Value)
+		m, err := labels.NewMatcher(mt, key, value)
+		if err != nil {
+			return nil, err
+		}
+		ms = append(ms, m)
+	}
+	return ms, nil
+}
+
+func mustCloseQuerier(q promstorage.Querier) {
+	if err := q.Close(); err != nil {
+		logger.Panicf("FATAL: cannot close querier: %s", err)
+	}
+}

app/vmui/packages/vmui/src/components/CustomPanel/Configurator/Query/QueryConfigurator.tsx

@@ -1,4 +1,4 @@
-import React, {FC, useState} from "preact/compat";
+import React, {FC, useState, useEffect} from "preact/compat";
 import Box from "@mui/material/Box";
 import IconButton from "@mui/material/IconButton";
 import Tooltip from "@mui/material/Tooltip";
@@ -11,6 +11,7 @@ import AdditionalSettings from "./AdditionalSettings";
 import {ErrorTypes} from "../../../../types";
 import Button from "@mui/material/Button";
 import Typography from "@mui/material/Typography";
+import usePrevious from "../../../../hooks/usePrevious";
 
 export interface QueryConfiguratorProps {
   error?: ErrorTypes | string;
@@ -23,6 +24,7 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({error, queryOptions}) =>
 
   const {query, queryHistory, queryControls: {autocomplete}} = useAppState();
   const [stateQuery, setStateQuery] = useState(query || []);
+  const prevStateQuery = usePrevious(stateQuery) as (undefined | string[]);
   const dispatch = useAppDispatch();
 
   const updateHistory = () => {
@@ -66,6 +68,13 @@ const QueryConfigurator: FC<QueryConfiguratorProps> = ({error, queryOptions}) =>
       payload: {value: {values, index: newIndexHistory}, queryNumber: indexQuery}
     });
   };
+
+  useEffect(() => {
+    if (prevStateQuery && (stateQuery.length < prevStateQuery.filter(q => q).length)) {
+      onRunQuery();
+    }
+  }, [stateQuery]);
+
   return <Box boxShadow="rgba(99, 99, 99, 0.2) 0px 2px 8px 0px;" p={4} pb={2} m={-4} mb={2}>
     <Box>
       {stateQuery.map((q, i) =>

docs/CHANGELOG.md

@@ -15,6 +15,15 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 ## tip
 
+## [v1.82.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.1)
+
+* BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): automatically update graph, legend and url after the removal of query field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3169) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3196#issuecomment-1269765205).
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove duplicate `alertname` JSON entry from generated alerts. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3053). Thanks to @Howie59 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3182)!
+* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix integration with Grafana via `-vmalert.proxyURL`, which has been broken in [v1.82.0](https://docs.victoriametrics.com/CHANGELOG.html#v1820). See [this issue](https://github.com/VictoriaMetrics/helm-charts/issues/391).
+* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): set default region to `us-east-1` if `AWS_REGION` environment variable isn't set. The issue was introduced in [vmbackup v1.82.0](https://docs.victoriametrics.com/CHANGELOG.html#v1820). See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3224).
+* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix deletion of old backups at [Azure blob storage](https://azure.microsoft.com/en-us/products/storage/blobs/).
+* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly apply regex filters when searching for time series. Previously unexpected time series could be returned from regex filter. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227). The issue was introduced in [v1.82.0](https://docs.victoriametrics.com/CHANGELOG.html#v1820).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmbagent.html): properly apply `if` section with regex filters. Previously unexpected metrics could be returned from `if` section. The issue was introduced in [v1.82.0](https://docs.victoriametrics.com/CHANGELOG.html#v1820).
 
 ## [v1.82.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.82.0)
 

go.mod

@@ -18,10 +18,17 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb/v3 v3.1.0
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/go-kit/kit v0.12.0
 	github.com/golang/snappy v0.0.4
 	github.com/googleapis/gax-go/v2 v2.5.1
 	github.com/influxdata/influxdb v1.10.0
 	github.com/klauspost/compress v1.15.11
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/oklog/ulid v1.3.1
+	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/prometheus v1.8.2-0.20201119142752-3ad25a6dc3d9
 	github.com/urfave/cli/v2 v2.17.1
 	github.com/valyala/fastjson v1.6.3
@@ -60,9 +67,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect
 	github.com/aws/smithy-go v1.13.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/fatih/color v1.13.0 // indirect
-	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -71,15 +75,11 @@ require (
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
-	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.13.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/rivo/uniseg v0.4.2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect

lib/backup/s3remote/s3.go

@@ -61,6 +61,7 @@ func (fs *FS) Init() error {
 	}
 	configOpts := []func(*config.LoadOptions) error{
 		config.WithSharedConfigProfile(fs.ProfileName),
+		config.WithDefaultRegion("us-east-1"),
 	}
 
 	if len(fs.CredsFilePath) > 0 {

lib/bytesutil/fast_string_matcher.go

@@ -1,6 +1,7 @@
 package bytesutil
 
 import (
+	"strings"
 	"sync"
 	"sync/atomic"
 )
@@ -38,6 +39,11 @@ func (fsm *FastStringMatcher) Match(s string) bool {
 	// Slow path - run matchFunc for s and store the result in the cache.
 	b := fsm.matchFunc(s)
 	bp := &b
+	// Make a copy of s in order to limit memory usage to the s length,
+	// since the s may point to bigger string.
+	// This also protects from the case when s contains unsafe string, which points to a temporary byte slice.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227
+	s = strings.Clone(s)
 	m.Store(s, bp)
 	n := atomic.AddUint64(&fsm.mLen, 1)
 	if n > 100e3 {

lib/bytesutil/fast_string_transformer.go

@@ -40,6 +40,8 @@ func (fst *FastStringTransformer) Transform(s string) string {
 	sTransformed := fst.transformFunc(s)
 	// Make a copy of s in order to limit memory usage to the s length,
 	// since the s may point to bigger string.
+	// This also protects from the case when s contains unsafe string, which points to a temporary byte slice.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3227
 	s = strings.Clone(s)
 	if sTransformed == s {
 		// point sTransformed to just allocated s, since it may point to s,

lib/storage/block.go

@@ -160,7 +160,8 @@ func (b *Block) deduplicateSamplesDuringMerge() {
 		// Nothing to dedup.
 		return
 	}
-	dedupInterval := GetDedupInterval()
+	maxTimestamp := srcTimestamps[len(srcTimestamps)-1]
+	dedupInterval := GetDedupInterval(maxTimestamp)
 	if dedupInterval <= 0 {
 		// Deduplication is disabled.
 		return

lib/storage/dedup.go

@@ -1,27 +1,7 @@
 package storage
 
-import (
-	"time"
-)
-
-// SetDedupInterval sets the deduplication interval, which is applied to raw samples during data ingestion and querying.
-//
-// De-duplication is disabled if dedupInterval is 0.
-//
-// This function must be called before initializing the storage.
-func SetDedupInterval(dedupInterval time.Duration) {
-	globalDedupInterval = dedupInterval.Milliseconds()
-}
-
-// GetDedupInterval returns the dedup interval in milliseconds, which has been set via SetDedupInterval.
-func GetDedupInterval() int64 {
-	return globalDedupInterval
-}
-
-var globalDedupInterval int64
-
 func isDedupEnabled() bool {
-	return globalDedupInterval > 0
+	return len(downsamplingPeriods) > 0
 }
 
 // DeduplicateSamples removes samples from src* if they are closer to each other than dedupInterval in millseconds.

lib/storage/downsampling.go

@@ -0,0 +1,123 @@
+package storage
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
+	"github.com/VictoriaMetrics/metricsql"
+)
+
+// SetDownsamplingPeriods configures downsampling.
+//
+// The function must be called before opening or creating any storage.
+func SetDownsamplingPeriods(periods []string, dedupInterval time.Duration) error {
+	dsps, err := parseDownsamplingPeriods(periods)
+	if err != nil {
+		return err
+	}
+	dedupIntervalMs := dedupInterval.Milliseconds()
+	if dedupIntervalMs > 0 {
+		if len(dsps) > 0 && dsps[len(dsps)-1].Offset == 0 {
+			return fmt.Errorf("-dedup.minScrapeInterval=%s cannot be used if -downsampling.period=%s contains zero offset", dedupInterval, periods)
+		}
+		// Deduplication is a special case of downsampling with zero offset.
+		dsps = append(dsps, DownsamplingPeriod{
+			Offset:   0,
+			Interval: dedupIntervalMs,
+		})
+	}
+	downsamplingPeriods = dsps
+	return nil
+}
+
+// DownsamplingPeriod describes downsampling period
+type DownsamplingPeriod struct {
+	// Offset in milliseconds from the current time when the downsampling with the given interval must be applied
+	Offset int64
+	// Interval for downsampling - only a single sample is left per each interval
+	Interval int64
+}
+
+// String implements interface
+func (dsp DownsamplingPeriod) String() string {
+	offset := time.Duration(dsp.Offset) * time.Millisecond
+	interval := time.Duration(dsp.Interval) * time.Millisecond
+	return fmt.Sprintf("%s:%s", offset, interval)
+}
+
+func (dsp *DownsamplingPeriod) parse(s string) error {
+	idx := strings.Index(s, ":")
+	if idx <= 0 {
+		return fmt.Errorf("incorrect format for downsampling period: %s, want `offset:interval` format", s)
+	}
+	offsetStr, intervalStr := s[:idx], s[idx+1:]
+	interval, err := metricsql.DurationValue(intervalStr, 0)
+	if err != nil {
+		return fmt.Errorf("incorrect interval: %s format for downsampling interval: %s err: %w", intervalStr, s, err)
+	}
+	offset, err := metricsql.DurationValue(offsetStr, 0)
+	if err != nil {
+		return fmt.Errorf("incorrect duration: %s format for downsampling offset: %s err: %w", offsetStr, s, err)
+	}
+	dsp.Interval = interval
+	dsp.Offset = offset
+	// sanity check
+	if offset > 0 && interval > offset {
+		return fmt.Errorf("downsampling interval=%d cannot exceed offset=%d", dsp.Interval, dsp.Offset)
+	}
+	return nil
+}
+
+var downsamplingPeriods []DownsamplingPeriod
+
+// GetDedupInterval returns dedup interval, which must be applied to samples with the given timestamp.
+func GetDedupInterval(timestamp int64) int64 {
+	dsp := getDownsamplingPeriod(timestamp)
+	return dsp.Interval
+}
+
+// getDownsamplingPeriod returns downsampling period, which must be used for the given timestamp
+func getDownsamplingPeriod(timestamp int64) DownsamplingPeriod {
+	offset := int64(fasttime.UnixTimestamp())*1000 - timestamp
+	for _, dsp := range downsamplingPeriods {
+		if offset >= dsp.Offset {
+			return dsp
+		}
+	}
+	return DownsamplingPeriod{}
+}
+
+func parseDownsamplingPeriods(periods []string) ([]DownsamplingPeriod, error) {
+	if len(periods) == 0 {
+		return nil, nil
+	}
+	var dsps []DownsamplingPeriod
+	for _, period := range periods {
+		var dsp DownsamplingPeriod
+		if err := dsp.parse(period); err != nil {
+			return nil, fmt.Errorf("cannot parse downsampling period %q: %w", period, err)
+		}
+		dsps = append(dsps, dsp)
+	}
+	sort.Slice(dsps, func(i, j int) bool {
+		return dsps[i].Offset > dsps[j].Offset
+	})
+	dspPrev := dsps[0]
+	// sanity checks.
+	for _, dsp := range dsps[1:] {
+		if dspPrev.Interval <= dsp.Interval {
+			return nil, fmt.Errorf("prev downsampling interval %d must be bigger than the next interval %d", dspPrev.Interval, dsp.Interval)
+		}
+		if dspPrev.Offset == dsp.Offset {
+			return nil, fmt.Errorf("duplicate downsampling offset: %d", dsp.Offset)
+		}
+		if dspPrev.Interval%dsp.Interval != 0 {
+			return nil, fmt.Errorf("downsamping intervals must be multiples; prev: %d, current: %d", dspPrev.Interval, dsp.Interval)
+		}
+		dspPrev = dsp
+	}
+	return dsps, nil
+}

lib/storage/downsampling_test.go

@@ -0,0 +1,62 @@
+package storage
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseDownsamplingPeriodsFailure(t *testing.T) {
+	f := func(name string, src []string) {
+		t.Helper()
+		t.Run(name, func(t *testing.T) {
+			if _, err := parseDownsamplingPeriods(src); err == nil {
+				t.Fatalf("want fail for input: %s", strings.Join(src, ","))
+			}
+		})
+	}
+	f("empty duration", []string{"15d"})
+	f("empty interval", []string{":1m"})
+	f("incorrect duration decrease", []string{"30d:15h", "60d:1h"})
+	f("duplicate offset", []string{"30d:15h", "30d:1h"})
+	f("duplicate interval", []string{"60d:1h", "30d:1h"})
+	f("not multiple intervals", []string{"90d:12h", "60:9h", "30d:7h"})
+}
+
+func TestParseDownsamplingPeriodsSuccess(t *testing.T) {
+	f := func(name string, src []string, expected []DownsamplingPeriod) {
+		t.Helper()
+		t.Run(name, func(t *testing.T) {
+			dsps, err := parseDownsamplingPeriods(src)
+			if err != nil {
+				t.Fatalf("cannot parse downsampling configuration for: %s, err: %s", strings.Join(src, ","), err)
+			}
+			assertDownsamplingPeriods(t, expected, dsps)
+		})
+	}
+	f("one period", []string{"30d:1m"}, []DownsamplingPeriod{
+		{Offset: 30 * 24 * 3600 * 1000, Interval: 60 * 1000},
+	})
+	f("three periods", []string{"15d:30s", "30d:1m", "60d:15m"}, []DownsamplingPeriod{
+		{Offset: 60 * 24 * 3600 * 1000, Interval: 15 * 60 * 1000},
+		{Offset: 30 * 24 * 3600 * 1000, Interval: 60 * 1000},
+		{Offset: 15 * 24 * 3600 * 1000, Interval: 30 * 1000},
+	})
+	f("with the same divider periods", []string{"15d:1m", "30d:7m", "60d:14m", "90d:28m"}, []DownsamplingPeriod{
+		{Offset: 90 * 24 * 3600 * 1000, Interval: 28 * 60 * 1000},
+		{Offset: 60 * 24 * 3600 * 1000, Interval: 14 * 60 * 1000},
+		{Offset: 30 * 24 * 3600 * 1000, Interval: 7 * 60 * 1000},
+		{Offset: 15 * 24 * 3600 * 1000, Interval: 60 * 1000},
+	})
+}
+
+func assertDownsamplingPeriods(t *testing.T, want, got []DownsamplingPeriod) {
+	t.Helper()
+	if len(want) != len(got) {
+		t.Fatalf("len mismatch, want: %d, got: %d", len(want), len(got))
+	}
+	for i := 0; i < len(want); i++ {
+		if want[i] != got[i] {
+			t.Fatalf("want period: %s, got period: %s, idx: %d", want[i], got[i], i)
+		}
+	}
+}

lib/storage/metric_name.go

@@ -359,6 +359,12 @@ func (mn *MetricName) String() string {
 	return fmt.Sprintf("%s{%s}", mnCopy.MetricGroup, tagsStr)
 }
 
+// SortAndMarshal sorts mn tags and then marshals them to dst.
+func (mn *MetricName) SortAndMarshal(dst []byte) []byte {
+	mn.sortTags()
+	return mn.Marshal(dst)
+}
+
 // Marshal appends marshaled mn to dst and returns the result.
 //
 // mn.sortTags must be called before calling this function

lib/storage/partition.go

@@ -1105,7 +1105,7 @@ func (pt *partition) runFinalDedup() error {
 func (pt *partition) getRequiredDedupInterval() (int64, int64) {
 	pws := pt.GetParts(nil)
 	defer pt.PutParts(pws)
-	dedupInterval := GetDedupInterval()
+	dedupInterval := GetDedupInterval(pt.tr.MaxTimestamp)
 	minDedupInterval := getMinDedupInterval(pws)
 	return dedupInterval, minDedupInterval
 }
@@ -1214,7 +1214,7 @@ func (pt *partition) mergeParts(pws []*partWrapper, stopCh <-chan struct{}) erro
 	}
 	bsrs = nil
 
-	ph.MinDedupInterval = GetDedupInterval()
+	ph.MinDedupInterval = GetDedupInterval(ph.MaxTimestamp)
 	if err := ph.writeMinDedupInterval(tmpPartPath); err != nil {
 		return fmt.Errorf("cannot store min dedup interval for part %q: %w", tmpPartPath, err)
 	}