Merge remote-tracking branch 'opensource/master' into vmagent-drain-in-memory-queue

vmagent would try to flush in-memory blocks to rw for 5 seconds only
after falling back and store them to the persisted queue

Fixes https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996

app/vmagent/remotewrite/client.go

@@ -2,6 +2,7 @@ package remotewrite
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -330,15 +331,20 @@ func (c *client) runWorker() {
 			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			return
 		case <-c.stopCh:
-			// c must be stopped. Wait for a while in the hope the block will be sent.
-			graceDuration := 5 * time.Second
+			// c must be stopped. Wait up to 5 seconds for the in-flight request to complete.
+			// If it succeeds, drain the remaining in-memory queue before returning.
+			stopCtx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+			defer cancel()
+
 			select {
 			case ok := <-ch:
 				if !ok {
 					// Return unsent block to the queue.
 					c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+				} else {
+					c.drainInMemoryQueue(stopCtx, block[:0])
 				}
-			case <-time.After(graceDuration):
+			case <-stopCtx.Done():
 				// Return unsent block to the queue.
 				c.fq.MustWriteBlockIgnoreDisabledPQ(block)
 			}
@@ -508,6 +514,36 @@ again:
 	goto again
 }
 
+func (c *client) drainInMemoryQueue(stopCtx context.Context, block []byte) {
+	var ok bool
+	for {
+		select {
+		case <-stopCtx.Done():
+			return
+		default:
+		}
+
+		block, ok = c.fq.MustReadInMemoryBlock(block[:0])
+		if !ok {
+			// The in memory queue has already been drained,
+			// or persisted queue is being used.
+			// In this case it is guaranteed that fq will be empty
+			return
+		}
+		if len(block) == 0 {
+			// skip empty data blocks from sending
+			continue
+		}
+
+		// at this stage c.stopCh should be closed
+		// so sendBlock function should not perform retries
+		if ok := c.sendBlock(block); !ok {
+			c.fq.MustWriteBlockIgnoreDisabledPQ(block)
+			return
+		}
+	}
+}
+
 var remoteWriteRejectedLogger = logger.WithThrottler("remoteWriteRejected", 5*time.Second)
 var remoteWriteRetryLogger = logger.WithThrottler("remoteWriteRetry", 5*time.Second)
 

app/vmagent/remotewrite/client_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/golang/snappy"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 )
 
 func TestParseRetryAfterHeader(t *testing.T) {
@@ -37,34 +36,6 @@ func TestParseRetryAfterHeader(t *testing.T) {
 	f(time.Now().Add(10*time.Second).Format("Mon, 02 Jan 2006 15:04:05 FAKETZ"), 0)
 }
 
-func TestInitSecretFlags(t *testing.T) {
-	showRemoteWriteURLOrig := *showRemoteWriteURL
-	defer func() {
-		*showRemoteWriteURL = showRemoteWriteURLOrig
-		flagutil.UnregisterAllSecretFlags()
-	}()
-
-	flagutil.UnregisterAllSecretFlags()
-	*showRemoteWriteURL = false
-	InitSecretFlags()
-	if !flagutil.IsSecretFlag("remotewrite.url") {
-		t.Fatalf("expecting remoteWrite.url to be secret")
-	}
-	if !flagutil.IsSecretFlag("remotewrite.headers") {
-		t.Fatalf("expecting remoteWrite.headers to be secret")
-	}
-
-	flagutil.UnregisterAllSecretFlags()
-	*showRemoteWriteURL = true
-	InitSecretFlags()
-	if flagutil.IsSecretFlag("remotewrite.url") {
-		t.Fatalf("remoteWrite.url must remain visible when -remoteWrite.showURL is set")
-	}
-	if !flagutil.IsSecretFlag("remotewrite.headers") {
-		t.Fatalf("expecting remoteWrite.headers to remain secret")
-	}
-}
-
 func TestRepackBlockFromZstdToSnappy(t *testing.T) {
 	expectedPlainBlock := []byte(`foobar`)
 

app/vmagent/remotewrite/remotewrite.go

@@ -151,8 +151,6 @@ func InitSecretFlags() {
 		// remoteWrite.url can contain authentication codes, so hide it at `/metrics` output.
 		flagutil.RegisterSecretFlag("remoteWrite.url")
 	}
-	// remoteWrite.headers can contain auth headers such as Authorization and API keys.
-	flagutil.RegisterSecretFlag("remoteWrite.headers")
 }
 
 var (
@@ -169,18 +167,6 @@ func Init() {
 	if len(*remoteWriteURLs) == 0 {
 		logger.Fatalf("at least one `-remoteWrite.url` command-line flag must be set")
 	}
-	if *shardByURL && len(*disableOnDiskQueue) > 1 {
-		disableOnDiskQueues := *disableOnDiskQueue
-
-		firstValue := disableOnDiskQueues[0]
-		for _, v := range disableOnDiskQueues[1:] {
-			if firstValue != v {
-				logger.Fatalf("all -remoteWrite.url targets must have the same -remoteWrite.disableOnDiskQueue setting when -remoteWrite.shardByURL is enabled; " +
-					"either enable or disable -remoteWrite.disableOnDiskQueue for all targets")
-			}
-		}
-	}
-
 	if limit := getMaxHourlySeries(); limit > 0 {
 		hourlySeriesLimiter = bloomfilter.NewLimiter(limit, time.Hour)
 		_ = metrics.NewGauge(`vmagent_hourly_series_limit_max_series`, func() float64 {
@@ -513,9 +499,7 @@ func tryPush(at *auth.Token, wr *prompb.WriteRequest, forceDropSamplesOnFailure
 //
 // calculateHealthyRwctxIdx will rely on the order of rwctx to be in ascending order.
 func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) ([]*remoteWriteCtx, bool) {
-	// When -remoteWrite.shardByURL=true always use all configured remote writes to preserve stable metrics distribution across shards.
-	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507
-	if !disableOnDiskQueueAny || *shardByURL {
+	if !disableOnDiskQueueAny {
 		return rwctxsGlobal, true
 	}
 
@@ -530,6 +514,12 @@ func getEligibleRemoteWriteCtxs(tss []prompb.TimeSeries, forceDropSamplesOnFailu
 				return nil, false
 			}
 			rowsCount := getRowsCount(tss)
+			if *shardByURL {
+				// Todo: When shardByURL is enabled, the following metrics won't be 100% accurate. Because vmagent don't know
+				// which rwctx should data be pushed to yet. Let's consider the hashing algorithm fair and will distribute
+				// data to all rwctxs evenly.
+				rowsCount = rowsCount / len(rwctxsGlobal)
+			}
 			rwctx.rowsDroppedOnPushFailure.Add(rowsCount)
 		}
 	}

app/vmalert-tool/unittest/unittest.go

@@ -61,7 +61,7 @@ func UnitTest(files []string, disableGroupLabel bool, externalLabels []string, e
 	}
 	eu, err := url.Parse(externalURL)
 	if err != nil {
-		logger.Fatalf("failed to parse external URL: %s", err)
+		logger.Fatalf("failed to parse external URL: %w", err)
 	}
 	if err := templates.Load([]string{}, *eu); err != nil {
 		logger.Fatalf("failed to load template: %v", err)

app/vmalert/notifier/config_watcher.go

@@ -105,7 +105,7 @@ func (cw *configWatcher) add(typeK TargetType, interval time.Duration, targetsFn
 			}
 			targetMetadata, errors := getTargetMetadata(targetsFn, cw.cfg)
 			for _, err := range errors {
-				logger.Errorf("failed to init notifier for %q: %s", typeK, err)
+				logger.Errorf("failed to init notifier for %q: %w", typeK, err)
 			}
 			cw.updateTargets(typeK, targetMetadata, cw.cfg, cw.genFn)
 		}
@@ -274,7 +274,7 @@ func (cw *configWatcher) updateTargets(key TargetType, targetMts map[string]targ
 	for addr, metadata := range targetMts {
 		am, err := NewAlertManager(addr, genFn, cfg.HTTPClientConfig, metadata.alertRelabelConfigs, cfg.Timeout.Duration())
 		if err != nil {
-			logger.Errorf("failed to init %s notifier with addr %q: %s", key, addr, err)
+			logger.Errorf("failed to init %s notifier with addr %q: %w", key, addr, err)
 			continue
 		}
 		updatedTargets = append(updatedTargets, Target{

dashboards/victoriametrics-cluster.json

@@ -3109,6 +3109,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/victoriametrics.json

@@ -3406,6 +3406,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/victoriametrics-cluster.json

@@ -3110,6 +3110,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/victoriametrics.json

@@ -3407,6 +3407,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/vmagent.json

@@ -2946,6 +2946,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vm/vmalert.json

@@ -2324,6 +2324,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vmagent.json

@@ -2945,6 +2945,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

dashboards/vmalert.json

@@ -2323,6 +2323,7 @@
                   "mode": "off"
                 }
               },
+              "decimals": 0,
               "links": [],
               "mappings": [],
               "min": 0,

deployment/docker/vmanomaly/vmanomaly-integration/compose.yml

@@ -59,7 +59,7 @@ services:
       - '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
     restart: always
   vmanomaly:
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.3
     depends_on:
       - "victoriametrics"
     ports:

docs/anomaly-detection/CHANGELOG.md

@@ -14,15 +14,6 @@ aliases:
 ---
 Please find the changelog for VictoriaMetrics Anomaly Detection below.
 
-## v1.29.4
-Released: 2026-05-15
-
-- IMPROVEMENT: Optimized [BacktestingScheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) exact mode for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) (which are also used in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/)-triggered tasks), leading to 2-200x faster results and more responsive UI interactions.
-
-- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.6.1](https://docs.victoriametrics.com/anomaly-detection/ui/#v161) to [v1.7.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v170), see respective [release notes](https://docs.victoriametrics.com/anomaly-detection/ui/#v170) for details. Notable changes include iterative anomaly detection with progress bar, model inference speedups, improved AI Copilot UX and various bug fixes.
-
-- BUGFIX: Made model-level `queries` and `schedulers` attachment semantics explicit and safer. Omitting these args still keeps the legacy default of attaching a model to all queries or schedulers with a warning, while `queries: null`, `queries: []`, `schedulers: null`, and `schedulers: []` now mean the model is intentionally unattached and will be dropped from active execution. Use `queries: all` or `schedulers: all` to request all attachments explicitly. Orphan models, queries, and schedulers are logged and removed during config validation, including hot reload.
-
 ## v1.29.3
 Released: 2026-04-16
 

docs/anomaly-detection/FAQ.md

@@ -423,7 +423,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.3
     # ...
     restart: always
     volumes:
@@ -641,7 +641,7 @@ options:
 Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.4 && docker image tag victoriametrics/vmanomaly:v1.29.4 vmanomaly
+docker pull victoriametrics/vmanomaly:v1.29.3 && docker image tag victoriametrics/vmanomaly:v1.29.3 vmanomaly
 ```
 
 ```sh

docs/anomaly-detection/QuickStart.md

@@ -122,7 +122,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
 1. Pull Docker image:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.4
+docker pull victoriametrics/vmanomaly:v1.29.3
 ```
 
 2. Create the license file with your license key.
@@ -142,7 +142,7 @@ docker run -it \
     -v ./license:/license \
     -v ./config.yaml:/config.yaml \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.4 \
+    victoriametrics/vmanomaly:v1.29.3 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -159,7 +159,7 @@ docker run -it \
     -e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
     -e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
     -p 8490:8490 \
-    victoriametrics/vmanomaly:v1.29.4 \
+    victoriametrics/vmanomaly:v1.29.3 \
     /config.yaml \
     --licenseFile=/license \
     --loggerLevel=INFO \
@@ -172,7 +172,7 @@ services:
   # ...
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.3
     # ...
     restart: always
     volumes:

docs/anomaly-detection/UI.md

@@ -315,7 +315,7 @@ docker run -it --rm \
   -e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
   -p 8080:8080 \
   -p 8490:8490 \
-  victoriametrics/vmanomaly:v1.29.4 \
+  victoriametrics/vmanomaly:v1.29.3 \
   vmanomaly_config.yaml
 ```
 
@@ -640,26 +640,6 @@ If the **results** look good and the **model configuration should be deployed in
 
 ## Changelog
 
-### v1.7.0
-Released: 2026-05-15
-
-vmanomaly version: [v1.29.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1294)
-
-- FEATURE: Improved [AI Copilot](#ai-assistance) to 
-  - support textual attachments, such as configuration files, query examples, model outputs, etc.
-  - operate with current time when suggesting time range changes, e.g. "current month" or "from the beginning of the last month until now"
-- FEATURE: improve infer jobs scheduled from UI to
-  - run 2-100x faster for [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) depending on a configuration vs [1.6.1](#v161) timings
-  - show stage-aware progress bar, e.g. "getting data", "fitting model", "inferring on chunk".
-- IMPROVEMENT: "Advanced Options" design is improved, section is collapsed by default to save vertical space, yet can be encoded as always open in UI state URL.
-- IMPROVEMENT: dropdowns in UI are now searcheable and constrained in size for better UX, especially when many options are available (e.g. models list, tenants list, etc.).
-- BUGFIX: in generated config ("Show Config" menu)
-  - special YAML values (like `-.inf`, `.inf`, `.nan`) are now properly quoted and formatted to avoid JSON converting issues when copied to Kubernetes CRs, which previously lead to rejection of the manifest before vmanomaly can consume it.
-  - with Data Source selected as "Logs/Traces" `reader.class` is now correctly set to `vlogs` (previously `vm`) in generated config, when "model only" toggle is turned off.
-- BUGFIX: now "Hide Common Labels" toggle in "Table View" works correctly and does not show common labels in the legend columns when turned on.
-- BUGFIX: improved dropdowns for better accessibility to be searcheable and constrained in size.
-- BUGFIX: "Prettify query" now works correctly for "Metrics" datasources.
-
 ### v1.6.1
 Released: 2026-04-16
 

docs/anomaly-detection/components/models.md

@@ -1265,7 +1265,7 @@ monitoring:
 Let's pull the docker image for `vmanomaly`:
 
 ```sh
-docker pull victoriametrics/vmanomaly:v1.29.4
+docker pull victoriametrics/vmanomaly:v1.29.2
 ```
 
 Now we can run the docker container putting as volumes both config and model file:
@@ -1279,7 +1279,7 @@ docker run -it \
 -v $(PWD)/license:/license \
 -v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
 -v $(PWD)/custom.yaml:/config.yaml \
-victoriametrics/vmanomaly:v1.29.4 /config.yaml \
+victoriametrics/vmanomaly:v1.29.2 /config.yaml \
 --licenseFile=/license
 --watch
 ```

docs/anomaly-detection/guides/guide-vmanomaly-vmalert/README.md

@@ -395,7 +395,7 @@ services:
     restart: always
   vmanomaly:
     container_name: vmanomaly
-    image: victoriametrics/vmanomaly:v1.29.4
+    image: victoriametrics/vmanomaly:v1.29.3
     depends_on:
       - "victoriametrics"
     ports:

docs/victoriametrics/Articles.md

@@ -33,7 +33,6 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
 * [Percona: How do We Keep Metrics for a Long Time in VictoriaMetrics](https://www.youtube.com/watch?v=SGZjY7xgDwE)
 * [Miro: Prometheus High Availability and Fault Tolerance strategy, long term storage with VictoriaMetrics](https://medium.com/miro-engineering/prometheus-high-availability-and-fault-tolerance-strategy-long-term-storage-with-victoriametrics-82f6f3f0409e)
 * [ZERODHA: Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/)
-* [ZERODHA: Monitoring K8s with Victoriametrics](https://www.youtube.com/watch?v=ZJQYW-cFOms)
 * [Criteo: VictoriaMetrics, a stress-free Prometheus Remote Storage for 1 Billion metrics](https://medium.com/criteo-engineering/victoriametrics-a-prometheus-remote-storage-solution-57081a3d8e61)
 * [Abios Gaming: Choosing a Time Series Database for High Cardinality Aggregations](https://abiosgaming.com/press/high-cardinality-aggregations/)
 * [Cybozu: Monitoring Kubernetes clusters with VictoriaMetrics and Grafana](https://blog.cybozu.io/entry/2021/03/18/115743)

docs/victoriametrics/changelog/CHANGELOG.md

@@ -30,15 +30,12 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. Thanks to @alexei38 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918).
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
-* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly establish [mtls](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection) connection between vmstorage and vminsert. Regression was introduced in v1.130.0 release for the enterprise version of vmstorage. See [#10958](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10958)
 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 
 ## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
 

docs/victoriametrics/enterprise.md

@@ -15,28 +15,25 @@ aliases:
 - /enterprise/
 ---
 
-VictoriaMetrics, [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) and [VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) components are provided
+VictoriaMetrics and [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/) components are provided
 in two kinds - [Community edition](https://victoriametrics.com/products/open-source/) and [Enterprise edition](https://victoriametrics.com/products/enterprise/).
 
-The community components are open source and are free to use:
+VictoriaMetrics and VictoriaLogs community components are open source and are free to use:
 
 - See [VictoriaMetrics source code](https://github.com/VictoriaMetrics/VictoriaMetrics/) and [VictoriaMetrics license](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE).
 - See [VictoriaLogs source code](https://github.com/VictoriaMetrics/VictoriaLogs/) and [VictoriaLogs license](https://github.com/VictoriaMetrics/VictoriaLogs/blob/master/LICENSE).
-- See [VictoriaTraces source code](https://github.com/VictoriaMetrics/VictoriaTraces/) and [VictoriaLogs license](https://github.com/VictoriaMetrics/VictoriaTraces/blob/master/LICENSE).
 
-Both open-source and enterprise components are available at the following places:
+Enterprise components of VictoriaMetrics and VictoriaLogs are available at the following places:
 
-- Binary executables are available at the corresponding releases pages:
-  - [releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-  - [releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
-  - [releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest)
+- Binary executables are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+  and [the release page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
 - Docker images are available at [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
 
 Enterprise executables and Docker images have `enterprise` suffix in their names and tags.
 
 ## Valid cases for VictoriaMetrics Enterprise
 
-The use of Enterprise components is permitted in the following cases:
+The use of Enterprise components of VictoriaMetrics and VictoriaLogs is permitted in the following cases:
 
 - Evaluation use in non-production setups. Please, request a [trial license](https://victoriametrics.com/products/enterprise/trial/)
   and then pass it via `-license` or `-licenseFile` command-line flags as described [in these docs](#running-victoriametrics-enterprise).
@@ -107,26 +104,25 @@ Contact us via [this page](https://victoriametrics.com/products/enterprise/) if
 
 ## Running VictoriaMetrics Enterprise
 
-Enterprise components are available in the following forms:
+Enterprise components of VictoriaMetrics and VictoriaLogs are available in the following forms:
 
-- [Binary releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases)
-- [Docker images](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)
-- [Helm charts](https://docs.victoriametrics.com/victoriametrics/enterprise/#helm-charts)
-- [Kubernetes operator](https://docs.victoriametrics.com/victoriametrics/enterprise/#kubernetes-operator)
+- [Binary releases](#binary-releases)
+- [Docker images](#docker-images)
+- [Helm charts](#helm-charts)
+- [Kubernetes operator](#kubernetes-operator)
 
 ### Binary releases
 
-It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
-Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
-[the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
-and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
+Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
+and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
 Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.143.0-enterprise.tar.gz`.
 
 In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
 from the corresponding releases page and unpack it. Then run the unpacked binary.
 
-All the Enterprise components require specifying the following command-line flags:
+All the Enterprise components of VictoriaMetrics and VictoriaLogs require specifying the following command-line flags:
 
 - `-license` - this flag accepts VictoriaMetrics Enterprise license key, which can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/)
 - `-licenseFile` - this flag accepts a path to file with VictoriaMetrics Enterprise license key,
@@ -152,13 +148,13 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
 
 ### Docker images
 
-It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
 Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.143.0-enterprise`.
 
 In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
-flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
+flag as described in the [binary-releases](#binary-releases) section.
 
 Enterprise license key can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/).
 
@@ -198,12 +194,12 @@ The example assumes that the license file is stored at `/vm-license` on the host
 
 ### Helm charts
 
-It is allowed to run Enterprise components in cases [listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Helm charts for Enterprise components are available in our VictoriaMetrics [Helm repository](https://github.com/VictoriaMetrics/helm-charts).
 
 In order to run Enterprise helm chart it is required to provide the license key via `license` value in `values.yaml` file
-and adjust the image tag to the Enterprise one as described in the [docker-images](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images) section.
+and adjust the image tag to the Enterprise one as described in the [docker-images](#docker-images) section.
 
 Enterprise license key can be obtained at [this page](https://victoriametrics.com/products/enterprise/trial/).
 
@@ -254,7 +250,7 @@ Note that the license key provided by using secret is mounted in a file. This al
 
 ### Kubernetes operator
 
-It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
+It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
 
 Enterprise components can be deployed via [VictoriaMetrics operator](https://docs.victoriametrics.com/operator/).
 In order to use Enterprise components it is required to provide the license key via `license` field and adjust the image tag to the enterprise one.
@@ -317,7 +313,7 @@ See the full list of the CRD specifications in the [Operator API](https://docs.v
 
 ### Updating the license key
 
-Updating the license key for Enterprise components depends on the way
+Updating the license key for VictoriaMetrics and VictoriaLogs Enterprise components depends on the way
 the license key is provided to the component:
 - If the license key is provided via `-license` command-line flag, then the component should be restarted
   with the new license key.
@@ -355,7 +351,7 @@ Example Docker image:
 
 ## Monitoring license expiration
 
-All the Enterprise components expose the following metrics at the `/metrics` page:
+All the VictoriaMetrics and VictoriaLogs Enterprise components expose the following metrics at the `/metrics` page:
 
 - `vm_license_expires_at` - license expiration date in unix timestamp format
 - `vm_license_expires_in_seconds` - the number of seconds left until the license expires

docs/victoriametrics/goals.md

@@ -12,21 +12,21 @@ aliases:
 ---
 ## Goals
 
-1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics products in the most efficient way**.
+1. The main goal - **to help users and [clients](https://docs.victoriametrics.com/victoriametrics/enterprise/) using VictoriaMetrics and VictoriaLogs component in the most efficient way**.
 1. Fixing bugs in the essential functionality of VictoriaMetrics components. Small usability bugs are usually the most annoying,
    so they **must be fixed first**. Bugs, which affect a small number of users at some rare edge cases, can be fixed later.
-1. Improving [public docs for VictoriaMetrics products](https://docs.victoriametrics.com).
-   so users could find answers to their questions via Google or any other AI-powered web search without the need
+1. Improving [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/docs) for VictoriaMetrics components,
+   so users could find answers to their questions via Google or [Perplexity](https://www.perplexity.ai/) without the need
    to ask these questions at our [support channels](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#community-and-contributions).
-1. Simplifying usage of VictoriaMetrics products without breaking backwards compatibility, so users could regularly
-   upgrade to [the latest available release](https://docs.victoriametrics.com/victoriametrics/changelog/) and become happier.
-1. Improving the usability for the existing functionality of VictoriaMetrics components.
+1. Simplifying usage of VictoriaMetrics components without breaking backwards compatibility, so users could regularly
+   upgrade to [the latest available release](https://docs.victoriametrics.com/victoriametrics/changelog/) and remain happy.
+1. Improving usability for the existing functionality of VictoriaMetrics components.
 1. Improving the readability and maintainability of the code base by removing unnecessary abstractions and simplifying the code whenever possible.
 1. Improving development velocity by optimizing and simplifying CI/CD tasks, so they take less time to execute and debug.
 
 ## Non-goals
 
-1. Convincing people to use VictoriaMetrics products when there are better suited solutions exist for their tasks,
+1. Convincing people to use VictoriaMetrics components when there are better suited solutions exist for their tasks,
    since users will become angry at VictoriaMetrics after they discover better solutions.
 1. Breaking links to [VictoriaMetrics docs](https://docs.victoriametrics.com/victoriametrics/), since users will be unhappy seeing 404 page
    or unexpected results after they click some old link somewhere on the Internet or in the internal knowledge base.
@@ -35,29 +35,25 @@ aliases:
    since these features may break the essential functionality of VictoriaMetrics components, so a big share
    of the existing users may become unhappy after the upgrade.
 1. Adding unnecessary abstractions, since they may worsen project maintainability in the future.
-1. Implementing all the features users ask. These features should fit [the goals](https://docs.victoriametrics.com/victoriametrics/goals/#goals) of VictoriaMetrics.
+1. Implementing all the features users ask. These features should fit [the goals](#goals) of VictoriaMetrics.
    Other feature requests must be closed as `won't implement`, with the link to this page.
-1. Merging all the pull requests users submit. These pull requests should fit [the goals](https://docs.victoriametrics.com/victoriametrics/goals/#goals) of VictoriaMetrics.
+1. Merging all the pull requests users submit. These pull requests should fit [the goals](#goals) of VictoriaMetrics.
    Other pull requests must be closed as `won't merge`, with the link to this page.
 1. Slowing down and complicating CI/CD pipelines with non-essential tasks, since this results in development velocity slowdown.
 1. Introducing non-essential requirements, since this slows down development velocity.
 
 ## VictoriaMetrics proverbs
 
-- **A small usability improvement is more valuable than a major new feature.**
-  The usability fix makes happy existing users. Non-trivial feature may make happy some new users,
-  while it may make upset a big share of existing users if the feature breaks the essential functionality
-  of VictoriaMetrics products or makes it less efficient.
+- **Small usability fix is better than non-trivial feature.** Usability fix makes happy existing users.
+  Non-trivial feature may make happy some new users, while it may make upset a big share of existing users
+  if the feature breaks some essential functionality of VictoriaMetrics components or makes it less efficient.
 
-- **Having clean and concise documentation is more valuable than having a lot of great but undocumented features.**
-  Good docs help users discovering new functionality and dealing with VictoriaMetrics products in the most efficient way.
-  Nobody uses new shiny features if they aren't documented properly.
+- **Good docs are better than new shiny feature.** Good docs help users discovering new functionality and dealing
+  with VictoriaMetrics components in the most efficient way. Nobody uses new shiny feature if it isn't documented properly.
 
-- **A simple solution is better than a smart one.**
-  The simple solution is easier to setup, operate, debug and troubleshoot than the smart solution. This saves users' time, costs and nerve cells.
+- **Happy users are more important than the momentary revenue.** Happy users spread the word about VictoriaMetrics,
+  so more people convert to VictoriaMetrics users. Happy users are eager to become happy [customers](https://docs.victoriametrics.com/victoriametrics/enterprise/).
+  This increases long-term revenue.
 
-- **Happy users are more important than the short-term profit.**
-  Happy users spread the word about VictoriaMetrics, so more people convert to VictoriaMetrics users.
-  Happy users are eager to become happy [customers](https://docs.victoriametrics.com/victoriametrics/enterprise/)
-  over time. This increases long-term profit.
-  Upset users may be forced to become customers, but they will constantily search for the opportunity to switch to competing solutions.
+- **Simple solution is better than smart solution.** Simple solution is easier to setup, operate, debug and troubleshoot than the smart solution.
+  This saves users' time, costs and nerve cells.

lib/mergeset/table.go

@@ -1538,11 +1538,11 @@ func (tb *Table) MustCreateSnapshotAt(dstDir string) {
 	srcDir := tb.path
 	srcDir, err = filepath.Abs(srcDir)
 	if err != nil {
-		logger.Panicf("FATAL: cannot obtain absolute dir for %q: %s", srcDir, err)
+		logger.Panicf("FATAL: cannot obtain absolute dir for %q: %w", srcDir, err)
 	}
 	dstDir, err = filepath.Abs(dstDir)
 	if err != nil {
-		logger.Panicf("FATAL: cannot obtain absolute dir for %q: %s", dstDir, err)
+		logger.Panicf("FATAL: cannot obtain absolute dir for %q: %w", dstDir, err)
 	}
 	prefix := srcDir + string(filepath.Separator)
 	if strings.HasPrefix(dstDir, prefix) {

lib/persistentqueue/fastqueue.go

@@ -228,7 +228,9 @@ func (fq *FastQueue) tryWriteBlock(block []byte, ignoreDisabledPQ bool) bool {
 	return true
 }
 
-// MustReadBlock reads the next block from fq to dst and returns it.
+// MustReadBlock reads the next block from fq into dst and returns it.
+// It first reads from the in-memory queue, then checks file-based queue.
+// It blocks until a block is available or the stop deadline is exceeded, in which case it returns (dst, false).
 func (fq *FastQueue) MustReadBlock(dst []byte) ([]byte, bool) {
 	fq.mu.Lock()
 	defer fq.mu.Unlock()
@@ -238,15 +240,7 @@ func (fq *FastQueue) MustReadBlock(dst []byte) ([]byte, bool) {
 			return dst, false
 		}
 		if len(fq.ch) > 0 {
-			if n := fq.pq.GetPendingBytes(); n > 0 {
-				logger.Panicf("BUG: the file-based queue must be empty when the inmemory queue is non-empty; it contains %d pending bytes", n)
-			}
-			bb := <-fq.ch
-			fq.pendingInmemoryBytes -= uint64(len(bb.B))
-			fq.lastInmemoryBlockReadTime = fasttime.UnixTimestamp()
-			dst = append(dst, bb.B...)
-			blockBufPool.Put(bb)
-			return dst, true
+			return fq.mustReadInMemoryBlockLocked(dst), true
 		}
 		if n := fq.pq.GetPendingBytes(); n > 0 {
 			data, ok := fq.pq.MustReadBlockNonblocking(dst)
@@ -265,6 +259,35 @@ func (fq *FastQueue) MustReadBlock(dst []byte) ([]byte, bool) {
 	}
 }
 
+// MustReadInMemoryBlock reads the next block from the in-memory queue into dst and returns it.
+// It returns (dst, true) if a block was available, or (nil, false) if the in-memory queue is empty.
+// It does not block waiting for new blocks.
+func (fq *FastQueue) MustReadInMemoryBlock(dst []byte) ([]byte, bool) {
+	fq.mu.Lock()
+	defer fq.mu.Unlock()
+
+	if len(fq.ch) > 0 {
+		return fq.mustReadInMemoryBlockLocked(dst), true
+	}
+
+	return nil, false
+}
+
+func (fq *FastQueue) mustReadInMemoryBlockLocked(dst []byte) []byte {
+	if len(fq.ch) == 0 {
+		logger.Panicf("BUG: the function must not be called when in-memory queue is empty. Caller should verify the queue len upfront")
+	}
+	if n := fq.pq.GetPendingBytes(); n > 0 {
+		logger.Panicf("BUG: the file-based queue must be empty when the in-memory queue is non-empty; it contains %d pending bytes", n)
+	}
+	bb := <-fq.ch
+	fq.pendingInmemoryBytes -= uint64(len(bb.B))
+	fq.lastInmemoryBlockReadTime = fasttime.UnixTimestamp()
+	dst = append(dst, bb.B...)
+	blockBufPool.Put(bb)
+	return dst
+}
+
 // Dirname returns the directory name for persistent queue.
 func (fq *FastQueue) Dirname() string {
 	return filepath.Base(fq.pq.dir)

lib/storage/index_db.go

@@ -764,7 +764,7 @@ func filterLabelValues(lvs map[string]struct{}, tf *tagFilter, key string) {
 		b = marshalTagValue(b, bytesutil.ToUnsafeBytes(lv))
 		ok, err := tf.match(b)
 		if err != nil {
-			logger.Panicf("BUG: cannot match label %q=%q with tagFilter %s: %s", key, lv, tf.String(), err)
+			logger.Panicf("BUG: cannot match label %q=%q with tagFilter %s: %w", key, lv, tf.String(), err)
 		}
 		if !ok {
 			delete(lvs, lv)

lib/storage/index_db_legacy.go

@@ -141,7 +141,7 @@ func (is *indexSearch) legacyContainsTimeRangeSlow(prefixBuf *bytesutil.ByteBuff
 	ts.Seek(prefixBuf.B)
 	if !ts.NextItem() {
 		if err := ts.Error(); err != nil {
-			logger.Panicf("FATAL: error when searching for minDate=%d, prefix %q: %s", minDate, prefixBuf.B, err)
+			logger.Panicf("FATAL: error when searching for minDate=%d, prefix %q: %w", minDate, prefixBuf.B, err)
 		}
 		return false
 	}

lib/storage/metricnamestats/tracker.go

@@ -106,7 +106,7 @@ func loadFrom(loadPath string, maxSizeBytes uint64) (*Tracker, error) {
 	}
 	defer func() {
 		if err := zr.Close(); err != nil {
-			logger.Panicf("FATAL: cannot close gzip reader: %s", err)
+			logger.Panicf("FATAL: cannot close gzip reader: %w", err)
 		}
 	}()
 

lib/storage/table.go

@@ -517,7 +517,7 @@ func (tb *table) historicalMergeWatcher() {
 
 			logger.Infof("start %s for partition (%s, %s)", strings.Join(logContext, " and "), pt.bigPartsPath, pt.smallPartsPath)
 			if err := pt.ForceMergeAllParts(tb.stopCh); err != nil {
-				logger.Errorf("cannot %s for partition (%s, %s): %s", strings.Join(logErrContext, " and "), pt.bigPartsPath, pt.smallPartsPath, err)
+				logger.Errorf("cannot %s for partition (%s, %s): %w", strings.Join(logErrContext, " and "), pt.bigPartsPath, pt.smallPartsPath, err)
 			}
 			logger.Infof("finished %s for partition (%s, %s) in %.3f seconds", strings.Join(logContext, " and "), pt.bigPartsPath, pt.smallPartsPath, time.Since(t).Seconds())