Update lib/persistentqueue/persistentqueue.go

Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Signed-off-by: Nikolay <nik@victoriametrics.com>

app/vmui/packages/vmui/src/pages/CardinalityPanel/CardinalityTotals/CardinalityTotals.tsx

@@ -32,7 +32,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
   const match = searchParams.get("match");
   const focusLabel = searchParams.get("focusLabel");
   const isMetric = /__name__/.test(match || "");
-  const showMetricNameStats = !(match || focusLabel);
+
   const progress = totalSeries / totalSeriesAll * 100;
   const diff = totalSeries - totalSeriesPrev;
   const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
@@ -56,7 +56,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
     },
   ].filter(t => t.display);
 
-  if (!totals.length && !showMetricNameStats) {
+  if (!totals.length) {
     return null;
   }
 
@@ -75,7 +75,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           <h4 className="vm-cardinality-totals-card__title">
             {info && (
               <Tooltip title={<p className="vm-cardinality-totals-card__tooltip">{info}</p>}>
-                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon /></div>
+                <div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon/></div>
               </Tooltip>
             )}
             {title}
@@ -99,10 +99,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
           )}
         </div>
       ))}
-      {
-        showMetricNameStats &&
-        <CardinalityMetricNameStats metricNameStats={metricNameStats} />
-      }
+      <CardinalityMetricNameStats metricNameStats={metricNameStats}/>
     </div>
   );
 };

docs/victoriametrics/Quick-Start.md

@@ -14,178 +14,6 @@ aliases:
 - /quick-start/index.html
 - /quick-start/
 ---
-There are two ways to get started with VictoriaMetrics:
-
-* [Try it locally](https://docs.victoriametrics.com/victoriametrics/quick-start/#try-it-locally) - if you just want to see how VictoriaMetrics works,
-  go with the single binary: download it, start it with one command, and see live metrics
-  in the built-in UI in a couple of minutes. No Docker, no configuration files
-  and no extra components are required;
-* [Install it](https://docs.victoriametrics.com/victoriametrics/quick-start/#how-to-install) - if you want to set up VictoriaMetrics for real use,
-  pick a distribution (single-node, cluster or cloud) and an installation method
-  (Docker, Helm, binary releases, etc.).
-
-If you'd rather not install anything at all, try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) -
-a list of publicly available playgrounds for VictoriaMetrics software.
-
-Whichever way you choose, you may also find interesting the other sections of this page,
-like how to [write](https://docs.victoriametrics.com/victoriametrics/quick-start/#write-data) and [query](https://docs.victoriametrics.com/victoriametrics/quick-start/#query-data) data,
-[alerting](https://docs.victoriametrics.com/victoriametrics/quick-start/#alerting),
-[data migration](https://docs.victoriametrics.com/victoriametrics/quick-start/#data-migration) from other TSDBs,
-and [productionization](https://docs.victoriametrics.com/victoriametrics/quick-start/#productionization)
-best practices for running VictoriaMetrics in production.
-
-## Try it locally
-
-The fastest way to try VictoriaMetrics on your own machine is its binary - the only thing needed to run it.
-
-### Step 1: Download the binary
-
-Create a directory for this test drive, so all the files created along the way stay in one place:
-
-```sh
-mkdir vm-quick-start && cd vm-quick-start
-```
-
-Download the `victoria-metrics-<os>-<arch>-<version>.tar.gz` archive for your OS and architecture
-from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
-and unpack it. It contains a single `victoria-metrics-prod` binary.
-
-For example, on Linux with `amd64` architecture:
-
-```sh
-wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0.tar.gz
-tar xzf victoria-metrics-linux-amd64-v1.147.0.tar.gz
-```
-
-The binary is self-contained and requires no installation - it is ready to run as is.
-
-### Step 2: Start VictoriaMetrics
-
-Starting VictoriaMetrics is as simple as executing the binary, with no arguments at all.
-But since it is nicer to have some data to explore right after the start, let's also enable self-scraping
-via the `-selfScrapeInterval` command-line flag, so VictoriaMetrics collects metrics about itself:
-
-```sh
-./victoria-metrics-prod -selfScrapeInterval=10s
-```
-
-VictoriaMetrics prints a couple of dozen log lines on start, describing the storage, caches
-and memory limits it sets up. Look for these two lines confirming that it is up and running:
-
-```sh
-2026-07-10T16:55:06.615Z    info    app/victoria-metrics/main.go:102    started VictoriaMetrics in 0.019 seconds
-...
-2026-07-10T16:55:06.615Z    info    lib/httpserver/httpserver.go:148    started server at http://0.0.0.0:8428/
-```
-
-That's it - VictoriaMetrics is running, listening on port `8428` and scraping its own metrics
-(CPU and memory usage, the number of stored metrics, request rates, etc.) every 10 seconds.
-If you list the `vm-quick-start` directory, you can see a new `victoria-metrics-data` directory
-created next to the binary - this is where the collected data is stored.
-
-Now that metrics are being collected, it's time to look at them.
-
-### Step 3: Explore the metrics
-
-Open [http://localhost:8428/vmui](http://localhost:8428/vmui) in your browser to access [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) -
-the built-in UI for querying and graphing metrics. You should see its query page:
-
-![vmui query page](quick-start-vmui.webp)
-
-Self-scraped metrics become queryable within ~30 seconds after the start.
-
-Try the following:
-
-* Open the [metrics explorer](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#metrics-explorer)
-  at [http://localhost:8428/vmui/#/metrics](http://localhost:8428/vmui/#/metrics) to browse all collected metrics;
-* Enter a query in the input field at [http://localhost:8428/vmui](http://localhost:8428/vmui) and press `Enter`. For example:
-  * `process_resident_memory_bytes` - memory used by VictoriaMetrics;
-  * `rate(process_cpu_seconds_total)` - its CPU usage;
-  * `vm_rows{type=~"storage/.+"}` - the number of stored [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples).
-
-Queries are written in [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) -
-a PromQL-compatible query language, so any PromQL query works here as well.
-
-So far the only available metrics are the ones VictoriaMetrics reports about itself - let's collect something more interesting.
-
-### Step 4 (optional): Collect metrics from your system
-
-Self-scraped metrics only describe VictoriaMetrics itself. To monitor your machine (CPU, memory, disk, network),
-run [node_exporter](https://github.com/prometheus/node_exporter) - the standard Prometheus exporter for host metrics -
-and let VictoriaMetrics scrape it. Single-node VictoriaMetrics has a built-in
-[Prometheus-compatible scraper](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter),
-so no other components are needed.
-
-1. Download the `node_exporter-<version>.<os>-<arch>.tar.gz` archive for your OS and architecture
-   from the [releases page](https://github.com/prometheus/node_exporter/releases/latest) and unpack it.
-   For example, on Linux with `amd64` architecture:
-
-   ```sh
-   wget https://github.com/prometheus/node_exporter/releases/download/v1.12.0/node_exporter-1.12.0.linux-amd64.tar.gz
-   tar xzf node_exporter-1.12.0.linux-amd64.tar.gz
-   ```
-
-   Unlike VictoriaMetrics, it unpacks into its own directory. Start it from there:
-
-   ```sh
-   ./node_exporter-1.12.0.linux-amd64/node_exporter
-   ```
-
-   It exposes host metrics at [http://localhost:9100/metrics](http://localhost:9100/metrics).
-
-1. Create a `scrape.yaml` file with the following contents:
-
-   ```yaml
-   scrape_configs:
-   - job_name: node-exporter
-     static_configs:
-     - targets:
-       - localhost:9100
-   ```
-
-1. Restart VictoriaMetrics with the `-promscrape.config` command-line flag pointing to this file:
-
-   ```sh
-   ./victoria-metrics-prod -selfScrapeInterval=10s -promscrape.config=scrape.yaml
-   ```
-
-Check [http://localhost:8428/targets](http://localhost:8428/targets) - the `node-exporter` target should have `state: up`.
-The target shows up as `state: down` until the first scrape happens, which can take some seconds -
-just reload the page a bit later.
-Now query host metrics in [vmui](http://localhost:8428/vmui). For example:
-
-* `node_memory_MemAvailable_bytes` - available memory on your machine;
-* `100 - avg(rate(node_cpu_seconds_total{mode="idle"})) * 100` - overall CPU usage in percent.
-
-See [scrape config examples](https://docs.victoriametrics.com/victoriametrics/scrape_config_examples/) for more advanced scrape configurations.
-
-Scraping pulls metrics from targets, but it is not the only way to get data in - metrics can also be pushed directly.
-
-### Step 5 (optional): Push your own metrics
-
-VictoriaMetrics also accepts metrics pushed via [many popular protocols](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model).
-For example, insert a measurement using the InfluxDB line protocol with a plain `curl` command:
-
-```sh
-curl -d 'room_temperature,room=kitchen value=21.5' http://localhost:8428/write
-```
-
-Then query `room_temperature_value` in [vmui](http://localhost:8428/vmui) to see it.
-Take into account that freshly ingested data can take up to 30 seconds to show up in query results,
-so don't worry if it doesn't appear immediately - just retry the query a bit later.
-
-Once you're done experimenting, tidying everything up takes a single command.
-
-### Cleanup
-
-Stop VictoriaMetrics and node_exporter with `Ctrl+C` in their respective terminals. All the collected data lives in the `victoria-metrics-data` directory -
-delete it if you want to start from scratch. To remove all traces of this test drive,
-delete the whole `vm-quick-start` directory created at [step 1](https://docs.victoriametrics.com/victoriametrics/quick-start/#step-1-download-the-binary).
-
-This test drive only scratches the surface of what VictoriaMetrics can do. Ready for a real setup?
-Continue with the installation options below, or learn the [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/)
-of writing and querying data first.
-
 ## How to install
 
 VictoriaMetrics is available in the following distributions:
@@ -214,6 +42,9 @@ Just download VictoriaMetrics and follow [these instructions](https://docs.victo
 See [available integrations](https://docs.victoriametrics.com/victoriametrics/integrations/) with other systems like
 [Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) or [Grafana](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/).
 
+> Want to see VictoriaMetrics in action, but without installing anything?
+> Try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) - a list of publicly available playgrounds for VictoriaMetrics software.
+
 VictoriaMetrics is developed at a fast pace, so it is recommended to periodically check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
 and perform [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
 
@@ -634,15 +465,6 @@ It is recommended to read [Replication and data safety](https://docs.victoriamet
 
 For backup configuration, please refer to [vmbackup documentation](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
 
-### Graceful shutdown
-
-To gracefully shut down a VictoriaMetrics process, send SIGTERM or SIGINT signal and wait until the process exits.
-See [how to send signals to processes](https://stackoverflow.com/questions/33239959/send-signal-to-process-from-command-line).
-
-Graceful shutdown is required for data safety. A successful graceful shutdown guarantees that pending in-memory data and 
-ongoing writes are flushed on disk before the process exits. During graceful shutdown, VictoriaMetrics stops accepting
-new HTTP connections and waits for in-flight requests to finish until `-http.maxGracefulShutdownDuration` expires.
-
 ### Configuring limits
 
 To avoid excessive resource usage or performance degradation, limits must be in place:

docs/victoriametrics/changelog/CHANGELOG.md

@@ -28,18 +28,18 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
 
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Improve background discovery performance for [http_sd](https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs) discovery. See [#8838](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8838).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): restore broken persistent queue chunk file from the last valid written block in case of ungraceful vmagent shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
 * FEATURE: [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-maxBackfillAge` command-line flag for limiting ingestion of samples with historical timestamps, for example, when older data has been moved between storage tiers (nvme/hdd, hot/cold). See [#11199](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11199). Thanks to @AshwinRamaniPsg for contribution.
 
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Now drops metadata blocks when communicating with vmstorage nodes over the legacy RPC protocol. To avoid this limitation, upgrade `vmstorage` to a version that supports the new RPC protocol (>= [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11370)). See [#11146](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11146).
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply limit to metrics metadata response. See [#11139](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11139).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a possible data race when processing OpenTelemetry metadata. See [#11238](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11238). Thanks to @nevgeny for contribution.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix increased CPU and memory usage when `-remoteWrite.urlRelabelConfig` or `-remoteWrite.streamAggr.config` flags are used. The bug was introduced in [#10854](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10854) and existed since [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0). See [#11250](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11250).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve newline formatting in alert and rule annotations on the Alerting page. See [#11171](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11171).
-* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
-* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
 
 ## [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0)
 
@@ -51,17 +51,17 @@ Released at 2026-07-06
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduces CPU usage by 10% at [sharding among remote storages](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages). See [#11113](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11113). Thanks to @bennf for contribution.
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `64KiB` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `vm_data_size_bytes{type="storage/metaindex"}` and `vm_data_size_bytes{type="indexdb/metaindex"}` metrics for tracking memory occupied by metaindex data. See [#11204](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11204). Thanks to @SamarthBagga for contribution.
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
-* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328), [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926), and the [documentation](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy).
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
-* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
+* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328), [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926), and the [documentation](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
 * FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_group_rule_results_limit` metric to indicate the number of alerts or recording results that a single rule within the group can produce. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
+* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
 
 * BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
@@ -93,8 +93,8 @@ Released at 2026-06-22
 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): disallow restoring parts outside the configured `-storageDataPath` directory. See [710c920d](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/710c920d6083327042a309e449fae4383617d817).
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly apply long tenant filters. Previously, such filters could be truncated, causing tenants to be matched incorrectly. See [#11096](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11096). Thanks for @fxrlv for the contribution.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix corrupted metrics metadata when a response contains multiple rows. See [#11115](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11115). Thanks for @fxrlv for the contribution.
-* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): don't cache empty responses for tenant IDs discovery during [multitenant queries](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenant-reads). This problem was visible during integration tests when multitenant queries were executed before the first ingestion happened. See [#10982](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10982)
 * BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): don't cache empty responses for tenant IDs discovery during [multitenant queries](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenant-reads). This problem was visible during integration tests when multitenant queries were executed before the first ingestion happened. See [#10982](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10982)
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
 
@@ -119,9 +119,9 @@ Released at 2026-06-08
 * BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly calculate number of loaded users to be printed in startup log. Previously, it was only accounting for static users and skipped JWT configuration entries. See [#11050](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11050/).
 * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): `integrate()` no longer extrapolates the last sample's value past the end of the time series. Previously, querying `integrate(metric[1h])` at a timestamp where the series had already ended would keep accruing area as if the last value continued indefinitely, producing values much larger than the true integral. See [#9474](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474). Thanks to @wtfashwin for contribution.
 * BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid returning HTTP 503 for queries with partial results when a storage group is unavailable and `-search.denyPartialResponse` is disabled. See [#11009](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11009). Thanks to @fxrlv for the contribution.
-* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `utf-8` label names for [/federate](https://docs.victoriametrics.com/victoriametrics/#federation) API requests. See [#10968](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10968).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the `Disable deduplication` toggle under its own local storage key. Before this fix, the toggle state was lost after reload and could overwrite the `Compact view` table setting. See [#11004](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11004). Thanks to @immanuwell for the contribution.
+* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
 
 ## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
 
@@ -129,28 +129,28 @@ Released at 2026-05-22
 
 * FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
 * FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
 * FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
-* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
 * FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Improve [slowness-based rerouting](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) to prevent rerouting storms under high cluster load. Previously, rerouting could cascade across storage nodes when the whole cluster was saturated, making the situation worse. Now rerouting only activates when the cluster p90 saturation is below 60%, and the slowest node is more than 20% slower than p90. See [#10876](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `{{.MetricsAccountID}}` and `{{.MetricsProjectID}}` [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) for use in `headers` and `url_prefix` config fields. Previously, only the combined `{{.MetricsTenant}}` (`accountID:projectID`) JWT placeholder was supported, making it impossible to configure [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). See [#10927](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10927). Thanks to @Vinayak9769 for the contribution.
+* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
 
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
 * BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive credentials.
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
-* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): hide values passed to `-remoteWrite.headers`,`remoteRead.headers`, `datasource.headers` and `notifier.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
 * BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly establish [mtls](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection) connection between vmstorage and vminsert. Regression was introduced in v1.130.0 release for the enterprise version of vmstorage. See [#10972](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10972).
 * BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
 * BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
-* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
-* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
 * BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
+* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
+* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
 
 ## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
 
@@ -160,10 +160,10 @@ Released at 2026-05-08
 
 * FEATURE: all VictoriaMetrics components: suppress TCP health check errors when `-tls` flag is set. See [#10538](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10538).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `__meta_hetzner_robot_datacenter` label for `robot` role in [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
-* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-rule.stripFilePath` to support stripping rule file paths in logs and all API responses, including /metrics. See [#5625](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5625).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `formatTime` template function for formatting a Unix timestamp using the provided layout. For example, `{{ now | formatTime "2006-01-02T15:04:05Z07:00" }}` returns the current time in RFC3339 format. See issue [#10624](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10624). Thanks to @andriibeee for the contribution.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
+* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
 * FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Kafka (Enterprise)` row with panels for monitoring traffic (bytes), messages in/out, producer and consumer errors. See [#10728](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10728).
 * FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support sending data to the configured `-remoteWrite.url` via [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol). See [#10929](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10929).
 

lib/persistentqueue/persistentqueue.go

@@ -25,6 +25,8 @@ const MaxBlockSize = 32 * 1024 * 1024
 // DefaultChunkFileSize represents default chunk file size
 const DefaultChunkFileSize = (MaxBlockSize + 8) * 16
 
+const blockHeaderSize = 8
+
 var chunkFileNameRegex = regexp.MustCompile("^[0-9A-F]{16}$")
 
 // queue represents persistent queue.
@@ -129,7 +131,7 @@ func mustOpen(path, name string, maxPendingBytes int64) *queue {
 }
 
 func mustOpenInternal(path, name string, chunkFileSize, maxBlockSize, maxPendingBytes uint64) *queue {
-	if chunkFileSize < 8 || chunkFileSize-8 < maxBlockSize {
+	if chunkFileSize < blockHeaderSize || chunkFileSize-blockHeaderSize < maxBlockSize {
 		logger.Panicf("BUG: too small chunkFileSize=%d for maxBlockSize=%d; chunkFileSize must fit at least one block", chunkFileSize, maxBlockSize)
 	}
 	if maxBlockSize <= 0 {
@@ -280,14 +282,30 @@ func tryOpeningQueue(path, name string, chunkFileSize, maxBlockSize, maxPendingB
 			q.writerFlushedOffset = mi.WriterOffset
 			if fileSize := fs.MustFileSize(q.writerPath); fileSize != q.writerLocalOffset {
 				if fileSize < q.writerLocalOffset {
-					logger.Errorf("%q size (%d bytes) is smaller than the writer offset (%d bytes); removing the file",
-						q.writerPath, fileSize, q.writerLocalOffset)
-					fs.MustRemovePath(q.writerPath)
-					continue
+					validChunkFileSize := mustGetChunkValidDataSize(q.writerPath, q.maxBlockSize)
+					logger.Warnf("%q size (%d bytes) is smaller than the writer offset (%d bytes); "+
+						"this may be the case on unclean shutdown (OOM, `kill -9`, hardware reset); resetting writer to fileSize: %d",
+						q.writerPath, fileSize, q.writerLocalOffset, validChunkFileSize)
+					mi.WriterOffset = offset + validChunkFileSize
+					q.writerOffset = mi.WriterOffset
+					q.writerLocalOffset = mi.WriterOffset % q.chunkFileSize
+					q.writerFlushedOffset = mi.WriterOffset
+					// The recovered writer offset may end up smaller than mi.ReaderOffset.
+					// Do not clamp the reader offset in this case: it means the reader has already
+					// consumed data that is now lost, so the remaining queue state cannot be trusted.
+					// The readerOffset > writerOffset check below will fail queue opening,
+					// and the caller (mustOpen) will drop the queue and recreate it from scratch.
+				} else {
+					logger.Warnf("%q size (%d bytes) is bigger than writer offset (%d bytes); "+
+						"this may be the case on unclean shutdown (OOM, `kill -9`, hardware reset); trying to fix it by adjusting fileSize to %d",
+						q.writerPath, fileSize, q.writerLocalOffset, q.writerLocalOffset)
+				}
+				if err := os.Truncate(q.writerPath, int64(q.writerLocalOffset)); err != nil {
+					logger.Panicf("FATAL: cannot truncate chunk: %q to size: %d: %s", q.writerPath, q.writerLocalOffset, err)
+				}
+				if err := mi.WriteToFile(metainfoPath); err != nil {
+					logger.Panicf("FATAL: cannot update metainfo file: %q: %s", metainfoPath, err)
 				}
-				logger.Warnf("%q size (%d bytes) is bigger than writer offset (%d bytes); "+
-					"this may be the case on unclean shutdown (OOM, `kill -9`, hardware reset); trying to fix it by adjusting fileSize to %d",
-					q.writerPath, fileSize, q.writerLocalOffset, q.writerLocalOffset)
 			}
 			w, err := filestream.OpenWriterAt(q.writerPath, int64(q.writerLocalOffset), false)
 			if err != nil {
@@ -357,7 +375,7 @@ func (q *queue) MustWriteBlock(block []byte) {
 	}
 	if q.maxPendingBytes > 0 {
 		// Drain the oldest blocks until the number of pending bytes becomes enough for the block.
-		blockSize := uint64(len(block) + 8)
+		blockSize := uint64(len(block) + blockHeaderSize)
 		maxPendingBytes := q.maxPendingBytes
 		if blockSize < maxPendingBytes {
 			maxPendingBytes -= blockSize
@@ -395,7 +413,7 @@ func (q *queue) writeBlock(block []byte) error {
 	defer func() {
 		writeDurationSeconds.Add(time.Since(startTime).Seconds())
 	}()
-	if q.writerLocalOffset+q.maxBlockSize+8 > q.chunkFileSize {
+	if q.writerLocalOffset+q.maxBlockSize+blockHeaderSize > q.chunkFileSize {
 		if err := q.nextChunkFileForWrite(); err != nil {
 			return fmt.Errorf("cannot create next chunk file: %w", err)
 		}
@@ -408,7 +426,7 @@ func (q *queue) writeBlock(block []byte) error {
 	err := q.write(header.B)
 	headerBufPool.Put(header)
 	if err != nil {
-		return fmt.Errorf("cannot write header with size 8 bytes to %q: %w", q.writerPath, err)
+		return fmt.Errorf("cannot write header with size %d bytes to %q: %w", blockHeaderSize, q.writerPath, err)
 	}
 
 	// Write block contents.
@@ -469,7 +487,7 @@ func (q *queue) readBlock(dst []byte) ([]byte, error) {
 	defer func() {
 		readDurationSeconds.Add(time.Since(startTime).Seconds())
 	}()
-	if q.readerLocalOffset+q.maxBlockSize+8 > q.chunkFileSize {
+	if q.readerLocalOffset+q.maxBlockSize+blockHeaderSize > q.chunkFileSize {
 		if err := q.nextChunkFileForRead(); err != nil {
 			return dst, fmt.Errorf("cannot open next chunk file: %w", err)
 		}
@@ -478,12 +496,12 @@ func (q *queue) readBlock(dst []byte) ([]byte, error) {
 again:
 	// Read block len.
 	header := headerBufPool.Get()
-	header.B = bytesutil.ResizeNoCopyMayOverallocate(header.B, 8)
+	header.B = bytesutil.ResizeNoCopyMayOverallocate(header.B, blockHeaderSize)
 	err := q.readFull(header.B)
 	blockLen := encoding.UnmarshalUint64(header.B)
 	headerBufPool.Put(header)
 	if err != nil {
-		logger.Errorf("skipping corrupted %q, since header with size 8 bytes cannot be read from it: %s", q.readerPath, err)
+		logger.Errorf("skipping corrupted %q, since header with size %d bytes cannot be read from it: %s", q.readerPath, blockHeaderSize, err)
 		if err := q.skipBrokenChunkFile(); err != nil {
 			return dst, err
 		}
@@ -669,3 +687,39 @@ func (mi *metainfo) ReadFromFile(path string) error {
 	}
 	return nil
 }
+
+func mustGetChunkValidDataSize(filePath string, maxBlockSize uint64) uint64 {
+	f, err := os.Open(filePath)
+	if err != nil {
+		logger.Panicf("FATAL: cannot open file: %s", err)
+	}
+	defer fs.MustClose(f)
+
+	fi, err := f.Stat()
+	if err != nil {
+		logger.Panicf("FATAL: cannot read file stat: %s", err)
+	}
+	fileSize := uint64(fi.Size())
+
+	var offset uint64
+	var header [blockHeaderSize]byte
+	for {
+		if offset+blockHeaderSize > fileSize {
+			return offset
+		}
+		if _, err := io.ReadFull(f, header[:]); err != nil {
+			return offset
+		}
+		blockLen := encoding.UnmarshalUint64(header[:])
+		if blockLen == 0 || blockLen > maxBlockSize {
+			return offset
+		}
+		if offset+blockHeaderSize+blockLen > fileSize {
+			return offset
+		}
+		offset += blockHeaderSize + blockLen
+		if _, err := f.Seek(int64(offset), io.SeekStart); err != nil {
+			return offset
+		}
+	}
+}

lib/persistentqueue/persistentqueue_test.go

@@ -137,6 +137,243 @@ func TestQueueOpen(t *testing.T) {
 		q.MustClose()
 		fs.MustRemoveDir(path)
 	})
+	t.Run("damaged-writer-file-tail", func(t *testing.T) {
+		path := "damaged-writer-file-tail"
+		fs.MustRemoveDir(path)
+
+		q := mustOpen(path, "foobar", 0)
+		block1 := []byte("valid block 1")
+		block2 := []byte("valid block 2")
+		q.MustWriteBlock(block1)
+		q.MustWriteBlock(block2)
+		q.MustClose()
+
+		chunkPath := filepath.Join(path, fmt.Sprintf("%016X", 0))
+		chunkFileSize := fs.MustFileSize(chunkPath)
+
+		if err := os.Truncate(chunkPath, int64(chunkFileSize)-1); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		q = mustOpen(path, "foobar", 0)
+
+		var buf []byte
+		var ok bool
+		buf, ok = q.MustReadBlockNonblocking(buf[:0])
+		if !ok {
+			t.Fatalf("expected block to be readable")
+		}
+		if string(buf) != string(block1) {
+			t.Fatalf("unexpected block got: %q, want: %q", buf, block1)
+		}
+
+		_, ok = q.MustReadBlockNonblocking(buf[:0])
+		if ok {
+			t.Fatalf("expected second block to be dropped")
+		}
+		q.MustClose()
+
+		expectedChunkFileSize := uint64(blockHeaderSize + len(block1))
+		chunkFileSize = fs.MustFileSize(chunkPath)
+		if chunkFileSize != expectedChunkFileSize {
+			t.Fatalf("unexpected chunk file size: got %d; want %d", chunkFileSize, expectedChunkFileSize)
+		}
+
+		fs.MustRemoveDir(path)
+	})
+	t.Run("damaged-writer-file-extra-tail", func(t *testing.T) {
+		path := "damaged-writer-file-extra-tail"
+		fs.MustRemoveDir(path)
+
+		q := mustOpen(path, "foobar", 0)
+		blocks := [][]byte{
+			[]byte("valid block 1"),
+			[]byte("valid block 2"),
+		}
+		for _, block := range blocks {
+			q.MustWriteBlock(block)
+		}
+		q.MustClose()
+
+		chunkPath := filepath.Join(path, fmt.Sprintf("%016X", 0))
+		originChunkFileSize := fs.MustFileSize(chunkPath)
+
+		f, err := os.OpenFile(chunkPath, os.O_WRONLY|os.O_APPEND, 0)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if _, err := f.Write([]byte("corrupted block")); err != nil {
+			t.Fatalf("cannot append corrupted tail: %s", err)
+		}
+		if err := f.Close(); err != nil {
+			t.Fatalf("cannot close chunk file: %s", err)
+		}
+
+		q = mustOpen(path, "foobar", 0)
+
+		var buf []byte
+		var ok bool
+		for _, block := range blocks {
+			buf, ok = q.MustReadBlockNonblocking(buf[:0])
+			if !ok {
+				t.Fatalf("expected block to be readable")
+			}
+			if string(buf) != string(block) {
+				t.Fatalf("unexpected block got: %q, want: %q", buf, block)
+			}
+		}
+		q.MustClose()
+		chunkFileSize := fs.MustFileSize(chunkPath)
+		if chunkFileSize != originChunkFileSize {
+			t.Fatalf("unexpected chunk file size: got %d; want %d", chunkFileSize, originChunkFileSize)
+		}
+		fs.MustRemoveDir(path)
+	})
+	t.Run("damaged-writer-file-partial-header", func(t *testing.T) {
+		path := "damaged-writer-file-partial-header"
+		fs.MustRemoveDir(path)
+
+		q := mustOpen(path, "foobar", 0)
+		block1 := []byte("valid block 1")
+		block2 := []byte("valid block 2")
+		q.MustWriteBlock(block1)
+		q.MustWriteBlock(block2)
+		q.MustClose()
+
+		chunkPath := filepath.Join(path, fmt.Sprintf("%016X", 0))
+		// Keep block1 intact plus only 3 bytes of the second block's header.
+		newSize := int64(blockHeaderSize + len(block1) + 3)
+		if err := os.Truncate(chunkPath, newSize); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		q = mustOpen(path, "foobar", 0)
+		buf, ok := q.MustReadBlockNonblocking(nil)
+		if !ok {
+			t.Fatalf("expected block to be readable")
+		}
+		if string(buf) != string(block1) {
+			t.Fatalf("unexpected block got: %q, want: %q", buf, block1)
+		}
+		if _, ok := q.MustReadBlockNonblocking(buf[:0]); ok {
+			t.Fatalf("expected second block to be dropped")
+		}
+		q.MustClose()
+
+		expectedChunkFileSize := uint64(blockHeaderSize + len(block1))
+		if chunkFileSize := fs.MustFileSize(chunkPath); chunkFileSize != expectedChunkFileSize {
+			t.Fatalf("unexpected chunk file size: got %d; want %d", chunkFileSize, expectedChunkFileSize)
+		}
+		fs.MustRemoveDir(path)
+	})
+	t.Run("damaged-writer-file-corrupted-header", func(t *testing.T) {
+		path := "damaged-writer-file-corrupted-header"
+		fs.MustRemoveDir(path)
+
+		q := mustOpen(path, "foobar", 0)
+		block1 := []byte("valid block 1")
+		block2 := []byte("valid block 2")
+		block3 := []byte("valid block 3")
+		q.MustWriteBlock(block1)
+		q.MustWriteBlock(block2)
+		q.MustWriteBlock(block3)
+		q.MustClose()
+
+		chunkPath := filepath.Join(path, fmt.Sprintf("%016X", 0))
+		chunkFileSize := fs.MustFileSize(chunkPath)
+
+		// Overwrite the second block's header with an impossible blockLen
+		// and drop the last byte so the recovery scan is triggered.
+		f, err := os.OpenFile(chunkPath, os.O_WRONLY, 0)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		garbage := [blockHeaderSize]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}
+		if _, err := f.WriteAt(garbage[:], int64(blockHeaderSize+len(block1))); err != nil {
+			t.Fatalf("cannot corrupt block header: %s", err)
+		}
+		if err := f.Close(); err != nil {
+			t.Fatalf("cannot close chunk file: %s", err)
+		}
+		if err := os.Truncate(chunkPath, int64(chunkFileSize)-1); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		q = mustOpen(path, "foobar", 0)
+		buf, ok := q.MustReadBlockNonblocking(nil)
+		if !ok {
+			t.Fatalf("expected block to be readable")
+		}
+		if string(buf) != string(block1) {
+			t.Fatalf("unexpected block got: %q, want: %q", buf, block1)
+		}
+		if _, ok := q.MustReadBlockNonblocking(buf[:0]); ok {
+			t.Fatalf("expected remaining blocks to be dropped")
+		}
+		q.MustClose()
+
+		expectedChunkFileSize := uint64(blockHeaderSize + len(block1))
+		if chunkFileSize := fs.MustFileSize(chunkPath); chunkFileSize != expectedChunkFileSize {
+			t.Fatalf("unexpected chunk file size: got %d; want %d", chunkFileSize, expectedChunkFileSize)
+		}
+		fs.MustRemoveDir(path)
+	})
+	t.Run("damaged-writer-file-second-chunk", func(t *testing.T) {
+		path := "damaged-writer-file-second-chunk"
+		fs.MustRemoveDir(path)
+
+		const maxBlockSize = 64
+		const chunkFileSize = (maxBlockSize + blockHeaderSize) * 2
+
+		newBlock := func(fill byte) []byte {
+			b := make([]byte, 32)
+			for i := range b {
+				b[i] = fill
+			}
+			return b
+		}
+		block1 := newBlock('1')
+		block2 := newBlock('2')
+		block3 := newBlock('3')
+		block4 := newBlock('4')
+
+		q := mustOpenInternal(path, "foobar", chunkFileSize, maxBlockSize, 0)
+		q.MustWriteBlock(block1)
+		q.MustWriteBlock(block2)
+		// blocks 3 and 4 land in the second chunk file
+		q.MustWriteBlock(block3)
+		q.MustWriteBlock(block4)
+		q.MustClose()
+
+		secondChunkPath := filepath.Join(path, fmt.Sprintf("%016X", chunkFileSize))
+		secondChunkSize := fs.MustFileSize(secondChunkPath)
+		if err := os.Truncate(secondChunkPath, int64(secondChunkSize)-1); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		q = mustOpenInternal(path, "foobar", chunkFileSize, maxBlockSize, 0)
+		var buf []byte
+		var ok bool
+		for _, block := range [][]byte{block1, block2, block3} {
+			buf, ok = q.MustReadBlockNonblocking(buf[:0])
+			if !ok {
+				t.Fatalf("expected block to be readable")
+			}
+			if string(buf) != string(block) {
+				t.Fatalf("unexpected block got: %q, want: %q", buf, block)
+			}
+		}
+		if _, ok := q.MustReadBlockNonblocking(buf[:0]); ok {
+			t.Fatalf("expected last block to be dropped")
+		}
+		q.MustClose()
+
+		expectedSize := uint64(blockHeaderSize + len(block3))
+		if gotSize := fs.MustFileSize(secondChunkPath); gotSize != expectedSize {
+			t.Fatalf("unexpected second chunk file size: got %d; want %d", gotSize, expectedSize)
+		}
+		fs.MustRemoveDir(path)
+	})
 }
 
 func TestQueueResetIfEmpty(t *testing.T) {