mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-08-15 04:02:25 +03:00
Compare commits
7 Commits
docs-artic
...
docs-singl
| Author | SHA1 | Date | |
|---|---|---|---|
|
7f9138b275 | ||
|
|
c2947586f5 | ||
|
|
45518b43ac | ||
|
|
87e6228313 | ||
|
|
c1a66abd88 | ||
|
|
7cc5a666ec | ||
|
|
6e2b77247b |
@@ -1,5 +0,0 @@
|
||||
<svg width="48" height="48" fill="#020202" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0"/>
|
||||
<path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z"/>
|
||||
<path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
197
app/vmselect/vmui/assets/index-B1dXK3k7.js
Normal file
197
app/vmselect/vmui/assets/index-B1dXK3k7.js
Normal file
File diff suppressed because one or more lines are too long
1
app/vmselect/vmui/assets/index-BJqoElx2.css
Normal file
1
app/vmselect/vmui/assets/index-BJqoElx2.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
1
app/vmselect/vmui/favicon.svg
Normal file
1
app/vmselect/vmui/favicon.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg width="48" height="48" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M24.5475 0C10.3246.0265251 1.11379 3.06365 4.40623 6.10077c0 0 12.32997 11.23333 16.58217 14.84083.8131.6896 2.1728 1.1936 3.5191 1.2201h.1199c1.3463-.0265 2.706-.5305 3.5191-1.2201 4.2522-3.5942 16.5422-14.84083 16.5422-14.84083C48.0478 3.06365 38.8636.0265251 24.6674 0" fill="#020202"/><path d="M28.1579 27.0159c-.8131.6896-2.1728 1.1936-3.5191 1.2201h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2201-2.9725-2.5067-13.35639-11.87-17.26201-15.3979v5.4112c0 .5968.22661 1.3793.6265 1.7506C7.00358 21.1936 17.2675 30.5437 20.9731 33.6737c.8132.6896 2.1728 1.1936 3.5191 1.2201h.12c1.3463-.0265 2.7059-.5305 3.519-1.2201 3.679-3.13 13.9429-12.4536 16.6089-14.8939.4132-.3713.6265-1.1538.6265-1.7506V11.618c-3.9323 3.5411-14.3162 12.931-17.2354 15.3979h.0267Z" fill="#020202"/><path d="M28.1579 39.748c-.8131.6897-2.1728 1.1937-3.5191 1.2202h-.12c-1.3463-.0265-2.7059-.5305-3.519-1.2202-2.9725-2.4933-13.35639-11.8567-17.26201-15.3978v5.4111c0 .5969.22661 1.3793.6265 1.7507C7.00358 33.9258 17.2675 43.2759 20.9731 46.4058c.8132.6897 2.1728 1.1937 3.5191 1.2202h.12c1.3463-.0265 2.7059-.5305 3.519-1.2202 3.679-3.1299 13.9429-12.4535 16.6089-14.8938.4132-.3714.6265-1.1538.6265-1.7507v-5.4111c-3.9323 3.5411-14.3162 12.931-17.2354 15.3978h.0267Z" fill="#020202"/></svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
@@ -2,9 +2,9 @@
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<link id="favicon" rel="icon" href="./assets/favicon.svg" />
|
||||
<link rel="apple-touch-icon" href="./assets/favicon.svg" />
|
||||
<link id="mask-icon" rel="mask-icon" href="./assets/favicon.svg" color="#000000">
|
||||
<link rel="icon" href="./favicon.svg"/>
|
||||
<link rel="apple-touch-icon" href="./favicon.svg"/>
|
||||
<link rel="mask-icon" href="./favicon.svg" color="#000000">
|
||||
|
||||
<meta name="robots" content="noindex">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
|
||||
@@ -37,11 +37,11 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-BiDX4bB6.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-B1dXK3k7.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-CymA7XYg.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"name": "vmui",
|
||||
"icons": [
|
||||
{
|
||||
"src": "./assets/favicon.svg",
|
||||
"src": "favicon.svg",
|
||||
"sizes": "any",
|
||||
"type": "image/svg+xml"
|
||||
}
|
||||
|
||||
@@ -156,13 +156,6 @@ See [our blog](https://victoriametrics.com/blog) for the latest articles written
|
||||
* [Why irate from Prometheus doesn't capture spikes](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
|
||||
* [VictoriaMetrics: PromQL compliance](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e)
|
||||
* [How do open source solutions for logs work: Elasticsearch, Loki and VictoriaLogs](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f)
|
||||
* [How vmagent Collects and Ships Metrics Fast with Aggregation, Deduplication, and More](https://victoriametrics.com/blog/vmagent-how-it-works/)
|
||||
* [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/)
|
||||
* [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/)
|
||||
* [How vmstorage Processes Data: Retention, Merging, Deduplication...](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/)
|
||||
* [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/)
|
||||
* [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/)
|
||||
* [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/)
|
||||
|
||||
### Tutorials, guides and how-to articles
|
||||
|
||||
@@ -180,12 +173,6 @@ See [our guides](https://docs.victoriametrics.com/guides/) for the up-to-date gu
|
||||
* [Prometheus storage: tech terms for humans](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
|
||||
* [Cardinality explorer](https://victoriametrics.com/blog/cardinality-explorer/)
|
||||
* [Rules backfilling via vmalert](https://victoriametrics.com/blog/rules-replay/)
|
||||
* [vmagent: Key Features Explained in Under 15 Minutes](https://victoriametrics.com/blog/vmagent-key-features-explained/)
|
||||
* [Prometheus Metrics Explained: Counters, Gauges, Histograms & Summaries](https://victoriametrics.com/blog/prometheus-monitoring-metrics-counters-gauges-histogram-summaries/)
|
||||
* [Prometheus Monitoring: Instant Queries and Range Queries Explained](https://victoriametrics.com/blog/prometheus-monitoring-instant-range-query/)
|
||||
* [Prometheus Monitoring: Functions, Subqueries, Operators, and Modifiers](https://victoriametrics.com/blog/prometheus-monitoring-function-operator-modifier/)
|
||||
* [Prometheus Alerting 101: Rules, Recording Rules, and Alertmanager](https://victoriametrics.com/blog/alerting-recording-rules-alertmanager/)
|
||||
* [Alerting Best Practices](https://victoriametrics.com/blog/alerting-best-practices/)
|
||||
|
||||
### Other articles
|
||||
|
||||
|
||||
@@ -59,11 +59,6 @@ It increases cluster availability, and simplifies cluster maintenance as well as
|
||||
|
||||
|
||||
|
||||
> Further reading, deep dives into how each service works internally:
|
||||
> - `vmstorage`: [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/), [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/), [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/).
|
||||
> - `vminsert`: [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/).
|
||||
> - `vmselect`: [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/).
|
||||
|
||||
## vmui
|
||||
|
||||
VictoriaMetrics cluster version provides UI for query troubleshooting and exploration. The UI is available at
|
||||
@@ -100,7 +95,7 @@ See also multitenancy [via headers](#multitenancy-via-headers) and [via labels](
|
||||
|
||||
### Multitenancy via headers
|
||||
|
||||
With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "v1.150.0" %}})
|
||||
With `--enableMultitenancyViaHeaders` {{% available_from "v1.143.0" %}} command-line flag enabled (enabled by default {{% available_from "#" %}})
|
||||
tenant ID can be specified via HTTP headers `AccountID` and `ProjectID`. This flag needs to be enabled on vminserts and vmselects.
|
||||
|
||||
With `--enableMultitenancyViaHeaders` enabled [URL format](#url-format) can be simplified to the following:
|
||||
@@ -833,8 +828,8 @@ See also [minimum downtime strategy](#minimum-downtime-strategy).
|
||||
|
||||
## Slowness-based re-routing
|
||||
|
||||
By default{{% available_from "v1.149.0" %}}, `vminsert` automatically [re-route writes](https://victoriametrics.com/blog/vminsert-how-it-works/#31-rerouting)
|
||||
away from the slowest `vmstorage` node to preserve maximum ingestion throughput. This prevents a single slow `vmstorage` node
|
||||
By default{{% available_from "v1.149.0" %}}, `vminsert` automatically re-routes writes away from the slowest `vmstorage` node
|
||||
to preserve maximum ingestion throughput. This prevents a single slow `vmstorage` node
|
||||
from throttling the entire cluster.
|
||||
|
||||
Re-routing occurs only when all of the following conditions hold:
|
||||
@@ -882,7 +877,7 @@ See also [resource usage limits docs](#resource-usage-limits).
|
||||
|
||||
## Rebalancing
|
||||
|
||||
Every `vminsert` node [evenly spreads (shards) incoming data](https://victoriametrics.com/blog/vminsert-how-it-works/#3-sharding-and-buffering) among `vmstorage` nodes specified in the `-storageNode` command-line flag.
|
||||
Every `vminsert` node evenly spreads (shards) incoming data among `vmstorage` nodes specified in the `-storageNode` command-line flag.
|
||||
This guarantees even distribution of the ingested data among `vmstorage` nodes. When new `vmstorage` nodes are added to the `-storageNode`
|
||||
command-line flag at `vminsert`, then only newly ingested data is distributed evenly among old and new `vmstorage` nodes, while
|
||||
historical data remains on the old `vmstorage` nodes. This speeds up data ingestion and querying for the majority of production workloads,
|
||||
@@ -1030,7 +1025,7 @@ By default, VictoriaMetrics offloads replication to the underlying storage point
|
||||
which guarantees data durability. VictoriaMetrics supports application-level replication if replicated durable persistent disks cannot be used for some reason.
|
||||
|
||||
The replication can be enabled by passing `-replicationFactor=N` command-line flag to `vminsert`. This instructs `vminsert` to store `N` copies for every ingested sample
|
||||
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable. See [how `vminsert` replicates each sample to `N` `vmstorage` nodes](https://victoriametrics.com/blog/vminsert-how-it-works/#4-replication-and-sending-data-to-vmstorage) for details.
|
||||
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
|
||||
|
||||
Passing `-replicationFactor=N` command-line flag to `vmselect` instructs it to not mark responses as `partial` if less than `-replicationFactor` vmstorage nodes are unavailable during the query.
|
||||
See [cluster availability docs](#cluster-availability) for details.
|
||||
@@ -1065,7 +1060,7 @@ deduplication can't be guaranteed when samples and sample duplicates for the sam
|
||||
- when `vmstorage` node has no enough capacity for processing incoming data stream. Then `vminsert` re-routes new samples to other `vmstorage` nodes.
|
||||
|
||||
It is recommended to set **the same** `-dedup.minScrapeInterval` command-line flag value to both `vmselect` and `vmstorage` nodes
|
||||
to ensure query results consistency, even if [storage layer didn't complete deduplication](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#deduplication) yet.
|
||||
to ensure query results consistency, even if storage layer didn't complete deduplication yet.
|
||||
|
||||
## Metrics Metadata
|
||||
|
||||
|
||||
@@ -1266,47 +1266,96 @@ See also [resource usage limits at VictoriaMetrics cluster](https://docs.victori
|
||||
|
||||
## High availability
|
||||
|
||||
The general approach for achieving high availability is the following:
|
||||
VictoriaMetrics supports high availability for both writes and reads by combining replication with multiple instances.
|
||||
|
||||
* To run two identically configured VictoriaMetrics instances in distinct datacenters (availability zones);
|
||||
* To store the collected data simultaneously into these instances via [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) or Prometheus.
|
||||
* To query the first VictoriaMetrics instance and to fail over to the second instance when the first instance becomes temporarily unavailable.
|
||||
This can be done via [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) according to [these docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#high-availability).
|
||||
You can achieve **high availability for writes** using replication:
|
||||
|
||||
Such a setup guarantees that the collected data isn't lost when one of VictoriaMetrics instance becomes unavailable.
|
||||
The collected data continues to be written to the available VictoriaMetrics instance, so it should be available for querying.
|
||||
Both [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and Prometheus buffer the collected data locally if they cannot send it
|
||||
to the configured remote storage. So the collected data will be written to the temporarily unavailable VictoriaMetrics instance
|
||||
after it becomes available.
|
||||
|
||||
If you use [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) for storing the data into VictoriaMetrics,
|
||||
then it can be configured with multiple `-remoteWrite.url` command-line flags, where every flag points to the VictoriaMetrics
|
||||
instance in a particular availability zone, in order to replicate the collected data to all the VictoriaMetrics instances.
|
||||
For example, the following command instructs `vmagent` to replicate data to `vm-az1` and `vm-az2` instances of VictoriaMetrics:
|
||||
* Run two or more identically configured VictoriaMetrics instances in distinct datacenters (availability zones);
|
||||
* Replicate collected metrics simultaneously into all these instances via one or more [vmagents](https://docs.victoriametrics.com/victoriametrics/vmagent/).
|
||||
|
||||
In this setup, configure vmagent [to replicate data](https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability)
|
||||
to each remote destination:
|
||||
```sh
|
||||
/path/to/vmagent \
|
||||
-remoteWrite.url=http://<vm-az1>:8428/api/v1/write \
|
||||
-remoteWrite.url=http://<vm-az2>:8428/api/v1/write
|
||||
-remoteWrite.url=https://victoriametrics-1:8428/api/v1/write \
|
||||
-remoteWrite.url=https://victoriametrics-2:8428/api/v1/write
|
||||
```
|
||||
|
||||
If you use Prometheus for collecting and writing the data to VictoriaMetrics,
|
||||
then the following [`remote_write`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write) section
|
||||
in Prometheus config can be used for replicating the collected data to `vm-az1` and `vm-az2` VictoriaMetrics instances:
|
||||
Each `--remoteWrite.url` creates its own replication queue. The queue temporarily stores data on disk while a remote destination is unavailable.
|
||||
See more about [on-disk persistence in vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#on-disk-persistence).
|
||||
|
||||
```yaml
|
||||
remote_write:
|
||||
- url: http://<vm-az1>:8428/api/v1/write
|
||||
- url: http://<vm-az2>:8428/api/v1/write
|
||||
When the remote destination becomes available, vmagent drains the queue and restores data consistency across destinations.
|
||||
|
||||
> The max size of the on-disk queue can be increased by [horizontally sharding vmagents](https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets).
|
||||
> To achieve high availability for vmagent itself, run multiple identically configured vmagent replicas.
|
||||
> In this case, the load on the remote destinations will increase proportionally to the number of vmagent replicas. The duplicated data in remote destinations
|
||||
> has to be [deduplicated](https://docs.victoriametrics.com/victoriametrics/#deduplication) on the VictoriaMetrics side.
|
||||
|
||||
You can achieve **high availability for reads** by choosing one of the following options:
|
||||
|
||||
- Load balancer: Use a load balancer to ensure read operations are always routed to an available VictoriaMetrics instance.
|
||||
- Top-level vmselect: Use vmselect to query all available VictoriaMetrics instances and merge the results
|
||||
|
||||
### Load balancer
|
||||
|
||||
In this mode, we use a load balancer to query the main VictoriaMetrics instance and fail over to a secondary instance if the first one becomes temporarily unavailable.
|
||||
|
||||
This can be done using [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) configured in [high-availability mode](https://docs.victoriametrics.com/victoriametrics/vmauth/#high-availability).
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client["Query Client<br/>Grafana/vmalert"]
|
||||
|
||||
VMAUTH["vmauth<br/>Load Balancer / Failover"]
|
||||
|
||||
VM1["VictoriaMetrics-1<br/>Primary read target"]
|
||||
VM2["VictoriaMetrics-2<br/>Failover read target"]
|
||||
|
||||
Client -->|"Read query"| VMAUTH
|
||||
|
||||
VMAUTH -->|"1. Send queries"| VM1
|
||||
VMAUTH -.->|"2. Fail over if VM1<br/>is unavailable"| VM2
|
||||
```
|
||||
|
||||
It is recommended to use [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) instead of Prometheus for highly loaded setups,
|
||||
since it uses lower amounts of RAM, CPU and network bandwidth than Prometheus.
|
||||
This is the most cost-efficient option because only one VictoriaMetrics instance is queried at a time.
|
||||
|
||||
If you use identically configured [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) instances for collecting the same data
|
||||
and sending it to VictoriaMetrics, then do not forget enabling [deduplication](#deduplication) at VictoriaMetrics side.
|
||||
The downside is that when one instance goes down and then comes back up, the load balancer may immediately start sending read queries to the recovering instance, even though it hasn't caught up yet. During this short recovery window:
|
||||
|
||||
See [VMDistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/) Kubernetes operator resource for an example.
|
||||
- The recovering VictoriaMetrics instance has not yet ingested all the data that was queued while it was down.
|
||||
- vmagent nodes may still be draining their backlogged queues into that instance
|
||||
- Read queries routed to the recovering instance can return incomplete results, because some recent data may still be in transit or not yet visible.
|
||||
|
||||
### Top-level vmselect
|
||||
|
||||
In this option, we use a top-level [vmselect](https://docs.victoriametrics.com/victoriametrics/vmselect/) to query all remote destinations simultaneously and merge the results.
|
||||
|
||||
This option is only possible if VictoriaMetrics single-node instances are configured with the `-vmselectAddr` flag. See more details in the [VictoriaMetrics multi-tenancy section](https://docs.victoriametrics.com/victoriametrics/#multi-tenancy).
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client["Query Client<br/>Grafana / vmalert"]
|
||||
|
||||
VMSELECT["vmselect<br/>Query all destinations<br/>
|
||||
<code><pre>-dedup.minScrapeInterval=1ms<br/>-replicationFactor=2</pre></code>"]
|
||||
|
||||
VM1["VictoriaMetrics-1<br/>Single-node<br/><code>-vmselectAddr=:8401</code>"]
|
||||
VM2["VictoriaMetrics-2<br/>Single-node<br/><code>-vmselectAddr=:8401</code>"]
|
||||
|
||||
Client -->|"Read query"| VMSELECT
|
||||
|
||||
VMSELECT --> VM1
|
||||
VMSELECT --> VM2
|
||||
|
||||
VMSELECT -->|"Merged and deduplicated results"| Client
|
||||
```
|
||||
|
||||
This option requires more resources because it queries all remote destinations simultaneously and merges their responses before returning the final result.
|
||||
|
||||
The benefit is that it can handle data gaps across destinations by merging responses from all VictoriaMetrics instances. Thus, a single recovering instance is less likely to cause incomplete results, unlike in the load balancer case.
|
||||
|
||||
Since vmselect queries identical data across multiple VictoriaMetrics instances, configure it with `-dedup.minScrapeInterval=1ms` to remove duplicate samples during merging. Also set `-replicationFactor=N` on vmselect, where `N` equals the number of remote storage destinations, so that queries can tolerate the unavailability of up to `N-1` destinations.
|
||||
|
||||
See [VMDistributed](https://docs.victoriametrics.com/operator/resources/vmdistributed/) Kubernetes operator resource for an example of configuring vmselect with deduplication and replication.
|
||||
|
||||
## Deduplication
|
||||
|
||||
@@ -1407,9 +1456,6 @@ for fast block lookups, which belong to the given `TSID` and cover the given tim
|
||||
* various background maintenance tasks such as [de-duplication](#deduplication), [downsampling](#downsampling)
|
||||
and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge
|
||||
|
||||
See how `vmstorage` [selects parts for background merging](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#merge-process),
|
||||
including merge limits and monitoring metrics.
|
||||
|
||||
Newly added `parts` either successfully appear in the storage or fail to appear.
|
||||
The newly added `part` is atomically registered in the `parts.json` file under the corresponding partition
|
||||
after it is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html) to the storage.
|
||||
@@ -1537,8 +1583,6 @@ are **eventually deleted** during [background merge](https://medium.com/@valyala
|
||||
The time range covered by data part is **not limited by retention period unit**. One data part can cover hours or days of
|
||||
data. Hence, a data part can be deleted only **when fully outside the configured retention**.
|
||||
See more about partitions and parts in the [Storage section](#storage).
|
||||
See how the [retention and free-disk watchers manage storage](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#retention-free-disk-space-guard-and-downsampling)
|
||||
for implementation details and monitoring metrics.
|
||||
|
||||
The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
|
||||
For example, if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
|
||||
|
||||
@@ -26,10 +26,6 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
## [v1.150.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.150.0)
|
||||
|
||||
Release candidate
|
||||
|
||||
**Update Note 1:** `vmselect` and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/), and `vmagent`: default value of `-enableMultitenancyViaHeaders` command-line flag has changed from `false` to `true`. This change enables support of [multitenancy via headers for cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) and [for vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#multitenancy-via-headers) by default. With this change, mentioned components will start supporting URLs with omitted tenant ID in the path: `https://<vmselect>:8481/select/prometheus/api/v1/query` will become a valid URL. To disable multitenancy via headers and simplified URLs set `--enableMultitenancyViaHeaders=false` on vmagent, vminsert and vmselect.
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.26.5 to Go1.26.6. See [the list of issues addressed in Go1.26.6](https://github.com/golang/go/issues?q=milestone%3AGo1.26.6%20label%3ACherryPickApproved).
|
||||
|
||||
@@ -1316,7 +1316,7 @@ The list of discovered Kuma targets is refreshed at the interval, which can be c
|
||||
|
||||
## linode_sd_configs
|
||||
|
||||
Linode SD configuration {{% available_from "v1.150.0" %}} allows retrieving scrape targets from [Linode](https://www.linode.com/) instances.
|
||||
Linode SD configuration {{% available_from "#" %}} allows retrieving scrape targets from [Linode](https://www.linode.com/) instances.
|
||||
The following [Linode API](https://www.linode.com/docs/api/) token scopes are required: `linodes:read_only` and `ips:read_only`.
|
||||
|
||||
Configuration example:
|
||||
|
||||
@@ -141,7 +141,6 @@ to other remote storage systems that support Prometheus `remote_write` protocol
|
||||
If a single remote storage instance is temporarily unavailable, the collected data remains available on the other remote storage instances.
|
||||
`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
|
||||
Then it sends the buffered data to the remote storage in order to prevent data gaps.
|
||||
See how `vmagent` [selects shards and places replicas](https://victoriametrics.com/blog/vmagent-how-it-works/#step-4-sharding--replication) for implementation details.
|
||||
|
||||
[VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) already supports replication,
|
||||
so there is no need to specify multiple `-remoteWrite.url` flags when writing data to the same cluster.
|
||||
@@ -151,8 +150,7 @@ See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victor
|
||||
|
||||
`vmagent` can add, remove, or update labels on the collected data before sending it to the remote storage.
|
||||
It can filter scrape targets or remove unwanted samples via Prometheus-like relabeling.
|
||||
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for configuration examples.
|
||||
For ingestion pipeline internals, see how `vmagent` [applies global relabeling and cardinality limits](https://victoriametrics.com/blog/vmagent-how-it-works/#step-2-global-relabeling-cardinality-reduction).
|
||||
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for details.
|
||||
|
||||
### Sharding among remote storages
|
||||
|
||||
@@ -270,9 +268,7 @@ for the collected samples. Examples:
|
||||
```sh
|
||||
./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
|
||||
```
|
||||
|
||||
See how `vmagent` [orders global deduplication and stream aggregation](https://victoriametrics.com/blog/vmagent-how-it-works/#step-3-global-deduplication--stream-aggregation) in the ingestion pipeline.
|
||||
|
||||
|
||||
### Monitoring Data eXchange
|
||||
|
||||
The Monitoring Data eXchange (MDX){{% available_from "v1.147.0" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
|
||||
@@ -363,8 +359,6 @@ in addition to the pull-based Prometheus-compatible targets' scraping:
|
||||
* Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
|
||||
* Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data).
|
||||
|
||||
See how `vmagent` [handles concurrency, decompression, and stream parsing](https://victoriametrics.com/blog/vmagent-how-it-works/#step-1-receiving-data-via-api-or-scrape) during ingestion.
|
||||
|
||||
## How to collect metrics in Prometheus format
|
||||
|
||||
Specify the path to the `prometheus.yml` file via the `-promscrape.config` command-line flag. `vmagent` takes into account the following
|
||||
@@ -648,7 +642,7 @@ specified via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` c
|
||||
|
||||
vmagent can write data to multiple distinct tenants if:
|
||||
* its `-remoteWrite.url` points to the [VictoriaMetrics cluster multitenant URL](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-labels)
|
||||
* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "v1.150.0" %}}) command-line flags are both set
|
||||
* its `-enableMultitenantHandlers` and `-enableMultitenancyViaHeaders` (enabled by default {{% available_from "#" %}}) command-line flags are both set
|
||||
* clients ingest data into vmagent with the tenants specified [via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers) {{% available_from "v1.143.0" %}}
|
||||
|
||||
```mermaid
|
||||
@@ -1072,7 +1066,6 @@ This behavior can be changed with the `-remoteWrite.inmemoryQueues` {{% availabl
|
||||
When set to a non-zero value, vmagent starts the given number of additional workers,
|
||||
which send only recently ingested data from the in-memory queue, while the workers configured via `-remoteWrite.queues` drain the file-based backlog concurrently.
|
||||
This reduces the delivery lag for fresh samples after remote storage outages or slowdowns. The flag can be set individually per each `-remoteWrite.url`.
|
||||
See how the [in-memory and file-based queues manage blocks](https://victoriametrics.com/blog/vmagent-how-it-works/#in-memory-queue) for implementation details.
|
||||
|
||||
Note that these workers are started in addition to the workers configured via `-remoteWrite.queues`, so the total number of concurrent connections to
|
||||
the remote storage becomes the sum of both flags. Take this into account if the remote storage limits the number of concurrent requests.
|
||||
|
||||
@@ -1025,9 +1025,6 @@ Try the following tips to avoid common issues:
|
||||
In that case, the default step will be used (`-datasource.queryStep`) and may cause unexpected results compared to
|
||||
executing this query in vmui/Grafana, where step is adjusted differently.
|
||||
|
||||
See [practical examples for reducing alert noise](https://victoriametrics.com/blog/alerting-best-practices/#reducing-noise),
|
||||
including aggregating alerts and configuring inhibition.
|
||||
|
||||
### Rule state
|
||||
|
||||
vmalert keeps the last `-rule.updateEntriesLimit` updates (or `update_entries_limit` [per-rule config](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules))
|
||||
@@ -1083,9 +1080,6 @@ Sometimes, it's hard to understand why a specific alert fired or not. Keep in mi
|
||||
If evaluation returns error (i.e. datasource is unavailable), alert state doesn't change.
|
||||
If at least one evaluation returns no data, then alert's `for` state resets.
|
||||
|
||||
See [how to tune the `for` parameter](https://victoriametrics.com/blog/alerting-best-practices/#the-for-param),
|
||||
including its tradeoff with the query lookbehind window.
|
||||
|
||||
> Note: The alert state is tracked separately for each time series returned during evaluation.
|
||||
> For example, if the 1st evaluation returns series A and B, and the 2nd evaluation returns only B – the alert will remain active **only for B**.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user