mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-28 04:57:20 +03:00
Compare commits
2 Commits
docs/trace
...
v1.102.24
| Author | SHA1 | Date | |
|---|---|---|---|
|
86d8095417 | ||
|
|
df847e62c0 |
@@ -68,7 +68,7 @@ services:
|
||||
# VictoriaMetrics instance, a single process responsible for
|
||||
# scraping, storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
volumes:
|
||||
- vmdata:/storage
|
||||
- ./prometheus-vl-cluster.yml:/etc/prometheus/prometheus.yml
|
||||
@@ -81,7 +81,7 @@ services:
|
||||
# It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
|
||||
# depending on the requested path.
|
||||
vmauth:
|
||||
image: victoriametrics/vmauth:v1.119.0
|
||||
image: victoriametrics/vmauth:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
- "vlselect-1"
|
||||
@@ -97,7 +97,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules according to given rule type.
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.119.0
|
||||
image: victoriametrics/vmalert:v1.120.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
- "alertmanager"
|
||||
|
||||
@@ -49,7 +49,7 @@ services:
|
||||
# VictoriaMetrics instance, a single process responsible for
|
||||
# scraping, storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
ports:
|
||||
- "8428:8428"
|
||||
volumes:
|
||||
@@ -64,7 +64,7 @@ services:
|
||||
# It proxies query requests from vmalert to either VictoriaMetrics or VictoriaLogs,
|
||||
# depending on the requested path.
|
||||
vmauth:
|
||||
image: victoriametrics/vmauth:v1.119.0
|
||||
image: victoriametrics/vmauth:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
- "victorialogs"
|
||||
@@ -78,7 +78,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules according to the given rule type.
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.119.0
|
||||
image: victoriametrics/vmalert:v1.120.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
- "alertmanager"
|
||||
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
# It scrapes targets defined in --promscrape.config
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.119.0
|
||||
image: victoriametrics/vmagent:v1.120.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
@@ -35,14 +35,14 @@ services:
|
||||
# vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
|
||||
# where N is number of vmstorages (2 in this case).
|
||||
vmstorage-1:
|
||||
image: victoriametrics/vmstorage:v1.119.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.120.0-cluster
|
||||
volumes:
|
||||
- strgdata-1:/storage
|
||||
command:
|
||||
- "--storageDataPath=/storage"
|
||||
restart: always
|
||||
vmstorage-2:
|
||||
image: victoriametrics/vmstorage:v1.119.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.120.0-cluster
|
||||
volumes:
|
||||
- strgdata-2:/storage
|
||||
command:
|
||||
@@ -52,7 +52,7 @@ services:
|
||||
# vminsert is ingestion frontend. It receives metrics pushed by vmagent,
|
||||
# pre-process them and distributes across configured vmstorage shards.
|
||||
vminsert-1:
|
||||
image: victoriametrics/vminsert:v1.119.0-cluster
|
||||
image: victoriametrics/vminsert:v1.120.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -61,7 +61,7 @@ services:
|
||||
- "--storageNode=vmstorage-2:8400"
|
||||
restart: always
|
||||
vminsert-2:
|
||||
image: victoriametrics/vminsert:v1.119.0-cluster
|
||||
image: victoriametrics/vminsert:v1.120.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -73,7 +73,7 @@ services:
|
||||
# vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
|
||||
# vmselect collects results from configured `--storageNode` shards.
|
||||
vmselect-1:
|
||||
image: victoriametrics/vmselect:v1.119.0-cluster
|
||||
image: victoriametrics/vmselect:v1.120.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -83,7 +83,7 @@ services:
|
||||
- "--vmalert.proxyURL=http://vmalert:8880"
|
||||
restart: always
|
||||
vmselect-2:
|
||||
image: victoriametrics/vmselect:v1.119.0-cluster
|
||||
image: victoriametrics/vmselect:v1.120.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -98,7 +98,7 @@ services:
|
||||
# read requests from Grafana, vmui, vmalert among vmselects.
|
||||
# It can be used as an authentication proxy.
|
||||
vmauth:
|
||||
image: victoriametrics/vmauth:v1.119.0
|
||||
image: victoriametrics/vmauth:v1.120.0
|
||||
depends_on:
|
||||
- "vmselect-1"
|
||||
- "vmselect-2"
|
||||
@@ -112,7 +112,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.119.0
|
||||
image: victoriametrics/vmalert:v1.120.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
# It scrapes targets defined in --promscrape.config
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.119.0
|
||||
image: victoriametrics/vmagent:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
# VictoriaMetrics instance, a single process responsible for
|
||||
# storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
- 8089:8089
|
||||
@@ -54,7 +54,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.119.0
|
||||
image: victoriametrics/vmalert:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
- "alertmanager"
|
||||
|
||||
@@ -19,7 +19,7 @@ services:
|
||||
retries: 10
|
||||
|
||||
dd-proxy:
|
||||
image: docker.io/victoriametrics/vmauth:v1.119.0
|
||||
image: docker.io/victoriametrics/vmauth:v1.120.0
|
||||
restart: on-failure
|
||||
volumes:
|
||||
- ./:/etc/vmauth
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.119.0
|
||||
image: victoriametrics/vmagent:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
restart: always
|
||||
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -40,7 +40,7 @@ services:
|
||||
restart: always
|
||||
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.119.0
|
||||
image: victoriametrics/vmalert:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
- To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
|
||||
- In the tutorial, we'll be using the following VictoriaMetrics components:
|
||||
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.119.0)
|
||||
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.119.0)
|
||||
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.119.0)
|
||||
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.120.0)
|
||||
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.120.0)
|
||||
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.120.0)
|
||||
- [Grafana](https://grafana.com/) (v.10.2.1)
|
||||
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
|
||||
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
|
||||
@@ -315,7 +315,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
|
||||
services:
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.119.0
|
||||
image: victoriametrics/vmagent:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -332,7 +332,7 @@ services:
|
||||
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -365,7 +365,7 @@ services:
|
||||
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.119.0
|
||||
image: victoriametrics/vmalert:v1.120.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -241,27 +241,27 @@ services:
|
||||
- grafana_data:/var/lib/grafana/
|
||||
|
||||
vmsingle:
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
command:
|
||||
- -httpListenAddr=0.0.0.0:8429
|
||||
|
||||
vmstorage:
|
||||
image: victoriametrics/vmstorage:v1.119.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.120.0-cluster
|
||||
|
||||
vminsert:
|
||||
image: victoriametrics/vminsert:v1.119.0-cluster
|
||||
image: victoriametrics/vminsert:v1.120.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8400
|
||||
- -httpListenAddr=0.0.0.0:8480
|
||||
|
||||
vmselect:
|
||||
image: victoriametrics/vmselect:v1.119.0-cluster
|
||||
image: victoriametrics/vmselect:v1.120.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8401
|
||||
- -httpListenAddr=0.0.0.0:8481
|
||||
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.119.0
|
||||
image: victoriametrics/vmagent:v1.120.0
|
||||
volumes:
|
||||
- ./scrape.yaml:/etc/vmagent/config.yaml
|
||||
command:
|
||||
@@ -270,7 +270,7 @@ services:
|
||||
- -remoteWrite.url=http://vmsingle:8429/api/v1/write
|
||||
|
||||
vmgateway-cluster:
|
||||
image: victoriametrics/vmgateway:v1.119.0-enterprise
|
||||
image: victoriametrics/vmgateway:v1.120.0-enterprise
|
||||
ports:
|
||||
- 8431:8431
|
||||
volumes:
|
||||
@@ -286,7 +286,7 @@ services:
|
||||
- -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
|
||||
|
||||
vmgateway-single:
|
||||
image: victoriametrics/vmgateway:v1.119.0-enterprise
|
||||
image: victoriametrics/vmgateway:v1.120.0-enterprise
|
||||
ports:
|
||||
- 8432:8431
|
||||
volumes:
|
||||
@@ -397,7 +397,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
|
||||
|
||||
```yaml
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.119.0
|
||||
image: victoriametrics/vmagent:v1.120.0
|
||||
volumes:
|
||||
- ./scrape.yaml:/etc/vmagent/config.yaml
|
||||
- ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret
|
||||
|
||||
@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
|
||||
|
||||
## Currently supported LTS release lines
|
||||
|
||||
- v1.110.x - the latest one is [v1.110.9 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.9)
|
||||
- v1.102.x - the latest one is [v1.102.22 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.22)
|
||||
- v1.110.x - the latest one is [v1.110.12 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.12)
|
||||
- v1.102.x - the latest one is [v1.102.24 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.24)
|
||||
|
||||
@@ -746,7 +746,7 @@ See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rat
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "v1.120.0" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
|
||||
@@ -57,9 +57,9 @@ and performing [regular upgrades](https://docs.victoriametrics.com/victoriametri
|
||||
Download the newest available [VictoriaMetrics release](https://docs.victoriametrics.com/victoriametrics/changelog/)
|
||||
from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
|
||||
```sh
|
||||
docker pull victoriametrics/victoria-metrics:v1.119.0
|
||||
docker pull victoriametrics/victoria-metrics:v1.120.0
|
||||
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
|
||||
victoriametrics/victoria-metrics:v1.119.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
|
||||
victoriametrics/victoria-metrics:v1.120.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
|
||||
```
|
||||
_For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
|
||||
|
||||
|
||||
@@ -305,6 +305,23 @@ Released at 2025-02-10
|
||||
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix discrepancies when using `or` binary operator. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7759) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7640) issues for details.
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly update number of unique series for [cardinality limiter](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cardinality-limiter) on ingestion. Previously, limit could undercount the real number of the ingested unique series.
|
||||
|
||||
## [v1.110.12](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.12)
|
||||
|
||||
Released at 2025-06-20
|
||||
|
||||
**v1.110.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/changelog/#v11100) release**
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.24.3 to Go1.24.4. See [the list of issues addressed in Go1.24.4](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved).
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
|
||||
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/vmagent/): fixed duplication in Datadog sketches aggregation metrics. See [#8836](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8836) for details.
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect sorting of tag filters, which led to suboptimal tag filter evaluation order and potentially degraded query performance in rare cases. See [#9127](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9127) for details.
|
||||
* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `/-/healthy` and `/-/ready` endpoints as Prometheus does on vmselect endpoints, thus achieving full Prometheus compatibility for external dependencies.
|
||||
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix server-side copying of objects for Azure Blob Storage when using managed identity for authentication. Previously, it wasn't possible to use [smart backups](https://docs.victoriametrics.com/victoriametrics/vmbackup/#smart-backups) strategy for `vmbackup` as server-side copy would fail. See [#9131](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9131).
|
||||
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): increase startup healthcheck delay for storage reachability from 30 seconds to 3 minutes. This is required to avoid vmbackupmanager restarts when storage node startup take more than 30 seconds (e.g. when storage nodes stores more than 5Tb of data).
|
||||
|
||||
## [v1.110.11](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.110.11)
|
||||
|
||||
Released at 2025-06-09
|
||||
@@ -568,6 +585,22 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
|
||||
|
||||
See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2024/#v11030)
|
||||
|
||||
## [v1.102.24](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.24)
|
||||
|
||||
Released at 2025-06-20
|
||||
|
||||
**v1.102.x is a line of [LTS releases](https://docs.victoriametrics.com/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/enterprise.html).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.102.x line will be supported for at least 12 months since [v1.102.0](https://docs.victoriametrics.com/changelog/#v11020) release**
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
|
||||
* SECURITY: upgrade Go builder from Go1.23.9 to Go1.23.10. See [the list of issues addressed in Go1.23.10](https://github.com/golang/go/issues?q=milestone%3AGo1.23.10+label%3ACherryPickApproved).
|
||||
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect sorting of tag filters, which led to suboptimal tag filter evaluation order and potentially degraded query performance in rare cases. See [#9127](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9127) for details.
|
||||
* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `/-/healthy` and `/-/ready` endpoints as Prometheus does on vmselect endpoints, thus achieving full Prometheus compatibility for external dependencies.
|
||||
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix server-side copying of objects for Azure Blob Storage when using managed identity for authentication. Previously, it wasn't possible to use [smart backups](https://docs.victoriametrics.com/victoriametrics/vmbackup/#smart-backups) strategy for `vmbackup` as server-side copy would fail. See [#9131](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9131).
|
||||
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): increase startup healthcheck delay for storage reachability from 30 seconds to 3 minutes. This is required to avoid vmbackupmanager restarts when storage node startup take more than 30 seconds (e.g. when storage nodes stores more than 5Tb of data).
|
||||
|
||||
## [v1.102.23](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.23)
|
||||
|
||||
Released at 2025-06-06
|
||||
|
||||
@@ -89,7 +89,7 @@ VictoriaMetrics Enterprise components are available in the following forms:
|
||||
It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
|
||||
|
||||
Binary releases of VictoriaMetrics Enterprise are available [at the releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz`.
|
||||
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz`.
|
||||
|
||||
In order to run binary release of VictoriaMetrics Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
|
||||
from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest) and unpack it. Then run the unpacked binary.
|
||||
@@ -107,8 +107,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
|
||||
obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
|
||||
|
||||
```sh
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz
|
||||
tar -xzf victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz
|
||||
tar -xzf victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz
|
||||
./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
|
||||
```
|
||||
|
||||
@@ -123,7 +123,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
|
||||
It is allowed to run VictoriaMetrics Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
|
||||
|
||||
Docker images for VictoriaMetrics Enterprise are available at VictoriaMetrics [Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay](https://quay.io/organization/victoriametrics).
|
||||
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.119.0-enterprise`.
|
||||
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.120.0-enterprise`.
|
||||
|
||||
In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via command-line
|
||||
flag as described [here](#binary-releases).
|
||||
@@ -133,13 +133,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
|
||||
For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
|
||||
|
||||
```sh
|
||||
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.119.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
|
||||
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.120.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
|
||||
```
|
||||
|
||||
Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
|
||||
|
||||
```sh
|
||||
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.119.0-enterprise -licenseFile=/path/to/vm-license
|
||||
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.120.0-enterprise -licenseFile=/path/to/vm-license
|
||||
```
|
||||
|
||||
Example docker-compose configuration:
|
||||
@@ -148,7 +148,7 @@ version: "3.5"
|
||||
services:
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.119.0
|
||||
image: victoriametrics/victoria-metrics:v1.120.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -180,7 +180,7 @@ is used to provide key in plain-text:
|
||||
```yaml
|
||||
server:
|
||||
image:
|
||||
tag: v1.119.0-enterprise
|
||||
tag: v1.120.0-enterprise
|
||||
|
||||
license:
|
||||
key: {BASE64_ENCODED_LICENSE_KEY}
|
||||
@@ -191,7 +191,7 @@ In order to provide key via existing secret, the following values file is used:
|
||||
```yaml
|
||||
server:
|
||||
image:
|
||||
tag: v1.119.0-enterprise
|
||||
tag: v1.120.0-enterprise
|
||||
|
||||
license:
|
||||
secret:
|
||||
@@ -240,7 +240,7 @@ spec:
|
||||
license:
|
||||
key: {BASE64_ENCODED_LICENSE_KEY}
|
||||
image:
|
||||
tag: v1.119.0-enterprise
|
||||
tag: v1.120.0-enterprise
|
||||
```
|
||||
|
||||
In order to provide key via existing secret, the following custom resource is used:
|
||||
@@ -257,7 +257,7 @@ spec:
|
||||
name: vm-license
|
||||
key: license
|
||||
image:
|
||||
tag: v1.119.0-enterprise
|
||||
tag: v1.120.0-enterprise
|
||||
```
|
||||
|
||||
Example secret with license key:
|
||||
@@ -283,10 +283,10 @@ See full list of CRD specifications [here](https://docs.victoriametrics.com/oper
|
||||
### FIPS compliance
|
||||
|
||||
VictoriaMetrics Enterprise components can be run in FIPS compliant mode {{% available_from "v1.118.0" %}}. Binary releases and Docker images
|
||||
of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.119.0-enterprise.tar.gz`
|
||||
of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.120.0-enterprise.tar.gz`
|
||||
archive includes `victoria-metrics-prod` and `victoria-metrics-fips` binaries. The latter binary is FIPS compliant.
|
||||
|
||||
Docker images of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.119.0-enterprise-fips`
|
||||
Docker images of VictoriaMetrics Enterprise components have `fips` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.120.0-enterprise-fips`
|
||||
image uses FIPS compliant binary version.
|
||||
|
||||
## Monitoring license expiration
|
||||
|
||||
@@ -36,8 +36,8 @@ scrape_configs:
|
||||
After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/) to the same directory:
|
||||
|
||||
```
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.119.0.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/victoria-metrics-linux-amd64-v1.120.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.120.0.tar.gz
|
||||
```
|
||||
|
||||
Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
|
||||
@@ -152,8 +152,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/) acco
|
||||
|
||||
```yaml
|
||||
# Download and unpack single-node VictoriaMetrics
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.119.0/victoria-metrics-linux-amd64-v1.119.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.119.0.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.120.0/victoria-metrics-linux-amd64-v1.120.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.120.0.tar.gz
|
||||
|
||||
# Run single-node VictoriaMetrics with the given scrape.yaml
|
||||
./victoria-metrics-prod -promscrape.config=scrape.yaml
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
|
||||
|
||||
VictoriaTraces is an open-source, user-friendly database designed for storing and querying distributed [tracing data](https://en.wikipedia.org/wiki/Tracing_(software)),
|
||||
built by the [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) team.
|
||||
|
||||
VictoriaTraces provides the following features:
|
||||
- It is resource-efficient and fast. It uses up to [**3.7x less RAM and up to 2.6x less CPU**](https://victoriametrics.com/blog/dev-note-distributed-tracing-with-victorialogs/) than other solutions such as Grafana Tempo.
|
||||
- VictoriaTraces' capacity and performance scales linearly with the available resources (CPU, RAM, disk IO, disk space).
|
||||
- It accepts trace spans in the popular [OpenTelemetry protocol](https://opentelemetry.io/docs/specs/otel/protocol/)(OTLP).
|
||||
- It provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json)
|
||||
to integrate with [Grafana](https://grafana.com/docs/grafana/latest/datasources/jaeger/) or [Jaeger Frontend](https://www.jaegertracing.io/docs/2.6/frontend-ui/).
|
||||
|
||||
## Quick Start
|
||||
|
||||
The easiest way to get started with VictoriaTraces is by using the pre-built Docker Compose file.
|
||||
It launches VictoriaTraces, Grafana, and HotROD (a sample application that generates tracing data).
|
||||
Everything is preconfigured and connected out of the box, so you can start exploring distributed tracing within minutes.
|
||||
|
||||
Clone the repository:
|
||||
```bash
|
||||
git clone -b victoriatraces --single-branch https://github.com/VictoriaMetrics/VictoriaMetrics.git;
|
||||
cd VictoriaMetrics;
|
||||
```
|
||||
|
||||
Run VictoriaTraces with Docker Compose:
|
||||
```bash
|
||||
make docker-vt-single-up;
|
||||
```
|
||||
|
||||
Now you can open HotROD at [http://localhost:8080](http://localhost:8080) and click around to generate some traces.
|
||||
Then, you can open Grafana at [http://localhost:3000/explore](http://localhost:3000/explore) and explore the traces using the Jaeger data source.
|
||||
|
||||
To stop the services, run:
|
||||
```bash
|
||||
make docker-vt-single-down;
|
||||
```
|
||||
|
||||
You can read more about docker compose and what's available there in the [Docker compose environment for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/deployment/victoriatraces/deployment/docker/README.md#victoriatraces-server).
|
||||
|
||||
### How to build from sources
|
||||
|
||||
Building from sources is reasonable when developing additional features specific to your needs or when testing bugfixes.
|
||||
|
||||
{{% collapse name="How to build from sources" %}}
|
||||
|
||||
Clone VictoriaMetrics repository:
|
||||
```bash
|
||||
git clone -b victoriatraces --single-branch https://github.com/VictoriaMetrics/VictoriaMetrics.git;
|
||||
cd VictoriaMetrics;
|
||||
```
|
||||
|
||||
#### Build binary with go build
|
||||
|
||||
1. [Install Go](https://golang.org/doc/install).
|
||||
1. Run `make victoria-traces` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces).
|
||||
It builds `victoria-traces` binary and puts it into the `bin` folder.
|
||||
|
||||
#### Build binary with Docker
|
||||
|
||||
1. [Install docker](https://docs.docker.com/install/).
|
||||
1. Run `make victoria-traces-prod` from the root folder of [the repository](https://github.com/VictoriaMetrics/VictoriaTraces).
|
||||
It builds `victoria-traces-prod` binary and puts it into the `bin` folder.
|
||||
|
||||
#### Building docker images
|
||||
|
||||
Run `make package-victoria-traces`. It builds `victoriametrics/victoria-traces:<PKG_TAG>` docker image locally.
|
||||
`<PKG_TAG>` is auto-generated image tag, which depends on source code in the repository.
|
||||
The `<PKG_TAG>` may be manually set via `PKG_TAG=foobar make package-victoria-traces`.
|
||||
|
||||
The base docker image is [alpine](https://hub.docker.com/_/alpine) but it is possible to use any other base image
|
||||
by setting it via `<ROOT_IMAGE>` environment variable.
|
||||
For example, the following command builds the image on top of [scratch](https://hub.docker.com/_/scratch) image:
|
||||
|
||||
```sh
|
||||
ROOT_IMAGE=scratch make package-victoria-traces
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
### Configure and run VictoriaTraces
|
||||
|
||||
VictoriaTraces can be run with:
|
||||
```shell
|
||||
/path/to/victoria-traces -storageDataPath=victoria-traces-data -retentionPeriod=7d
|
||||
```
|
||||
|
||||
or with Docker:
|
||||
```shell
|
||||
docker run --rm -it -p 9428:9428 -v ./victoria-traces-data:/victoria-traces-data \
|
||||
docker.io/victoriametrics/victoria-traces:latest -storageDataPath=victoria-traces-data
|
||||
```
|
||||
|
||||
VictoriaTraces is configured via command-line flags.
|
||||
All the command-line flags have sane defaults, so there is no need in tuning them in general case.
|
||||
VictoriaTraces runs smoothly in most environments without additional configuration.
|
||||
|
||||
Pass `-help` to VictoriaTraces in order to see the list of supported command-line flags with their description and default values:
|
||||
|
||||
```bash
|
||||
/path/to/victoria-traces -help
|
||||
```
|
||||
|
||||
The following command-line flags are used the most:
|
||||
|
||||
* `-storageDataPath` - VictoriaTraces stores all the data in this directory. The default path is `victoria-traces-data` in the current working directory.
|
||||
* `-retentionPeriod` - retention for stored data. Older data is automatically deleted. Default retention is 7 days.
|
||||
|
||||
Once it's running, it will listen to port `9428` (`-httpListenAddr`) and provide the following APIs:
|
||||
1. for ingestion:
|
||||
```
|
||||
http://<victoria-traces>:9428/insert/opentelemetry/v1/traces
|
||||
```
|
||||
2. for querying:
|
||||
```
|
||||
http://<victoria-traces>:9428/select/jaeger/<endpoints>
|
||||
```
|
||||
|
||||
See [data ingestion](https://docs.victoriametrics.com/victoriatraces/data-ingestion/) and [querying](https://docs.victoriametrics.com/VictoriaTraces/querying/) for more details.
|
||||
|
||||
## How does it work
|
||||
|
||||
VictoriaTraces was initially built on top of [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/), a log database.
|
||||
It receives trace spans in OTLP format, transforms them into structured logs, and provides [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json) for querying.
|
||||
|
||||
For detailed data model and example, see: [Key Concepts](https://docs.victoriametrics.com/victoriatraces/keyConcepts).
|
||||
|
||||
|
||||
|
||||
Building VictoriaTraces in this way enables it to scale easily and linearly with the available resources, like VictoriaLogs.
|
||||
|
||||
## List of command-line flags
|
||||
|
||||
```shell
|
||||
-search.traceMaxDurationWindow
|
||||
The lookbehind/lookahead window of searching for the rest trace spans after finding one span.
|
||||
It allows extending the search start time and end time by `-search.traceMaxDurationWindow` to make sure all spans are included.
|
||||
It affects both Jaeger's `/api/traces` and `/api/traces/<trace_id>` APIs. (default: 10m)
|
||||
-search.traceMaxServiceNameList
|
||||
The maximum number of service name can return in a get service name request.
|
||||
This limit affects Jaeger's `/api/services` API. (default: 1000)
|
||||
-search.traceMaxSpanNameList
|
||||
The maximum number of span name can return in a get span name request.
|
||||
This limit affects Jaeger's `/api/services/*/operations` API. (default: 1000)
|
||||
-search.traceSearchStep
|
||||
Splits the [0, now] time range into many small time ranges by -search.traceSearchStep
|
||||
when searching for spans by `trace_id`. Once it finds spans in a time range, it performs an additional search according to `-search.traceMaxDurationWindow` and then stops.
|
||||
It affects Jaeger's `/api/traces/<trace_id>` API. (default: 1d)
|
||||
-search.traceServiceAndSpanNameLookbehind
|
||||
The time range of searching for service names and span names.
|
||||
It affects Jaeger's `/api/services` and `/api/services/*/operations` APIs. (default: 7d)
|
||||
```
|
||||
|
||||
See also: [VictoriaLogs - List of Command-line flags](https://docs.victoriametrics.com/victorialogs/#list-of-command-line-flags)
|
||||
@@ -1,11 +0,0 @@
|
||||
---
|
||||
title: VictoriaTraces
|
||||
menu:
|
||||
docs:
|
||||
weight: 17
|
||||
identifier: victoriatraces
|
||||
pageRef: "/"
|
||||
tags:
|
||||
- traces
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -1,90 +0,0 @@
|
||||
> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
|
||||
|
||||
[VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/) can accept trace spans via [the OpenTelemetry protocol (OTLP)](https://opentelemetry.io/docs/specs/otlp/).
|
||||
|
||||
## HTTP APIs
|
||||
|
||||
### Opentelemetry API
|
||||
|
||||
VictoriaTraces provides the following API for OpenTelemetry data ingestion:
|
||||
|
||||
- `/insert/opentelemetry/v1/traces`
|
||||
|
||||
See more details [in this docs](https://docs.victoriametrics.com/victoriatraces/data-ingestion/opentelemetry/).
|
||||
|
||||
### HTTP parameters
|
||||
|
||||
VictoriaTraces accepts optional HTTP parameters at data ingestion HTTP API via [HTTP query string parameters](https://en.wikipedia.org/wiki/Query_string), or via [HTTP headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields).
|
||||
|
||||
HTTP query string parameters have priority over HTTP Headers.
|
||||
|
||||
#### HTTP Query string parameters
|
||||
|
||||
All the [HTTP-based data ingestion protocols](#http-apis) support the following [HTTP query string](https://en.wikipedia.org/wiki/Query_string) args:
|
||||
|
||||
- `extra_fields` - an optional comma-separated list of [trace fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model),
|
||||
which must be added to all the ingested traces. The format of every `extra_fields` entry is `field_name=field_value`.
|
||||
If the trace entry contains fields from the `extra_fields`, then they are overwritten by the values specified in `extra_fields`.
|
||||
|
||||
- `debug` - if this arg is set to `1`, then the ingested traces aren't stored in VictoriaTraces. Instead,
|
||||
the ingested data is traceged by VictoriaTraces, so it can be investigated later.
|
||||
|
||||
See also [HTTP headers](#http-headers).
|
||||
|
||||
#### HTTP headers
|
||||
|
||||
All the [HTTP-based data ingestion protocols](#http-apis) support the following [HTTP Headers](https://en.wikipedia.org/wiki/List_of_HTTP_header_fields)
|
||||
additionally to [HTTP query args](#http-query-string-parameters):
|
||||
|
||||
- `AccountID` - accountID of the tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victoriatraces/#multitenancy) for details.
|
||||
|
||||
- `ProjectID`- projectID of the tenant to ingest data to. See [multitenancy docs](https://docs.victoriametrics.com/victoriatraces/#multitenancy) for details.
|
||||
|
||||
- `VL-Extra-Fields` - an optional comma-separated list of [trace fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model),
|
||||
which must be added to all the ingested traces. The format of every `extra_fields` entry is `field_name=field_value`.
|
||||
If the trace entry contains fields from the `extra_fields`, then they are overwritten by the values specified in `extra_fields`.
|
||||
|
||||
- `VL-Debug` - if this parameter is set to `1`, then the ingested traces aren't stored in VictoriaTraces. Instead,
|
||||
the ingested data is traceged by VictoriaTraces, so it can be investigated later.
|
||||
|
||||
See also [HTTP Query string parameters](#http-query-string-parameters).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
The following command can be used for verifying whether the data is successfully ingested into VictoriaTraces:
|
||||
|
||||
```sh
|
||||
curl http://<victoria-traces>:9428/select/logsql/query -d 'query=*' | head
|
||||
```
|
||||
|
||||
This command selects all the data ingested into VictoriaTraces via [HTTP query API](https://docs.victoriametrics.com/victoriatraces/querying/#http-api)
|
||||
using [any value filter](https://docs.victoriametrics.com/victorialogs/logsql/#any-value-filter),
|
||||
while `head` cancels query execution after reading the first 10 trace spans. See [these docs](https://docs.victoriametrics.com/victoriatraces/querying/#command-line)
|
||||
for more details on how `head` integrates with VictoriaTraces.
|
||||
|
||||
The response by default contains all the [trace span fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model).
|
||||
See [how to query specific fields](https://docs.victoriametrics.com/victoriatraces/logsql/#querying-specific-fields).
|
||||
|
||||
VictoriaTraces provides the following command-line flags, which can help debugging data ingestion issues:
|
||||
|
||||
- `-logNewStreams` - if this flag is passed to VictoriaTraces, then it traces all the newly
|
||||
registered [streams](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields).
|
||||
This may help debugging [high cardinality issues](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#high-cardinality).
|
||||
- `-logIngestedRows` - if this flag is passed to VictoriaTraces, then it traces all the ingested
|
||||
[trace span entries](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model).
|
||||
See also `debug` [parameter](#http-parameters).
|
||||
|
||||
VictoriaTraces exposes various [metrics](https://docs.victoriametrics.com/victoriatraces/#monitoring), which may help debugging data ingestion issues:
|
||||
|
||||
- `vl_rows_ingested_total` - the number of ingested [trace span entries](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model)
|
||||
since the last VictoriaTraces restart. If this number increases over time, then trace spans are successfully ingested into VictoriaTraces.
|
||||
The ingested trace spans can be inspected in the following ways:
|
||||
- By passing `debug=1` parameter to every request to [data ingestion APIs](#http-apis). The ingested spans aren't stored in VictoriaTraces
|
||||
in this case. Instead, they are logged, so they can be investigated later.
|
||||
The `vl_rows_dropped_total` [metric](https://docs.victoriametrics.com/victoriatraces/#monitoring) is incremented for each logged row.
|
||||
- By passing `-logIngestedRows` command-line flag to VictoriaTraces. In this case it traces all the ingested data, so it can be investigated later.
|
||||
- `vl_streams_created_total` - the number of created [trace streams](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields)
|
||||
since the last VictoriaTraces restart. If this metric grows rapidly during extended periods of time, then this may lead
|
||||
to [high cardinality issues](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#high-cardinality).
|
||||
The newly created trace streams can be inspected in traces by passing `-logNewStreams` command-line flag to VictoriaTraces.
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
---
|
||||
title: Data ingestion
|
||||
weight: 3
|
||||
menu:
|
||||
docs:
|
||||
identifier: victoriatraces-data-ingestion
|
||||
parent: "victoriatraces"
|
||||
weight: 3
|
||||
tags:
|
||||
- traces
|
||||
aliases:
|
||||
- /victoriatraces/data-ingestion/
|
||||
- /victoriatraces/data-ingestion/index.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
@@ -1,82 +0,0 @@
|
||||
---
|
||||
weight: 4
|
||||
title: OpenTelemetry setup
|
||||
disableToc: true
|
||||
menu:
|
||||
docs:
|
||||
parent: "victoriatraces-data-ingestion"
|
||||
weight: 4
|
||||
tags:
|
||||
- traces
|
||||
aliases:
|
||||
- /victoriatraces/data-ingestion/OpenTelemetry.html
|
||||
---
|
||||
|
||||
> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
|
||||
|
||||
VictoriaTraces supports both client open-telemetry [SDK](https://opentelemetry.io/docs/languages/) and [collector](https://opentelemetry.io/docs/collector/).
|
||||
|
||||
## Client SDK
|
||||
|
||||
The OpenTelemetry provides detailed document and examples for various programming languages:
|
||||
- [C++](https://opentelemetry.io/docs/languages/cpp/)
|
||||
- [C#/.NET](https://opentelemetry.io/docs/languages/dotnet/)
|
||||
- [Erlang/Elixir](https://opentelemetry.io/docs/languages/erlang/)
|
||||
- [Go](https://opentelemetry.io/docs/languages/go/)
|
||||
- [Java](https://opentelemetry.io/docs/languages/java/)
|
||||
- [JavaScript](https://opentelemetry.io/docs/languages/js/)
|
||||
- [PHP](https://opentelemetry.io/docs/languages/php/)
|
||||
- [Python](https://opentelemetry.io/docs/languages/python/)
|
||||
- [Ruby](https://opentelemetry.io/docs/languages/ruby/)
|
||||
- [Rust](https://opentelemetry.io/docs/languages/rust/)
|
||||
- [Swift](https://opentelemetry.io/docs/languages/swift/)
|
||||
|
||||
To send data to VictoriaTraces, specify the `EndpointURL` for http-exporter builder to `http://<victoria-traces>:9428/insert/opentelemetry/v1/traces`.
|
||||
|
||||
Consider the following example for Go SDK:
|
||||
|
||||
```go
|
||||
traceExporter, err := otlptracehttp.New(ctx,
|
||||
otlptracehttp.WithEndpointURL("http://<victoria-traces>:9428/insert/opentelemetry/v1/traces"),
|
||||
)
|
||||
```
|
||||
|
||||
VictoriaTraces automatically use `service.name` in **resource attributes** and `name` in **span** as [stream fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#stream-fields).
|
||||
While the remaining data (including [resource](https://opentelemetry.io/docs/specs/otel/overview/#resources), [instrumentation scope](https://opentelemetry.io/docs/specs/otel/common/instrumentation-scope/), and fields in [span](https://opentelemetry.io/docs/specs/otel/trace/api/#span), like `trace_id`, `span_id`, span `attributes` and more) are stored as [regular fields](https://docs.victoriametrics.com/victoriatraces/keyconcepts/#data-model):
|
||||
|
||||
VictoriaTraces supports other HTTP headers - see the list [here](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers).
|
||||
|
||||
The ingested trace spans can be queried according to [these docs](https://docs.victoriametrics.com/victoriatraces/querying/).
|
||||
|
||||
## Collector configuration
|
||||
|
||||
VictoriaTraces supports receiving traces from the following OpenTelemetry collector:
|
||||
|
||||
* [OpenTelemetry](#opentelemetry)
|
||||
|
||||
### OpenTelemetry
|
||||
|
||||
To send the collected traces to VictoriaTraces, specify traces endpoint for [OTLP/HTTP exporter](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/otlphttpexporter/README.md) in configuration file:
|
||||
|
||||
```yaml
|
||||
exporters:
|
||||
otlphttp:
|
||||
traces_endpoint: http://<victoria-traces>:9428/insert/opentelemetry/v1/traces
|
||||
```
|
||||
|
||||
VictoriaTraces supports various HTTP headers, which can be used during data ingestion - see the list [here](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#http-headers).
|
||||
These headers can be passed to OpenTelemetry exporter config via `headers` options. For example, the following configs add (or overwrites) `foo: bar` field to each trace span during data ingestion:
|
||||
|
||||
```yaml
|
||||
exporters:
|
||||
otlphttp:
|
||||
traces_endpoint: http://<victoria-traces>:9428/insert/opentelemetry/v1/traces
|
||||
headers:
|
||||
VL-Extra-Fields: foo=bar
|
||||
```
|
||||
|
||||
See also:
|
||||
|
||||
* [Data ingestion troubleshooting](https://docs.victoriametrics.com/victoriatraces/data-ingestion/#troubleshooting).
|
||||
* [How to query VictoriaTraces](https://docs.victoriametrics.com/victoriatraces/querying/).
|
||||
* [Docker-compose demo for OpenTelemetry collector integration with VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victoriatraces/opentelemetry-collector).
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 28 KiB |
@@ -1,234 +0,0 @@
|
||||
---
|
||||
weight: 2
|
||||
title: Key concepts
|
||||
menu:
|
||||
docs:
|
||||
identifier: vt-key-concepts
|
||||
parent: victoriatraces
|
||||
weight: 2
|
||||
title: Key concepts
|
||||
tags:
|
||||
- traces
|
||||
aliases:
|
||||
- /victoriatraces/keyConcepts.html
|
||||
---
|
||||
|
||||
> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
|
||||
|
||||
## Data model
|
||||
|
||||
VictoriaTraces is built on VictoriaLogs. It's recommended to go through [the data model of VictoriaLogs](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) first.
|
||||
|
||||
**Every [trace span](https://github.com/open-telemetry/opentelemetry-proto/blob/v1.7.0/opentelemetry/proto/trace/v1/trace.proto) must contain `service.name` in its resource attributes and a span `name`**. They will be used as the [stream fields](#stream-fields). All other data of the span will be mapped as the ordinary fields.
|
||||
|
||||
For example, here's how a trace span looks like in an OTLP export request:
|
||||
|
||||
{{% collapse name="OTLP trace span" %}}
|
||||
|
||||
```json
|
||||
{
|
||||
"ResourceSpans": [{
|
||||
"Resource": {
|
||||
"Attributes": {
|
||||
"service.name": "payment",
|
||||
"telemetry.sdk.language": "nodejs",
|
||||
"telemetry.sdk.name": "opentelemetry",
|
||||
"telemetry.sdk.version": "1.30.1",
|
||||
"container.id": "18ee03279d38ed0e0eedad037c260df78dfc3323aa662ca14a2d38fcc8bf3762",
|
||||
"service.namespace": "opentelemetry-demo",
|
||||
"service.version": "2.0.2",
|
||||
"host.name": "18ee03279d38",
|
||||
"host.arch": "arm64",
|
||||
"os.type": "linux",
|
||||
"os.version": "6.10.14-linuxkit",
|
||||
"process.pid": 17,
|
||||
"process.executable.name": "node",
|
||||
"process.executable.path": "/usr/local/bin/node",
|
||||
"process.command_args": [
|
||||
"/usr/local/bin/node",
|
||||
"--require",
|
||||
"./opentelemetry.js",
|
||||
"/usr/src/app/index.js"
|
||||
],
|
||||
"process.runtime.version": "22.16.0",
|
||||
"process.runtime.name": "nodejs",
|
||||
"process.runtime.description": "Node.js",
|
||||
"process.command": "/usr/src/app/index.js",
|
||||
"process.owner": "node"
|
||||
}
|
||||
},
|
||||
"ScopeSpans": [{
|
||||
"Scope": {
|
||||
"Name": "@opentelemetry/instrumentation-net",
|
||||
"Version": "0.43.1",
|
||||
"Attributes": null,
|
||||
"DroppedAttributesCount": 0
|
||||
},
|
||||
"Spans": [{
|
||||
"TraceID": "769d28c4b8633dc9de2cc421d1a1616f",
|
||||
"SpanID": "2a1f3c4bda1d0e43",
|
||||
"TraceState": "",
|
||||
"ParentSpanID": "3ea61f2d2a5a003d",
|
||||
"Flags": 0,
|
||||
"Name": "tcp.connect",
|
||||
"Kind": 1,
|
||||
"StartTimeUnixNano": 1750044408780000000,
|
||||
"EndTimeUnixNano": 1750044408796828584,
|
||||
"Attributes": {
|
||||
"net.transport": "ip_tcp",
|
||||
"net.peer.name": "169.254.169.254",
|
||||
"net.peer.port": 80,
|
||||
"net.peer.ip": "169.254.169.254",
|
||||
"net.host.ip": "172.18.0.14",
|
||||
"net.host.port": 50722
|
||||
},
|
||||
"DroppedAttributesCount": 0,
|
||||
"Events": null,
|
||||
"DroppedEventsCount": 0,
|
||||
"Links": null,
|
||||
"DroppedLinksCount": 0,
|
||||
"Status": {
|
||||
"Message": "",
|
||||
"Code": 0
|
||||
}
|
||||
}],
|
||||
"SchemaURL": ""
|
||||
}],
|
||||
"SchemaURL": ""
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
And here's how this trace span looks like in VictoriaTraces:
|
||||
|
||||
{{% collapse name="span in VictoriaTraces" %}}
|
||||
|
||||
```json
|
||||
{
|
||||
"_time": "2025-06-16T03:26:48.796828584Z",
|
||||
"_stream_id": "00000000000000006fc12c07dedb6f693e73fc7b11b943e6",
|
||||
"_stream": "{name=\"tcp.connect\",resource_attr:service.name=\"payment\"}",
|
||||
"_msg": "-",
|
||||
"dropped_attributes_count": "0",
|
||||
"dropped_events_count": "0",
|
||||
"dropped_links_count": "0",
|
||||
"flags": "0",
|
||||
"kind": "1",
|
||||
"name": "tcp.connect",
|
||||
"resource_attr:container.id": "18ee03279d38ed0e0eedad037c260df78dfc3323aa662ca14a2d38fcc8bf3762",
|
||||
"resource_attr:host.arch": "arm64",
|
||||
"resource_attr:host.name": "18ee03279d38",
|
||||
"resource_attr:os.type": "linux",
|
||||
"resource_attr:os.version": "6.10.14-linuxkit",
|
||||
"resource_attr:process.command": "/usr/src/app/index.js",
|
||||
"resource_attr:process.command_args": "[\"/usr/local/bin/node\",\"--require\",\"./opentelemetry.js\",\"/usr/src/app/index.js\"]",
|
||||
"resource_attr:process.executable.name": "node",
|
||||
"resource_attr:process.executable.path": "/usr/local/bin/node",
|
||||
"resource_attr:process.owner": "node",
|
||||
"resource_attr:process.pid": "17",
|
||||
"resource_attr:process.runtime.description": "Node.js",
|
||||
"resource_attr:process.runtime.name": "nodejs",
|
||||
"resource_attr:process.runtime.version": "22.16.0",
|
||||
"resource_attr:service.name": "payment",
|
||||
"resource_attr:service.namespace": "opentelemetry-demo",
|
||||
"resource_attr:service.version": "2.0.2",
|
||||
"resource_attr:telemetry.sdk.language": "nodejs",
|
||||
"resource_attr:telemetry.sdk.name": "opentelemetry",
|
||||
"resource_attr:telemetry.sdk.version": "1.30.1",
|
||||
"scope_name": "@opentelemetry/instrumentation-net",
|
||||
"scope_version": "0.43.1",
|
||||
"span_attr:net.transport": "ip_tcp",
|
||||
"duration": "16828584",
|
||||
"end_time_unix_nano": "1750044408796828584",
|
||||
"parent_span_id": "3ea61f2d2a5a003d",
|
||||
"span_attr:net.host.ip": "172.18.0.14",
|
||||
"span_attr:net.host.port": "50722",
|
||||
"span_attr:net.peer.ip": "169.254.169.254",
|
||||
"span_attr:net.peer.name": "169.254.169.254",
|
||||
"span_attr:net.peer.port": "80",
|
||||
"span_id": "2a1f3c4bda1d0e43",
|
||||
"start_time_unix_nano": "1750044408780000000",
|
||||
"status_code": "0",
|
||||
"trace_id": "769d28c4b8633dc9de2cc421d1a1616f"
|
||||
}
|
||||
```
|
||||
|
||||
{{% /collapse %}}
|
||||
|
||||
### Special mappings
|
||||
|
||||
There are some special mappings when transforming a trace span into VictoriaTraces data model:
|
||||
1. Empty attribute values in trace spans are replaced with `-`.
|
||||
2. Resource, scope and span attributes are stored with corresponding prefixes `resource_attr`, `scope_attr` and `span_attr:` accordingly.
|
||||
3. For some attributes within a list (event list, link list in span), a corresponding prefix and index (such as `event:0:` and `event:0:event_attr:`) is added.
|
||||
4. The `duration` field does not exist in the OTLP request, but for query efficiency, it's calculated during ingestion and stored as a separated field.
|
||||
|
||||
VictoriaTraces automatically indexes all the fields for ingested trace spans.
|
||||
This enables [full-text search](https://docs.victoriametrics.com/victorialogs/logsql/) across all the fields.
|
||||
|
||||
VictoriaTraces stores data in different fields. There are some special fields in addition to [arbitrary fields](#other-fields):
|
||||
|
||||
* [`_time` field](#time-field)
|
||||
* [`_stream` and `_stream_id` fields](#stream-fields)
|
||||
|
||||
### Time field
|
||||
|
||||
The ingested [trace spans](#data-model) may contain `_time` field with the timestamp of the ingested trace span.
|
||||
|
||||
By default, VictoriaTraces use the `EndTimeUnixNano` in trace span as the `_time` field.
|
||||
|
||||
The `_time` field is used by [time filter](https://docs.victoriametrics.com/victorialogs/logsql/#time-filter) for quickly narrowing down the search to the selected time range.
|
||||
|
||||
### Stream fields
|
||||
|
||||
As service name and span name usually identify the application instance, VictoriaTraces uses `service.name` in resource attributes and `name` in span
|
||||
as the stream fields.
|
||||
|
||||
VictoriaTraces optimizes storing and [querying](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter) of individual trace span streams.
|
||||
This provides the following benefits:
|
||||
- Reduced disk space usage, since a trace span stream from a single application instance is usually compressed better
|
||||
than a mixed trace span stream from multiple distinct applications.
|
||||
|
||||
- Increased query performance, since VictoriaTraces needs to scan lower amounts of data
|
||||
when [searching by stream fields](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter).
|
||||
|
||||
Every ingested trace span is associated with a trace span stream. Every trace span stream consists of the following special fields:
|
||||
|
||||
- `_stream_id` - this is an unique identifier for the trace span stream. All the trace spans for the particular stream can be selected
|
||||
via [`_stream_id:...` filter](https://docs.victoriametrics.com/victorialogs/logsql/#_stream_id-filter).
|
||||
|
||||
- `_stream` - this field contains stream labels in the format similar to [labels in Prometheus metrics](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels):
|
||||
```
|
||||
{resource_attr:service_name="svc name", name="span name"}
|
||||
```
|
||||
The `_stream` field can be searched with [stream filters](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter).
|
||||
|
||||
#### High cardinality
|
||||
|
||||
Some fields in the [trace spans](#data-model) may contain big number of unique values across log entries.
|
||||
For example, fields with names such as `trace_id`, `span_id` or `ip` tend to contain big number of unique values.
|
||||
VictoriaTraces works perfectly with such fields unless they are associated with [trace span streams](#stream-fields).
|
||||
|
||||
**Never** associate high-cardinality fields with [trace span streams](#stream-fields), since this may lead to the following issues:
|
||||
|
||||
- Performance degradation during [data ingestion](https://docs.victoriametrics.com/victoriatraces/data-ingestion/)
|
||||
and [querying](https://docs.victoriametrics.com/victoriatraces/querying/)
|
||||
- Increased memory usage
|
||||
- Increased CPU usage
|
||||
- Increased disk space usage
|
||||
- Increased disk read / write IO
|
||||
|
||||
VictoriaTraces exposes `vl_streams_created_total` [metric](https://docs.victoriametrics.com/victorialogs/#monitoring),
|
||||
which shows the number of created streams since the last VictoriaTraces restart. If this metric grows at a rapid rate
|
||||
during long period of time, then there are high chances of high cardinality issues mentioned above.
|
||||
VictoriaTraces can log all the newly registered streams when `-logNewStreams` command-line flag is passed to it.
|
||||
This can help narrowing down and eliminating high-cardinality fields from [trace span streams](#stream-fields).
|
||||
|
||||
### Other fields
|
||||
|
||||
Every ingested log entry may contain arbitrary number of [fields](#data-model) additionally to [`_time`](#time-field).
|
||||
For example, `name`, `span_attr:ip`, `span_attr:ip`, etc. Such fields can be used for simplifying and optimizing [search queries](https://docs.victoriametrics.com/victorialogs/logsql/).
|
||||
|
||||
See [LogsQL docs](https://docs.victoriametrics.com/victorialogs/logsql/) for more details.
|
||||
File diff suppressed because one or more lines are too long
@@ -1,16 +0,0 @@
|
||||
---
|
||||
title: Querying
|
||||
weight: 4
|
||||
menu:
|
||||
docs:
|
||||
identifier: victoriatraces-querying
|
||||
parent: "victoriatraces"
|
||||
weight: 4
|
||||
tags:
|
||||
- traces
|
||||
aliases:
|
||||
- /VictoriaTraces/querying/
|
||||
- /victoriatraces/querying/
|
||||
- /victoriatraces/querying/index.html
|
||||
---
|
||||
{{% content "README.md" %}}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 58 KiB |
@@ -1,28 +0,0 @@
|
||||
---
|
||||
weight: 4
|
||||
title: Visualization in Grafana
|
||||
disableToc: true
|
||||
menu:
|
||||
docs:
|
||||
parent: "victoriatraces-querying"
|
||||
weight: 4
|
||||
tags:
|
||||
- traces
|
||||
aliases:
|
||||
- /victoriatraces/querying/grafana.html
|
||||
---
|
||||
|
||||
> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
|
||||
|
||||
[Grafana Jaeger Datasource](https://grafana.com/docs/grafana/latest/datasources/jaeger/) allows you to query and visualize VictoriaTraces data in Grafana.
|
||||
|
||||
|
||||
|
||||
Simply click "Add new data source" on Grafana, and then fill your VictoriaTraces URL to "Connection.URL".
|
||||
|
||||
The URL format for VictoriaTraces single-node is:
|
||||
```
|
||||
http://<victoria-traces>:9428/select/jaeger
|
||||
```
|
||||
|
||||
Finally, click "Save & Test" at the bottom to complete the process.
|
||||
@@ -1,65 +0,0 @@
|
||||
---
|
||||
weight: 4
|
||||
title: Visualization in Jaeger UI
|
||||
disableToc: true
|
||||
menu:
|
||||
docs:
|
||||
parent: "victoriatraces-querying"
|
||||
weight: 4
|
||||
tags:
|
||||
- traces
|
||||
aliases:
|
||||
- /victoriatraces/querying/jaeger-frontend.html
|
||||
---
|
||||
|
||||
> VictoriaTraces is currently under active development and not ready for production use. It is built on top of VictoriaLogs and therefore shares some flags and APIs. These will be fully separated once VictoriaTraces reaches a stable release. Until then, features may change or break without notice.
|
||||
|
||||
[Jaeger UI](https://github.com/jaegertracing/jaeger-ui) is the official frontend that ships with Jaeger. It queries [Jaeger Query Service JSON APIs](https://www.jaegertracing.io/docs/2.6/apis/#internal-http-json)
|
||||
and visualizes the response trace data.
|
||||
|
||||
## Deploy Jaeger UI
|
||||
|
||||
You can get Jaeger UI from [release page](https://github.com/jaegertracing/jaeger-ui/releases/tag/v1.70.0).
|
||||
|
||||
As it provides only assets and source code, an HTTP server is needed for serving requests.
|
||||
|
||||
### Nginx Example
|
||||
|
||||
Here's an example where we use Nginx to:
|
||||
- Serve static content of Jaeger UI.
|
||||
- Forward query requests to VictoriaTraces.
|
||||
|
||||
Assume you already have:
|
||||
1. VictoriaTraces running locally and listening on port `:9428`.
|
||||
2. Jaeger UI assets (`index.html` and `/static`) located under `/path/to/jaeger-ui/build/`.
|
||||
3. Nginx Installed.
|
||||
|
||||
Create the following config file `jaeger.conf` and place it to the Nginx config folder:
|
||||
|
||||
```
|
||||
server {
|
||||
listen 8080;
|
||||
listen [::]:8080;
|
||||
server_name localhost;
|
||||
|
||||
location / {
|
||||
root /path/to/jaeger-ui/build; # change this path to your asserts location.
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
location /api {
|
||||
proxy_pass http://127.0.0.1:9428/select/jaeger/api; # change this address to VictoriaTraces' address.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Here are some common paths of Nginx config folder:
|
||||
```sh
|
||||
# Ubuntu & Install with apt
|
||||
cd /etc/nginx/sites-available/
|
||||
|
||||
# MacOS & Install with homebrew
|
||||
cd /opt/homebrew/etc/nginx/servers/
|
||||
```
|
||||
|
||||
After reloading Nginx, you should be able to visit Jaeger UI on: [http://127.0.0.1:8080/](http://127.0.0.1:8080/).
|
||||
Reference in New Issue
Block a user