mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-04 17:42:21 +03:00
Compare commits
29 Commits
issue-1060
...
v1.136.8
| Author | SHA1 | Date | |
|---|---|---|---|
|
ebc9d49e50 | ||
|
|
b2a6fba673 | ||
|
|
6100b8ba10 | ||
|
|
403d32f57f | ||
|
|
ed8ebb8314 | ||
|
|
55c8bb26db | ||
|
|
129358f9ea | ||
|
|
5d5e5b3e44 | ||
|
|
88882227f7 | ||
|
|
64e43e59a7 | ||
|
|
200a764d32 | ||
|
|
b29ad9e6ce | ||
|
|
00c0c149da | ||
|
|
542ea4788e | ||
|
|
124bdbd383 | ||
|
|
1b3e549833 | ||
|
|
c37b78f366 | ||
|
|
017bfc094d | ||
|
|
411ec81619 | ||
|
|
64ccd2ed44 | ||
|
|
89c0b1c1aa | ||
|
|
387a54d3c8 | ||
|
|
20928171a8 | ||
|
|
ff79527c7f | ||
|
|
492419c2e8 | ||
|
|
f42c56fc48 | ||
|
|
684f96759f | ||
|
|
5c3dc0f429 | ||
|
|
ca5bc3a4c4 |
40
SECURITY.md
40
SECURITY.md
@@ -1,42 +1,4 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
You can find out about our security policy and VictoriaMetrics version support on the [security page](https://docs.victoriametrics.com/victoriametrics/#security) in the documentation.
|
||||
|
||||
The following versions of VictoriaMetrics receive regular security fixes:
|
||||
|
||||
| Version | Supported |
|
||||
|--------------------------------------------------------------------------------|--------------------|
|
||||
| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
|
||||
| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
|
||||
| other releases | :x: |
|
||||
|
||||
See [this page](https://victoriametrics.com/security/) for more details.
|
||||
|
||||
## Software Bill of Materials (SBOM)
|
||||
|
||||
Every VictoriaMetrics container{{% available_from "#" %}} image published to
|
||||
[Docker Hub](https://hub.docker.com/u/victoriametrics)
|
||||
and [Quay.io](https://quay.io/organization/victoriametrics)
|
||||
includes an [SPDX](https://spdx.dev/) SBOM attestation
|
||||
generated automatically by BuildKit during
|
||||
`docker buildx build`.
|
||||
|
||||
To inspect the SBOM for an image:
|
||||
|
||||
```sh
|
||||
docker buildx imagetools inspect \
|
||||
docker.io/victoriametrics/victoria-metrics:latest \
|
||||
--format "{{ json .SBOM }}"
|
||||
```
|
||||
|
||||
To scan an image using its SBOM attestation with
|
||||
[Trivy](https://github.com/aquasecurity/trivy):
|
||||
|
||||
```sh
|
||||
trivy image --sbom-sources oci \
|
||||
docker.io/victoriametrics/victoria-metrics:latest
|
||||
```
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report any security issues to <security@victoriametrics.com>
|
||||
|
||||
@@ -14,7 +14,7 @@ type Notifier interface {
|
||||
Send(ctx context.Context, alerts []Alert, alertLabels [][]prompb.Label, notifierHeaders map[string]string) error
|
||||
// Addr returns address where alerts are sent.
|
||||
Addr() string
|
||||
// LastError returns error, that occured during last attempt to send data
|
||||
// LastError returns error, that occurred during last attempt to send data
|
||||
LastError() string
|
||||
// Close is a destructor for the Notifier
|
||||
Close()
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"hash/fnv"
|
||||
"maps"
|
||||
"net/url"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -42,6 +43,9 @@ var (
|
||||
"For example, if lookback=1h then range from now() to now()-1h will be scanned.")
|
||||
maxStartDelay = flag.Duration("group.maxStartDelay", 5*time.Minute, "Defines the max delay before starting the group evaluation. Group's start is artificially delayed for random duration on interval"+
|
||||
" [0..min(--group.maxStartDelay, group.interval)]. This helps smoothing out the load on the configured datasource, so evaluations aren't executed too close to each other.")
|
||||
ruleStripFilePath = flag.Bool("rule.stripFilePath", false, "Whether to strip rule file paths in logs and all API responses, including /metrics. "+
|
||||
"For example, file path '/path/to/tenant_id/rules.yml' will be stripped to 'groupHashID/rules.yml'. "+
|
||||
"This flag may be useful for hiding sensitive information in file paths, such as S3 bucket details.")
|
||||
)
|
||||
|
||||
// Group is an entity for grouping rules
|
||||
@@ -147,6 +151,12 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
|
||||
g.EvalDelay = &cfg.EvalDelay.D
|
||||
}
|
||||
g.id = g.CreateID()
|
||||
// strip file path from group.File after generated group ID when ruleStripFilePath is set,
|
||||
// so it won't be exposed in logs and api responses
|
||||
if *ruleStripFilePath {
|
||||
_, filename := path.Split(g.File)
|
||||
g.File = fmt.Sprintf("%d/%s", g.id, filename)
|
||||
}
|
||||
for _, h := range cfg.Headers {
|
||||
g.Headers[h.Key] = h.Value
|
||||
}
|
||||
|
||||
@@ -742,3 +742,64 @@ func parseTime(t *testing.T, s string) time.Time {
|
||||
}
|
||||
return tt
|
||||
}
|
||||
|
||||
func TestRuleStripFilePath(t *testing.T) {
|
||||
configG := config.Group{
|
||||
Name: "group",
|
||||
File: "/var/local/test/rules.yaml",
|
||||
Type: config.NewRawType("prometheus"),
|
||||
Concurrency: 1,
|
||||
Rules: []config.Rule{
|
||||
{
|
||||
ID: 0,
|
||||
Alert: "alert",
|
||||
},
|
||||
{
|
||||
ID: 1,
|
||||
Record: "record",
|
||||
},
|
||||
}}
|
||||
qb := &datasource.FakeQuerier{}
|
||||
g := NewGroup(configG, qb, 1*time.Minute, nil)
|
||||
|
||||
gID := g.id
|
||||
if g.File != "/var/local/test/rules.yaml" {
|
||||
t.Fatalf("expected file path to be unchanged; got %q instead", g.File)
|
||||
}
|
||||
|
||||
for _, r := range g.Rules {
|
||||
if ar, ok := r.(*AlertingRule); ok {
|
||||
if ar.File != "/var/local/test/rules.yaml" {
|
||||
t.Fatalf("expected rule file path to be unchanged; got %q instead", ar.File)
|
||||
}
|
||||
}
|
||||
if rr, ok := r.(*RecordingRule); ok {
|
||||
if rr.File != "/var/local/test/rules.yaml" {
|
||||
t.Fatalf("expected rule file path to be unchanged; got %q instead", rr.File)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
oldRuleStripFilePath := *ruleStripFilePath
|
||||
*ruleStripFilePath = true
|
||||
defer func() {
|
||||
*ruleStripFilePath = oldRuleStripFilePath
|
||||
}()
|
||||
g = NewGroup(configG, qb, 1*time.Minute, nil)
|
||||
|
||||
if g.File != fmt.Sprintf("%d/rules.yaml", gID) {
|
||||
t.Fatalf("expected file path to be stripped to %q; got %q instead", fmt.Sprintf("%d/rules.yaml", gID), g.File)
|
||||
}
|
||||
for _, r := range g.Rules {
|
||||
if ar, ok := r.(*AlertingRule); ok {
|
||||
if ar.File != fmt.Sprintf("%d/rules.yaml", gID) {
|
||||
t.Fatalf("expected rule file path to be unchanged; got %q instead", ar.File)
|
||||
}
|
||||
}
|
||||
if rr, ok := r.(*RecordingRule); ok {
|
||||
if rr.File != fmt.Sprintf("%d/rules.yaml", gID) {
|
||||
t.Fatalf("expected rule file path to be unchanged; got %q instead", rr.File)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,6 +252,9 @@ func (r *ApiRule) ExtendState() {
|
||||
|
||||
// ToAPI returns ApiGroup representation of g
|
||||
func (g *Group) ToAPI() *ApiGroup {
|
||||
if g == nil {
|
||||
return &ApiGroup{}
|
||||
}
|
||||
g.mu.RLock()
|
||||
defer g.mu.RUnlock()
|
||||
ag := ApiGroup{
|
||||
|
||||
@@ -402,6 +402,20 @@ func templateFuncs() textTpl.FuncMap {
|
||||
return t, nil
|
||||
},
|
||||
|
||||
// formatTime formats the given Unix timestamp with the provided layout.
|
||||
// For example: {{ now | formatTime "2006-01-02T15:04:05Z07:00" }}
|
||||
"formatTime": func(layout string, i any) (string, error) {
|
||||
v, err := toFloat64(i)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("formatTime: %w", err)
|
||||
}
|
||||
if math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return "", fmt.Errorf("formatTime: cannot convert %v to time", v)
|
||||
}
|
||||
t := timeFromUnixTimestamp(v).Time().UTC()
|
||||
return t.Format(layout), nil
|
||||
},
|
||||
|
||||
/* URLs */
|
||||
|
||||
// externalURL returns value of `external.url` flag
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
textTpl "text/template"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTemplateFuncs_StringConversion(t *testing.T) {
|
||||
@@ -103,6 +104,26 @@ func TestTemplateFuncs_Formatting(t *testing.T) {
|
||||
f("humanizeTimestamp", 1679055557, "2023-03-17 12:19:17 +0000 UTC")
|
||||
}
|
||||
|
||||
func TestTemplateFuncs_FormatTime(t *testing.T) {
|
||||
funcs := templateFuncs()
|
||||
formatTime := funcs["formatTime"].(func(layout string, i any) (string, error))
|
||||
|
||||
f := func(layout string, input any, expected string) {
|
||||
t.Helper()
|
||||
result, err := formatTime(layout, input)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for formatTime(%q, %v): %s", layout, input, err)
|
||||
}
|
||||
if result != expected {
|
||||
t.Fatalf("unexpected result for formatTime(%q, %v); got\n%s\nwant\n%s", layout, input, result, expected)
|
||||
}
|
||||
}
|
||||
|
||||
f(time.RFC3339, float64(1679055557), "2023-03-17T12:19:17Z")
|
||||
f("2006-01-02T15:04:05", int64(1679055557), "2023-03-17T12:19:17")
|
||||
f(time.RFC822, int(1679055557), "17 Mar 23 12:19 UTC")
|
||||
}
|
||||
|
||||
func mkTemplate(current, replacement any) textTemplate {
|
||||
tmpl := textTemplate{}
|
||||
if current != nil {
|
||||
|
||||
@@ -202,6 +202,7 @@
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"noValue": "0",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
@@ -239,7 +240,7 @@
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.2.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -561,8 +562,7 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -599,7 +599,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "sum(min_over_time(vm_app_uptime_seconds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
|
||||
"expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
|
||||
"format": "time_series",
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
@@ -623,6 +623,342 @@
|
||||
"title": "Overview",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows the rate of requests per user.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Requests rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows duration in seconds of user requests by quantile.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"id": 19,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Max",
|
||||
"sortDesc": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", username=~\"$user\", quantile=~\"(0.99|0.5)\"}) by (quantile, username) > 0",
|
||||
"legendFormat": "user: {{username}} q: {{ quantile}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_unauthorized_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.99|0.5)\"}) by (quantile) > 0",
|
||||
"legendFormat": "user: unauthorized q: {{ quantile}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Requests duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows the request error rate per user.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 37,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_request_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Requests error rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
@@ -688,8 +1024,8 @@
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"id": 16,
|
||||
"options": {
|
||||
@@ -705,7 +1041,7 @@
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -713,8 +1049,7 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (reason)",
|
||||
"hide": false,
|
||||
"expr": "sum(rate(vmauth_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (reason) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -728,353 +1063,7 @@
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": " The number of concurrent connections processed by vmauth reached one of limits. Possible solutions:\n- increase global limit with flag -maxConcurrentRequests\n- increase limit with flag: -maxConcurrentPerUserRequests for all users or with config option `max_concurrent_requests` per user.\n- deploy additional vmauth replicas\n- check requests latency at backend service and allocate resources to it if needed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0",
|
||||
"interval": "1m",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "unauthorized",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_concurrent_requests_limit_reached_total[$__rate_interval])) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "global at {{ $instance }}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Concurrent limit reached",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)",
|
||||
"hide": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "User requests rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"id": 37,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_request_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "User requests error rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows percent utilization of per concurrent requests capacity.",
|
||||
"description": "Shows the percentage utilization of concurrent request capacity per user.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
@@ -1158,7 +1147,7 @@
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1167,14 +1156,13 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n",
|
||||
"hide": false,
|
||||
"interval": "5m",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "User concurrent requests usage",
|
||||
"title": "Concurrent requests usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -1182,7 +1170,7 @@
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows duration in seconds of user requests by quantile.",
|
||||
"description": " The number of concurrent connections processed by vmauth reached one of limits. Possible solutions:\n- increase global limit with flag -maxConcurrentRequests\n- increase limit with flag: -maxConcurrentPerUserRequests for all users or with config option `max_concurrent_requests` per user.\n- deploy additional vmauth replicas\n- check requests latency at backend service and allocate resources to it if needed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
@@ -1196,7 +1184,7 @@
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
@@ -1235,8 +1223,7 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -1246,18 +1233,13 @@
|
||||
"x": 12,
|
||||
"y": 27
|
||||
},
|
||||
"id": 19,
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Max",
|
||||
"sortDesc": true
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
@@ -1265,7 +1247,7 @@
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1273,9 +1255,9 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", username=~\"$user\", quantile=~\"(0.99|0.5)\"}) by (quantile, username) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "user: {{username}} q: {{ quantile}}",
|
||||
"expr": "sum(rate(vmauth_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0",
|
||||
"interval": "1m",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
@@ -1285,14 +1267,24 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_unauthorized_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.99|0.5)\"}) by (quantile) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "user: unauthorized q: {{ quantile}}",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_concurrent_requests_limit_reached_total[$__rate_interval])) > 0",
|
||||
"legendFormat": "global at {{ $instance }}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "User requests duration",
|
||||
"title": "Concurrent limit reached",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -2599,7 +2591,7 @@
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
|
||||
"description": "Shows the backend request error rate per user.\nThe value can be higher than the request error rate if:\n1. A single request is retried multiple times due to transient network errors, such as proxy idle timeout misconfiguration or sockets being closed by the OS.\n2. The request fails and is retried on other backends because of configured retry_status_codes.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
@@ -2611,8 +2603,8 @@
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"axisSoftMin": 0,
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
@@ -2622,13 +2614,14 @@
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "stepAfter",
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
@@ -2638,23 +2631,20 @@
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"decimals": 0,
|
||||
"links": [],
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -2662,26 +2652,23 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 47
|
||||
"y": 46
|
||||
},
|
||||
"id": 37,
|
||||
"id": 41,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Last *",
|
||||
"sortDesc": true
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.1.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -2689,14 +2676,24 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
|
||||
"format": "time_series",
|
||||
"instant": false,
|
||||
"legendFormat": "{{job}}",
|
||||
"expr": "sum(rate(vmauth_user_request_backend_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_request_backend_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Restarts ($job)",
|
||||
"title": "Requests backend error rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -201,6 +201,7 @@
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"noValue": "0",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
@@ -238,7 +239,7 @@
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.2.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -560,8 +561,7 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -598,7 +598,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "sum(min_over_time(vm_app_uptime_seconds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
|
||||
"expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)",
|
||||
"format": "time_series",
|
||||
"instant": false,
|
||||
"interval": "",
|
||||
@@ -622,6 +622,342 @@
|
||||
"title": "Overview",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows the rate of requests per user.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Requests rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows duration in seconds of user requests by quantile.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"id": 19,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Max",
|
||||
"sortDesc": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", username=~\"$user\", quantile=~\"(0.99|0.5)\"}) by (quantile, username) > 0",
|
||||
"legendFormat": "user: {{username}} q: {{ quantile}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_unauthorized_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.99|0.5)\"}) by (quantile) > 0",
|
||||
"legendFormat": "user: unauthorized q: {{ quantile}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Requests duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows the request error rate per user.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 37,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_request_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Requests error rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -687,8 +1023,8 @@
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 9
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"id": 16,
|
||||
"options": {
|
||||
@@ -704,7 +1040,7 @@
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -712,8 +1048,7 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (reason)",
|
||||
"hide": false,
|
||||
"expr": "sum(rate(vmauth_http_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (reason) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -727,353 +1062,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": " The number of concurrent connections processed by vmauth reached one of limits. Possible solutions:\n- increase global limit with flag -maxConcurrentRequests\n- increase limit with flag: -maxConcurrentPerUserRequests for all users or with config option `max_concurrent_requests` per user.\n- deploy additional vmauth replicas\n- check requests latency at backend service and allocate resources to it if needed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 9
|
||||
},
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0",
|
||||
"interval": "1m",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "unauthorized",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_concurrent_requests_limit_reached_total[$__rate_interval])) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "global at {{ $instance }}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Concurrent limit reached",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 11,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_requests_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username)",
|
||||
"hide": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"hide": false,
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "User requests rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 9,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"id": 37,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_user_request_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_request_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "User requests error rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows percent utilization of per concurrent requests capacity.",
|
||||
"description": "Shows the percentage utilization of concurrent request capacity per user.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
@@ -1157,7 +1146,7 @@
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1166,14 +1155,13 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(\nmax_over_time(vmauth_user_concurrent_requests_current{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])\n/ \nvmauth_user_concurrent_requests_capacity{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}\n) by(username) > 0\n",
|
||||
"hide": false,
|
||||
"interval": "5m",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "User concurrent requests usage",
|
||||
"title": "Concurrent requests usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -1181,7 +1169,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows duration in seconds of user requests by quantile.",
|
||||
"description": " The number of concurrent connections processed by vmauth reached one of limits. Possible solutions:\n- increase global limit with flag -maxConcurrentRequests\n- increase limit with flag: -maxConcurrentPerUserRequests for all users or with config option `max_concurrent_requests` per user.\n- deploy additional vmauth replicas\n- check requests latency at backend service and allocate resources to it if needed",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
@@ -1195,7 +1183,7 @@
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
@@ -1234,8 +1222,7 @@
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -1245,18 +1232,13 @@
|
||||
"x": 12,
|
||||
"y": 27
|
||||
},
|
||||
"id": 19,
|
||||
"id": 10,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Max",
|
||||
"sortDesc": true
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
@@ -1264,7 +1246,7 @@
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1272,9 +1254,9 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", username=~\"$user\", quantile=~\"(0.99|0.5)\"}) by (quantile, username) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "user: {{username}} q: {{ quantile}}",
|
||||
"expr": "sum(rate(vmauth_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by(username) > 0",
|
||||
"interval": "1m",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
@@ -1284,14 +1266,24 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(vmauth_unauthorized_user_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=~\"(0.99|0.5)\"}) by (quantile) > 0",
|
||||
"hide": false,
|
||||
"legendFormat": "user: unauthorized q: {{ quantile}}",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_concurrent_requests_limit_reached_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_concurrent_requests_limit_reached_total[$__rate_interval])) > 0",
|
||||
"legendFormat": "global at {{ $instance }}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "User requests duration",
|
||||
"title": "Concurrent limit reached",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -2598,7 +2590,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"description": "Shows the number of restarts per job. The chart can be useful to identify periodic process restarts and correlate them with potential issues or anomalies. Normally, processes shouldn't restart unless restart was inited by user. The reason of restarts should be figured out by checking the logs of each specific service. ",
|
||||
"description": "Shows the backend request error rate per user.\nThe value can be higher than the request error rate if:\n1. A single request is retried multiple times due to transient network errors, such as proxy idle timeout misconfiguration or sockets being closed by the OS.\n2. The request fails and is retried on other backends because of configured retry_status_codes.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
@@ -2610,8 +2602,8 @@
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"axisSoftMin": 0,
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
@@ -2621,13 +2613,14 @@
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "stepAfter",
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
@@ -2637,23 +2630,20 @@
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"decimals": 0,
|
||||
"links": [],
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -2661,26 +2651,23 @@
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 47
|
||||
"y": 46
|
||||
},
|
||||
"id": 37,
|
||||
"id": 41,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true,
|
||||
"sortBy": "Last *",
|
||||
"sortDesc": true
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "9.1.0",
|
||||
"pluginVersion": "12.4.3",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@@ -2688,14 +2675,24 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(changes(vm_app_start_timestamp{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]) > 0) by(job)",
|
||||
"format": "time_series",
|
||||
"instant": false,
|
||||
"legendFormat": "{{job}}",
|
||||
"expr": "sum(rate(vmauth_user_request_backend_errors_total{job=~\"$job\", instance=~\"$instance\", username=~\"$user\"}[$__rate_interval])) by (username) > 0",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vmauth_unauthorized_user_request_backend_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0",
|
||||
"legendFormat": "unauthorized_user",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Restarts ($job)",
|
||||
"title": "Requests backend error rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
# It scrapes targets defined in --promscrape.config
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.140.0
|
||||
image: victoriametrics/vmagent:v1.142.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
@@ -42,14 +42,14 @@ services:
|
||||
# vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
|
||||
# where N is number of vmstorages (2 in this case).
|
||||
vmstorage-1:
|
||||
image: victoriametrics/vmstorage:v1.140.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.142.0-cluster
|
||||
volumes:
|
||||
- strgdata-1:/storage
|
||||
command:
|
||||
- "--storageDataPath=/storage"
|
||||
restart: always
|
||||
vmstorage-2:
|
||||
image: victoriametrics/vmstorage:v1.140.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.142.0-cluster
|
||||
volumes:
|
||||
- strgdata-2:/storage
|
||||
command:
|
||||
@@ -59,7 +59,7 @@ services:
|
||||
# vminsert is ingestion frontend. It receives metrics pushed by vmagent,
|
||||
# pre-process them and distributes across configured vmstorage shards.
|
||||
vminsert-1:
|
||||
image: victoriametrics/vminsert:v1.140.0-cluster
|
||||
image: victoriametrics/vminsert:v1.142.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -68,7 +68,7 @@ services:
|
||||
- "--storageNode=vmstorage-2:8400"
|
||||
restart: always
|
||||
vminsert-2:
|
||||
image: victoriametrics/vminsert:v1.140.0-cluster
|
||||
image: victoriametrics/vminsert:v1.142.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -80,7 +80,7 @@ services:
|
||||
# vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
|
||||
# vmselect collects results from configured `--storageNode` shards.
|
||||
vmselect-1:
|
||||
image: victoriametrics/vmselect:v1.140.0-cluster
|
||||
image: victoriametrics/vmselect:v1.142.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -90,7 +90,7 @@ services:
|
||||
- "--vmalert.proxyURL=http://vmalert:8880"
|
||||
restart: always
|
||||
vmselect-2:
|
||||
image: victoriametrics/vmselect:v1.140.0-cluster
|
||||
image: victoriametrics/vmselect:v1.142.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -105,7 +105,7 @@ services:
|
||||
# read requests from Grafana, vmui, vmalert among vmselects.
|
||||
# It can be used as an authentication proxy.
|
||||
vmauth:
|
||||
image: victoriametrics/vmauth:v1.140.0
|
||||
image: victoriametrics/vmauth:v1.142.0
|
||||
depends_on:
|
||||
- "vmselect-1"
|
||||
- "vmselect-2"
|
||||
@@ -119,7 +119,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.140.0
|
||||
image: victoriametrics/vmalert:v1.142.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
# It scrapes targets defined in --promscrape.config
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.140.0
|
||||
image: victoriametrics/vmagent:v1.142.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
# VictoriaMetrics instance, a single process responsible for
|
||||
# storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.140.0
|
||||
image: victoriametrics/victoria-metrics:v1.142.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
- 8089:8089
|
||||
@@ -59,7 +59,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.140.0
|
||||
image: victoriametrics/vmalert:v1.142.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
- "alertmanager"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.140.0
|
||||
image: victoriametrics/vmagent:v1.142.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
restart: always
|
||||
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.140.0
|
||||
image: victoriametrics/victoria-metrics:v1.142.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -40,7 +40,7 @@ services:
|
||||
restart: always
|
||||
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.140.0
|
||||
image: victoriametrics/vmalert:v1.142.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
VictoriaMetrics Observability Stack integrates with AI assistants through MCP servers and agent skills.
|
||||
These integrations allow AI agents and automation tools to query metrics, logs, and traces, analyze telemetry data,
|
||||
and assist engineers with debugging and observability tasks.
|
||||
VictoriaMetrics Observability Stack integrates with AI assistants through [MCP servers](https://docs.victoriametrics.com/ai-tools/#mcp-servers)
|
||||
and [agent skills](https://docs.victoriametrics.com/ai-tools/#agent-skills).
|
||||
The integrations allow AI agents and automation tools to query Metrics, Logs, and Traces, analyze telemetry data,
|
||||
and assist engineers with debugging, observability tasks, root cause analysis, anomaly detection, etc.
|
||||
|
||||
Support of [OpenTelemetry](https://docs.victoriametrics.com/opentelemetry/) for Metrics, Logs, and Traces
|
||||
makes VictoriaMetrics Observability Stack optimal for [AI observability](https://docs.victoriametrics.com/ai-tools/#ai-observability).
|
||||
Any SDK or AI assistant that can emit telemetry signals in OpenTelemetry format can be integrated with VictoriaMetrics.
|
||||
|
||||
# MCP Servers
|
||||
|
||||
@@ -72,7 +77,6 @@ Capabilities include:
|
||||
|
||||
See more details at [VictoriaMetrics/mcp-vmanomaly](https://github.com/VictoriaMetrics/mcp-vmanomaly).
|
||||
|
||||
|
||||
# Agent Skills
|
||||
|
||||
[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
|
||||
@@ -90,4 +94,17 @@ To install the available skills for AI agents, run:
|
||||
npx skills add VictoriaMetrics/skills
|
||||
```
|
||||
|
||||
See more details at [VictoriaMetrics/skills](https://github.com/VictoriaMetrics/skills).
|
||||
See more details at [VictoriaMetrics/skills](https://github.com/VictoriaMetrics/skills).
|
||||
|
||||
# AI observability
|
||||
|
||||
VictoriaMetrics Observability Stack is optimal for monitoring AI agents using auto-instrumentation libraries
|
||||
like [OpenLLMetry](https://github.com/traceloop/openllmetry), [OpenInference](https://github.com/Arize-ai/openinference),
|
||||
[OpenLIT](https://victoriametrics.com/blog/ai-agents-observability/#using-openlit).
|
||||
Please see more details in [AI Agents Observability with OpenTelemetry and the VictoriaMetrics Stack](https://victoriametrics.com/blog/ai-agents-observability).
|
||||
|
||||
AI code assistants like Claude Code, OpenAI Codex, Gemini CLI, Qwen Code, and OpenCode expose internal telemetry that
|
||||
helps to monitor cost usage, analytics, performance, compliance and improves troubleshooting experience. All major
|
||||
AI coding tools support OpenTelemetry and can be easily integrated into VictoriaMetrics Observability Stack.
|
||||
Please see more details in [Vibe coding tools observability with VictoriaMetrics Stack and OpenTelemetry
|
||||
](https://victoriametrics.com/blog/vibe-coding-observability/).
|
||||
@@ -6,31 +6,133 @@ build:
|
||||
sitemap:
|
||||
disable: true
|
||||
---
|
||||
This guide covers OpenShift cluster configuration for collecting and storing logs in Victoria Logs.
|
||||
|
||||
This guide explains how to collect and store logs from an OpenShift cluster in VictoriaLogs.
|
||||
|
||||
## Pre-Requirements
|
||||
|
||||
* [OpenShift cluster](https://www.redhat.com/en/technologies/cloud-computing/openshift)
|
||||
* Admin access to OpenShift cluster
|
||||
* [kubectl installed](https://kubernetes.io/docs/tasks/tools/install-kubectl) and configured to access OpenShift cluster
|
||||
* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl) or [oc](https://github.com/openshift/oc) installed and configured to access OpenShift cluster
|
||||
* [Helm installed](https://helm.sh/docs/intro/install)
|
||||
|
||||
## Install Red Hat OpenShift Logging operator
|
||||
> [!NOTE] Note
|
||||
> You can replace every `kubectl` command in this guide with `oc`. They are interchangeable in most cases on OpenShift clusters.
|
||||
|
||||
[Cluster logging operator](https://github.com/openshift/cluster-logging-operator) is a logging solution to support aggregated cluster logging. It is using [Vector](https://vector.dev/) for logs collection and shipping to remote storage.
|
||||
## Overview
|
||||
|
||||
In order to install the operator, navigate to the OpenShift web console and select the `Operators` tab. Then, click on `OperatorHub` and search for `Red Hat OpenShift Logging`. Click on the `Red Hat OpenShift Logging` operator and then click on `Install`.
|
||||
To collect OpenShift logs, we're going to:
|
||||
|
||||
|
||||
1. [Install VictoriaLogs](https://docs.victoriametrics.com/guides/collecting-openshift-logs-with-victoria-logs/#install-victoria-logs) in the OpenShift Cluster
|
||||
2. [Configure a service account](https://docs.victoriametrics.com/guides/collecting-openshift-logs-with-victoria-logs/#rbac-configuration) to access the logs
|
||||
3. [Install the OpenShift Logging operator](https://docs.victoriametrics.com/guides/collecting-openshift-logs-with-victoria-logs/#install-red-hat-openshift-logging-operator)
|
||||
4. [Configure a Log Forwarder](https://docs.victoriametrics.com/guides/collecting-openshift-logs-with-victoria-logs/#configure-logs-forwarding)
|
||||
5. [Test log ingestion](https://docs.victoriametrics.com/guides/collecting-openshift-logs-with-victoria-logs/#verify-logs-ingestion) in VictoriaLogs
|
||||
|
||||
## Install VictoriaLogs {#install-victoria-logs}
|
||||
|
||||
## RBAC configuration
|
||||
Run the following command to add the [VictoriaMetrics Helm repository](https://github.com/VictoriaMetrics/helm-charts):
|
||||
|
||||
Create a service account and cluster role binding for the service account to access the logs.
|
||||
OpenShift provides separate `ClusterRoles` for monitoring of different types of logs: `audit`, `infrastructure` and `application`.
|
||||
```shell
|
||||
helm repo add vm https://victoriametrics.github.io/helm-charts/
|
||||
helm repo update
|
||||
```
|
||||
|
||||
The following configuration will allow the service account to collect all types of logs:
|
||||
```yaml
|
||||
To verify that everything is set up correctly, you may run this command:
|
||||
|
||||
```shell
|
||||
helm search repo vm/
|
||||
```
|
||||
|
||||
You should get a list similar to this:
|
||||
|
||||
```text
|
||||
NAME CHART VERSION APP VERSION DESCRIPTION
|
||||
vm/victoria-logs-agent 0.1.1 v1.50.0 VictoriaLogs Agent - accepts logs from various ...
|
||||
vm/victoria-logs-collector 0.3.1 v1.50.0 VictoriaLogs Collector - collects logs from Kub...
|
||||
vm/victoria-logs-single 0.12.2 v1.50.0 The VictoriaLogs single Helm chart deploys Vict...
|
||||
...
|
||||
```
|
||||
|
||||
Create a minimal configuration file to run VictoriaLogs in OpenShift:
|
||||
|
||||
```shell
|
||||
cat <<EOF >vl-values.yml
|
||||
securityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
readOnlyRootFilesystem: true
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
runAsNonRoot: true
|
||||
EOF
|
||||
```
|
||||
|
||||
> Note that, depending on the OpenShift cluster configuration, additional security settings might be required.
|
||||
|
||||
Create a namespace for VictoriaLogs called `vl`:
|
||||
|
||||
```shell
|
||||
kubectl create namespace vl
|
||||
```
|
||||
|
||||
Install VictoriaLogs with the following command:
|
||||
|
||||
```shell
|
||||
helm upgrade --namespace vl --install vl vm/victoria-logs-single -f vl-values.yml
|
||||
```
|
||||
|
||||
You should see a message like this:
|
||||
|
||||
```text
|
||||
Release "vl" does not exist. Installing it now.
|
||||
NAME: vl
|
||||
LAST DEPLOYED: Fri Apr 17 01:05:42 2026
|
||||
NAMESPACE: vl
|
||||
STATUS: deployed
|
||||
REVISION: 1
|
||||
DESCRIPTION: Install complete
|
||||
TEST SUITE: None
|
||||
NOTES:
|
||||
The VictoriaLogs write api can be accessed via port 9428 on the following DNS name from within your cluster:
|
||||
vl-victoria-logs-single-server-0.vl-victoria-logs-single-server.vl.svc.cluster.local.
|
||||
|
||||
Logs Ingestion:
|
||||
Get the VictoriaLogs service URL by running these commands in the same shell:
|
||||
kubectl --namespace vl port-forward svc/vl-victoria-logs-single-server 9428:9428
|
||||
echo http://localhost:9428
|
||||
|
||||
Write URL inside the kubernetes cluster:
|
||||
http://vl-victoria-logs-single-server.vl.svc.cluster.local.:9428<protocol-specific-write-endpoint>
|
||||
|
||||
See the documentation for log ingestion and supported write endpoints at https://docs.victoriametrics.com/victorialogs/data-ingestion/.
|
||||
|
||||
Read Data:
|
||||
The following URL can be used to query data:
|
||||
http://vl-victoria-logs-single-server.vl.svc.cluster.local.:9428
|
||||
|
||||
See the documentation for log querying UI at https://docs.victoriametrics.com/victorialogs/querying/#web-ui or HTTP API at https://docs.victoriametrics.com/victorialogs/querying/#http-api
|
||||
```
|
||||
|
||||
Note the "Write URL" value as you'll need it later. In the example above, the value is:
|
||||
|
||||
```text
|
||||
http://vl-victoria-logs-single-server.vl.svc.cluster.local.:9428
|
||||
```
|
||||
|
||||
## RBAC Configuration
|
||||
|
||||
Create a service account and cluster role binding for the service account to collect and forward the logs.
|
||||
OpenShift provides separate `ClusterRoles` for monitoring of different types of logs: `audit`, `infrastructure`, and `application`.
|
||||
|
||||
Create a file to configure the service account and cluster role bindings:
|
||||
|
||||
```shell
|
||||
cat <<EOF >vl-rbac.yml
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@@ -77,52 +179,42 @@ roleRef:
|
||||
name: collect-application-logs
|
||||
```
|
||||
|
||||
Alternatively, you can use OpenShift console to create the service account and cluster role binding.
|
||||
Navigate to `ServiceAccounts` and click on `Create Service Account`. Fill in the name `victorialogs` and namespace `openshift-logging` and click on `Create`.
|
||||
Then, navigate to `RoleBindings` and create a binding for each `ClusterRole` for subject `victorialogs` in `openshift-logging` namespace.
|
||||
Install the roles in OpenShift with:
|
||||
|
||||
|
||||
## Install Victoria Logs
|
||||
|
||||
Add Victoria Metrics Helm [repository](https://github.com/VictoriaMetrics/helm-charts):
|
||||
```bash
|
||||
helm repo add vm https://victoriametrics.github.io/helm-charts/
|
||||
```shell
|
||||
kubectl apply -f vl-rbac.yml
|
||||
```
|
||||
|
||||
Minimal configuration for VictoriaLogs running in OpenShift:
|
||||
```yaml
|
||||
securityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
readOnlyRootFilesystem: true
|
||||
Alternatively, you can use the OpenShift web console to create the service account and cluster role binding. To do this:
|
||||
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
runAsNonRoot: true
|
||||
```
|
||||
Save the configuration to `vl.yaml`.
|
||||
1. Navigate to **ServiceAccounts**.
|
||||
2. Click on **Create Service Account**.
|
||||
3. Fill in the name `victorialogs` and namespace `openshift-logging`.
|
||||
4. Click on **Create**.
|
||||
5. Navigate to **RoleBindings**.
|
||||
6. Create a binding for each `ClusterRole` for subject `victorialogs` in `openshift-logging` namespace.
|
||||
|
||||
> Note, that depending on the OpenShift cluster configuration, additional security settings might be required.
|
||||
## Install Red Hat OpenShift Logging operator
|
||||
|
||||
Create namespace for VictoriaLogs:
|
||||
```bash
|
||||
kubectl create namespace vl
|
||||
```
|
||||
The [Cluster logging operator](https://github.com/openshift/cluster-logging-operator) is a logging solution to support aggregated cluster logging. It is using [Vector](https://vector.dev/) for log collection and shipping to remote storage.
|
||||
|
||||
Install VictoriaLogs with the following command:
|
||||
```bash
|
||||
helm upgrade --namespace vl --install vl vm/victoria-logs-single -f vl.yaml
|
||||
```
|
||||
To install the operator:
|
||||
|
||||
1. Navigate to **Ecosystem** > **Software Catalog**
|
||||
2. Search for "OpenShift Logging" and select the operator.
|
||||
|
||||
3. Press **Install**.
|
||||
4. Confirm the settings and press **Install** again.
|
||||
|
||||
|
||||
## Configure logs forwarding
|
||||
|
||||
Cluster logging operator uses `ClusterLogForwarder` resource for logs forwarding configuration.
|
||||
We need to create a `ClusterLogForwarder` resource to forward logs from the OpenShift Logging operator to VictoriaLogs.
|
||||
|
||||
Here is an example configuration to forward all cluster logs to VictoriaLogs:
|
||||
```yaml
|
||||
Run the following command to create the resource file:
|
||||
|
||||
```shell
|
||||
cat <<EOF > vl-forwarder.yml
|
||||
apiVersion: observability.openshift.io/v1
|
||||
kind: ClusterLogForwarder
|
||||
metadata:
|
||||
@@ -160,7 +252,7 @@ spec:
|
||||
- application
|
||||
name: application
|
||||
outputRefs:
|
||||
- victorialogs-audit
|
||||
- victorialogs
|
||||
- inputRefs:
|
||||
- infrastructure
|
||||
name: infrastructure
|
||||
@@ -170,21 +262,66 @@ spec:
|
||||
- audit
|
||||
name: audit
|
||||
outputRefs:
|
||||
- victorialogs
|
||||
- victorialogs-audit
|
||||
serviceAccount:
|
||||
name: victorialogs
|
||||
EOF
|
||||
```
|
||||
|
||||
It is also possible to configure logs forwarding using OpenShift console. Navigate to `Operators` tab and click on `Installed Operators`. Then, click on `Red Hat OpenShift Logging` and navigate to `ClusterLogForwarders`. Click on `Create ClusterLogForwarder`.
|
||||
Configure the forwarding to VictoriaLogs using the provided form and click on `Create`.
|
||||
|
||||
|
||||
Then install the resource with:
|
||||
|
||||
## Verify logs collection
|
||||
```shell
|
||||
kubectl apply -f vl-forwarder.yml
|
||||
```
|
||||
|
||||
To verify that logs are collected and stored in VictoriaLogs, navigate to the VictoriaLogs web interface.
|
||||
Alternatively, you can configure log forwarding in the OpenShift web console. To do this:
|
||||
1. Navigate to **Operators** tab
|
||||
2. Click on **Installed Operators**.
|
||||
3. Find **Red Hat OpenShift Logging**
|
||||
4. Navigate to **ClusterLogForwarders**.
|
||||
5. Click on **Create ClusterLogForwarder**.
|
||||
|
||||
6. Use the form to configure each type of forwarder.
|
||||
|
||||
7. Click **Create**
|
||||
|
||||
|
||||
## Verify logs ingestion
|
||||
|
||||
We can verify that logs are being collected using the VictoriaLogs VMUI.
|
||||
|
||||
First, find the service name for the VMUI with:
|
||||
|
||||
```shell
|
||||
kubectl get svc -n vl -l app.kubernetes.io/instance=vl
|
||||
```
|
||||
|
||||
You should get a result similar to this. Note the name of the service:
|
||||
|
||||
```text
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
vl-victoria-logs-single-server ClusterIP None <none> 9428/TCP 113s
|
||||
```
|
||||
|
||||
Then, port-forward the VMUI service console with:
|
||||
|
||||
```shell
|
||||
kubectl -n vl port-forward svc/vl-victoria-logs-single-server 9428:9428
|
||||
```
|
||||
|
||||
Open your browser in `http://localhost:9428/select/vmui/#/overview` and verify that logs are being collected. This overview page shows the number of log entries being consumed in real time.
|
||||
|
||||
|
||||
<figcaption style="text-align: center; font-style: italic;">Overview pane in VMUI</figcaption>
|
||||
|
||||
You can query your logs in the **Query** tab, found in `http://localhost:9428/select/vmui`.
|
||||
You can filter streams on the left side pane to filter logs and use [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/) to search for entries. Note that logs will have `log_type` attached to them to distinguish between different types of logs.
|
||||
|
||||
|
||||
<figcaption style="text-align: center; font-style: italic;">Query pane in VMUI</figcaption>
|
||||
|
||||
## See also
|
||||
|
||||
- [VictoriaLogs Quickstart](https://docs.victoriametrics.com/victorialogs/quickstart/)
|
||||
- [Logs Reference](https://docs.victoriametrics.com/victorialogs/logsql/)
|
||||
- [LogsQL Examples](https://docs.victoriametrics.com/victorialogs/logsql-examples/)
|
||||
|
||||
Logs will be available in the interface and can be queried using [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/).
|
||||
Note that logs will have `log_type` attached to them to distinguish between different types of logs.
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 653 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 402 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 848 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 569 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 380 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 542 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 469 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 378 KiB |
@@ -4,6 +4,8 @@ Those tokens can include information about the user and their tenant, which vmau
|
||||
|
||||
This guide walks through configuring Grafana with OIDC to query metrics from both single-node and cluster deployments of VictoriaMetrics.
|
||||
|
||||
> Public playground configured by using this guide can be found here: [Grafana SSO Playground](https://docs.victoriametrics.com/playgrounds/#grafana-sso-playground)
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* [Docker](https://docs.docker.com/engine/install/) and [docker compose](https://docs.docker.com/compose/) must be installed.
|
||||
@@ -238,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
|
||||
# compose.yaml
|
||||
services:
|
||||
vmsingle:
|
||||
image: victoriametrics/victoria-metrics:v1.140.0
|
||||
image: victoriametrics/victoria-metrics:v1.142.0
|
||||
|
||||
vmstorage:
|
||||
image: victoriametrics/vmstorage:v1.140.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.142.0-cluster
|
||||
|
||||
vminsert:
|
||||
image: victoriametrics/vminsert:v1.140.0-cluster
|
||||
image: victoriametrics/vminsert:v1.142.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8400
|
||||
|
||||
vmselect:
|
||||
image: victoriametrics/vmselect:v1.140.0-cluster
|
||||
image: victoriametrics/vmselect:v1.142.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8401
|
||||
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.140.0
|
||||
image: victoriametrics/vmagent:v1.142.0
|
||||
volumes:
|
||||
- ./scrape.yaml:/etc/vmagent/config.yaml
|
||||
command:
|
||||
@@ -306,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
|
||||
# compose.yaml
|
||||
services:
|
||||
vmauth:
|
||||
image: docker.io/victoriametrics/vmauth:v1.140.0
|
||||
image: docker.io/victoriametrics/vmauth:v1.142.0
|
||||
ports:
|
||||
- 8427:8427
|
||||
volumes:
|
||||
@@ -460,4 +462,3 @@ While in VictoriaMetrics single `vmauth-single` must apply the `team=admin` labe
|
||||
## Summary
|
||||
|
||||
In this guide, we demonstrated how to set up vmauth with OIDC authorization using Keycloak as the identity provider. We also showed how to provide multi-tenant access to your metrics stored in VictoriaMetrics, single-node or cluster, using Grafana and vmauth with OIDC authorization enabled.
|
||||
|
||||
|
||||
@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
|
||||
# compose.yaml
|
||||
services:
|
||||
vmstorage:
|
||||
image: victoriametrics/vmstorage:v1.140.0-cluster
|
||||
image: victoriametrics/vmstorage:v1.142.0-cluster
|
||||
|
||||
vminsert:
|
||||
image: victoriametrics/vminsert:v1.140.0-cluster
|
||||
image: victoriametrics/vminsert:v1.142.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8400
|
||||
|
||||
vmselect:
|
||||
image: victoriametrics/vmselect:v1.140.0-cluster
|
||||
image: victoriametrics/vmselect:v1.142.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8401
|
||||
ports:
|
||||
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
|
||||
# compose.yaml
|
||||
services:
|
||||
vmauth:
|
||||
image: victoriametrics/vmauth:v1.140.0-enterprise
|
||||
image: victoriametrics/vmauth:v1.142.0-enterprise
|
||||
ports:
|
||||
- 8427:8427
|
||||
volumes:
|
||||
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
|
||||
# compose.yaml
|
||||
services:
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.140.0
|
||||
image: victoriametrics/vmagent:v1.142.0
|
||||
volumes:
|
||||
- ./scrape.yaml:/etc/vmagent/config.yaml
|
||||
command:
|
||||
|
||||
@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
|
||||
# compose.yml
|
||||
services:
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.140.0
|
||||
image: victoriametrics/victoria-metrics:v1.142.0
|
||||
command:
|
||||
- "--storageDataPath=/victoria-metrics-data"
|
||||
- "--selfScrapeInterval=10s"
|
||||
@@ -128,7 +128,7 @@ services:
|
||||
- ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.140.0
|
||||
image: victoriametrics/vmalert:v1.142.0
|
||||
depends_on:
|
||||
- victoriametrics
|
||||
- alertmanager
|
||||
|
||||
@@ -93,6 +93,20 @@ Try viewing available dashboards or just browse via Grafana's explore page to qu
|
||||
|
||||
|
||||
|
||||
## Grafana SSO Playground
|
||||
|
||||
- Try it: <https://play-grafana-sso.victoriametrics.com/>
|
||||
|
||||
This playground showcases vmauth integration with Keycloak as an SSO provider to enforce data visibility for different users.
|
||||
See this guide for details: [Setup vmauth - Multi-Tenant Access with Grafana & OIDC](https://docs.victoriametrics.com/guides/grafana-vmauth-openid-configuration/)
|
||||
|
||||
Credentials for login:
|
||||
|
||||
- `developer` / `developer`
|
||||
- `platform` / `platform`
|
||||
|
||||
Both users have their access limited to show data which is necessary for their role, so you can see how vmauth and Keycloak work together to enforce multi-tenant access control.
|
||||
|
||||
## VMAnomaly Playground
|
||||
|
||||
VMAnomaly analyzes metrics, logs, or traces using VictoriaMetrics' built-in anomaly detection model to generate an [anomaly score](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score). An `anomaly_score > 1` indicates an anomalous condition that deserves attention.
|
||||
@@ -130,3 +144,14 @@ VictoriaMetrics UIs are also included in the [Explore](https://docs.victoriametr
|
||||
You can experiment with your own data during the month‑long trial without deploying VictoriaStack in your infrastructure. To get started, follow [this guide](https://docs.victoriametrics.com/victoriametrics-cloud/get-started/quickstart/).
|
||||
|
||||
|
||||
## Iximiuz Labs Playgrounds
|
||||
|
||||
[Iximiuz Labs](https://labs.iximiuz.com/) is a learning-by-doing platform. Its [playgrounds](https://labs.iximiuz.com/docs/playgrounds/what-are-playgrounds)
|
||||
are fast-booting Linux microVMs that run on a fleet of large bare-metal servers. You can start a playground right from your browser.
|
||||
Once up and running, accessing a playground is no different from SSH-ing into a remote server rented from your favorite VPS or Cloud provider.
|
||||
|
||||
Iximiuz Labs provides playgrounds for VictoriaMetrics software:
|
||||
- [VictoriaMetrics single node (on Ubuntu)](https://labs.iximiuz.com/playgrounds/victoriametrics-e2f9b613)
|
||||
- [VictoriaMetrics cluster (on Ubuntu)](https://labs.iximiuz.com/playgrounds/victoriametrics-cluster-8eacb19d)
|
||||
- [Getting Started with VictoriaMetrics on Kubernetes](https://labs.iximiuz.com/tutorials/victoriametrics-getting-started-kubernetes-0e9c0993)
|
||||
- [VictoriaMetrics Operator](https://labs.iximiuz.com/playgrounds/victoriametrics-kubernetes-9eebc258)
|
||||
|
||||
@@ -49,8 +49,8 @@ We use [labels](https://docs.github.com/en/issues/using-labels-and-milestones-to
|
||||
For example, if we weren't able to reproduce the reported bug based on the ticket description then we ask additional
|
||||
questions which could help to reproduce the issue and add `need more info` label. This label helps other maintainers
|
||||
to understand that this issue wasn't forgotten but waits for the feedback from user.
|
||||
1. `waiting for release`, assigned to issues that required code changes and those changes were merged to upstream, but not released yet.
|
||||
Once a release is made, maintainers go through all labeled issues, leave a comment about the new release, remove the label, and close the issue.
|
||||
1. `completed`, assigned to issues that required code changes and those changes were merged to upstream, but not released yet.
|
||||
Once a release is made, maintainers go through all labeled issues, leave a comment about the new release, and close the issue.
|
||||
1. `vmui`, assigned to issues related to [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) or [VictoriaLogs webui](https://docs.victoriametrics.com/victorialogs/querying/#web-ui)
|
||||
|
||||
## Pull Request checklist
|
||||
@@ -98,7 +98,7 @@ The person who merges the Pull Request is responsible for satisfying requirement
|
||||
and mention in the PR comment what was or wasn't cherry-picked.
|
||||
1. Update related issues with a meaningful message of what has changed and when it will be
|
||||
released. _This helps users to understand the change without reading PR._
|
||||
1. Add label `waiting for release` to related issues.
|
||||
1. Add label `completed` to related issues.
|
||||
1. Do not close related tickets until release is made. If ticket was auto-closed by GitHub or user - re-open it.
|
||||
|
||||
## KISS principle
|
||||
|
||||
@@ -136,7 +136,8 @@ VictoriaMetrics has no limitation on backfilling of old (historical) or out-of-o
|
||||
the specified [retention period](https://docs.victoriametrics.com/victoriametrics/#retention).
|
||||
See more about [backfilling](https://docs.victoriametrics.com/victoriametrics/#backfilling).
|
||||
|
||||
## How does VictoriaMetrics compare to other remote storage solutions for Prometheus such as [M3DB](https://github.com/m3db/m3), [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex), [Mimir](https://github.com/grafana/mimir), etc.?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## How does VictoriaMetrics compare to other remote storage solutions for Prometheus such as [M3DB](https://github.com/m3db/m3), [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex), [Mimir](https://github.com/grafana/mimir), etc.? {#how-does-victoriametrics-compare-to-other-remote-storage-solutions-for-prometheus-such-as-m3db--thanos--cortex--mimir--etc}
|
||||
|
||||
* VictoriaMetrics is easier to configure and operate than competing solutions.
|
||||
* VictoriaMetrics is more cost-efficient, since it requires less RAM, disk space, disk IO and network IO than competing solutions.
|
||||
@@ -153,7 +154,8 @@ The following articles and talks provide additional details:
|
||||
|
||||
VictoriaMetrics also [uses less RAM than Thanos components](https://github.com/thanos-io/thanos/issues/448).
|
||||
|
||||
## What is the difference between VictoriaMetrics and [QuestDB](https://questdb.io/)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## What is the difference between VictoriaMetrics and [QuestDB](https://questdb.io/)? {#what-is-the-difference-between-victoriametrics-and-questdb-}
|
||||
|
||||
* QuestDB needs 20x more storage space than VictoriaMetrics. This translates to higher storage costs and slower queries over historical data, which must be read from the disk.
|
||||
* QuestDB is significantly more difficult to set up and operate than VictoriaMetrics. Compare [setup instructions for QuestDB](https://questdb.io/docs/get-started/binaries) to [setup instructions for VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics).
|
||||
@@ -164,14 +166,16 @@ VictoriaMetrics also [uses less RAM than Thanos components](https://github.com/t
|
||||
* QuestDB [supports a smaller range of popular data ingestion protocols](https://questdb.io/docs/develop/insert-data) compared to VictoriaMetrics (compare to [the list of supported data ingestion protocols for VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data)).
|
||||
* [VictoriaMetrics supports backfilling (e.g. storing historical data) out of the box](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#backfilling), while QuestDB provides [very limited support for backfilling](https://questdb.io/blog/2021/05/10/questdb-release-6-0-tsbs-benchmark#the-problem-with-out-of-order-data).
|
||||
|
||||
## What is the difference between VictoriaMetrics and [Grafana Mimir](https://github.com/grafana/mimir)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## What is the difference between VictoriaMetrics and [Grafana Mimir](https://github.com/grafana/mimir)? {#what-is-the-difference-between-victoriametrics-and-grafana-mimir-}
|
||||
|
||||
Grafana Mimir is a [Cortex](https://github.com/cortexproject/cortex) fork, so it has the same differences
|
||||
as Cortex. See [what is the difference between VictoriaMetrics and Cortex](#what-is-the-difference-between-victoriametrics-and-cortex).
|
||||
as Cortex. See [what is the difference between VictoriaMetrics and Cortex](https://docs.victoriametrics.com/victoriametrics/faq/#what-is-the-difference-between-victoriametrics-and-cortex-).
|
||||
|
||||
See also [Grafana Mimir vs VictoriaMetrics benchmark](https://victoriametrics.com/blog/mimir-benchmark/).
|
||||
|
||||
## What is the difference between VictoriaMetrics and [Cortex](https://github.com/cortexproject/cortex)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## What is the difference between VictoriaMetrics and [Cortex](https://github.com/cortexproject/cortex)? {#what-is-the-difference-between-victoriametrics-and-cortex-}
|
||||
|
||||
VictoriaMetrics is similar to Cortex in the following aspects:
|
||||
|
||||
@@ -202,7 +206,8 @@ The main differences between Cortex and VictoriaMetrics:
|
||||
* VictoriaMetrics provides the [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) query language, while Cortex provides the [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) query language.
|
||||
* VictoriaMetrics can be queried via [Graphite's API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#graphite-api-usage).
|
||||
|
||||
## What is the difference between VictoriaMetrics and [Thanos](https://github.com/thanos-io/thanos)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## What is the difference between VictoriaMetrics and [Thanos](https://github.com/thanos-io/thanos)? {#what-is-the-difference-between-victoriametrics-and-thanos-}
|
||||
|
||||
* Thanos reuses Prometheus source code, while VictoriaMetrics is written from scratch.
|
||||
* VictoriaMetrics accepts data via the [standard remote_write API for Prometheus](https://prometheus.io/docs/practices/remote_write/),
|
||||
@@ -224,7 +229,8 @@ The main differences between Cortex and VictoriaMetrics:
|
||||
* VictoriaMetrics provides the [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) query language, while Thanos provides the [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) query language.
|
||||
* VictoriaMetrics can be queried via [Graphite's API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#graphite-api-usage).
|
||||
|
||||
## How does VictoriaMetrics compare to [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## How does VictoriaMetrics compare to [InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)? {#how-does-victoriametrics-compare-to-influxdb-}
|
||||
|
||||
* VictoriaMetrics requires [10x less RAM](https://medium.com/@valyala/insert-benchmarks-with-inch-influxdb-vs-victoriametrics-e31a41ae2893) and it [performs faster](https://medium.com/@valyala/measuring-vertical-scalability-for-time-series-databases-in-google-cloud-92550d78d8ae).
|
||||
* VictoriaMetrics uses less storage space than InfluxDB for production data.
|
||||
@@ -235,7 +241,8 @@ The main differences between Cortex and VictoriaMetrics:
|
||||
|
||||
See [How to migrate from InfluxDB to VictoriaMetrics](https://docs.victoriametrics.com/guides/migrate-from-influx/).
|
||||
|
||||
## How does VictoriaMetrics compare to [TimescaleDB](https://www.timescale.com/)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## How does VictoriaMetrics compare to [TimescaleDB](https://www.timescale.com/)? {#how-does-victoriametrics-compare-to-timescaledb-}
|
||||
|
||||
* TimescaleDB insists on using SQL as a query language. While SQL is more powerful than PromQL, this power is rarely required during typical usages of a TSDB. Real-world queries usually [look clearer and simpler when written in PromQL than in SQL](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085).
|
||||
* VictoriaMetrics requires [up to 70x less storage space compared to TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) for storing the same amount of time series data. The gap in storage space usage can be decreased from 70x to 3x if [compression in TimescaleDB is properly configured](https://docs.timescale.com/use-timescale/latest/compression/) (it isn't an easy task in general :)).
|
||||
@@ -244,7 +251,8 @@ See [How to migrate from InfluxDB to VictoriaMetrics](https://docs.victoriametri
|
||||
* VictoriaMetrics accepts data in multiple popular data ingestion protocols – InfluxDB, OpenTSDB, Graphite, CSV – while TimescaleDB supports only SQL inserts.
|
||||
* VictoriaMetrics can be queried via [Graphite's API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#graphite-api-usage).
|
||||
|
||||
## Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [Thanos](https://github.com/thanos-io/thanos) or [Cortex](https://github.com/cortexproject/cortex)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## Does VictoriaMetrics use Prometheus technologies like other clustered TSDBs built on top of Prometheus such as [Thanos](https://github.com/thanos-io/thanos) or [Cortex](https://github.com/cortexproject/cortex)? {#does-victoriametrics-use-prometheus-technologies-like-other-clustered-tsdbs-built-on-top-of-prometheus-such-as-thanos-or-cortex-}
|
||||
|
||||
No. VictoriaMetrics core is written in Go from scratch by [fasthttp](https://github.com/valyala/fasthttp)'s [author](https://github.com/valyala).
|
||||
The architecture is [optimized for storing and querying large amounts of time series data with high cardinality](https://medium.com/devopslinks/victoriametrics-creating-the-best-remote-storage-for-prometheus-5d92d66787ac). VictoriaMetrics storage uses [certain ideas from ClickHouse](https://medium.com/@valyala/how-victoriametrics-makes-instant-snapshots-for-multi-terabyte-time-series-data-e1f3fb0e0282). Special thanks to [Alexey Milovidov](https://github.com/alexey-milovidov).
|
||||
@@ -260,7 +268,8 @@ We provide commercial support for both versions. [Contact us](https://victoriame
|
||||
|
||||
[VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_faq) – the most cost-efficient hosted monitoring platform, operated by VictoriaMetrics core team.
|
||||
|
||||
## Why doesn't VictoriaMetrics support the [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)?
|
||||
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
|
||||
## Why doesn't VictoriaMetrics support the [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)? {#why-doesnrsquot-victoriametrics-support-the-prometheus-remote-read-api-}
|
||||
|
||||
The remote read API requires transferring all the raw data for all the requested metrics over the given time range. For instance,
|
||||
if a query covers 1000 metrics with 10K values each, then the remote read API has to return `1000*10K`=10M metric values to Prometheus.
|
||||
|
||||
@@ -58,9 +58,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
|
||||
from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/victoria-metrics:v1.140.0
|
||||
docker pull victoriametrics/victoria-metrics:v1.142.0
|
||||
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
|
||||
victoriametrics/victoria-metrics:v1.140.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
|
||||
victoriametrics/victoria-metrics:v1.142.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
|
||||
```
|
||||
|
||||
_For Enterprise images see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
|
||||
|
||||
@@ -1681,7 +1681,41 @@ Additionally, alerting can be set up with the following tools:
|
||||
|
||||
## Security
|
||||
|
||||
General security recommendations:
|
||||
### Supported Versions
|
||||
|
||||
The following versions of VictoriaMetrics receive regular security fixes:
|
||||
|
||||
| Version | Supported |
|
||||
|--------------------------------------------------------------------------------|--------------------|
|
||||
| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | ✅ |
|
||||
| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | ✅ |
|
||||
| other releases | ❌ |
|
||||
|
||||
### Software Bill of Materials (SBOM)
|
||||
|
||||
Every VictoriaMetrics container{{% available_from "v1.137.0" %}} image published to
|
||||
[Docker Hub](https://hub.docker.com/u/victoriametrics) and [Quay.io](https://quay.io/organization/victoriametrics) include an [SPDX](https://spdx.dev/) SBOM attestation generated automatically by BuildKit during `docker buildx build`.
|
||||
|
||||
To inspect the SBOM for an image:
|
||||
|
||||
```sh
|
||||
docker buildx imagetools inspect \
|
||||
docker.io/victoriametrics/victoria-metrics:latest \
|
||||
--format "{{ json .SBOM }}"
|
||||
```
|
||||
|
||||
To scan an image using its SBOM attestation with [Trivy](https://github.com/aquasecurity/trivy):
|
||||
|
||||
```sh
|
||||
trivy image --sbom-sources oci \
|
||||
docker.io/victoriametrics/victoria-metrics:latest
|
||||
```
|
||||
|
||||
### Reporting a Vulnerability
|
||||
|
||||
Please report any security issues to <security@victoriametrics.com>
|
||||
|
||||
### General security recommendations:
|
||||
|
||||
* All the VictoriaMetrics components must run in protected private networks without direct access from untrusted networks such as Internet.
|
||||
The exception is [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) and [vmgateway](https://docs.victoriametrics.com/victoriametrics/vmgateway/),
|
||||
@@ -2323,9 +2357,12 @@ Feel free asking any questions regarding VictoriaMetrics:
|
||||
* [X (Twitter)](https://x.com/VictoriaMetrics/)
|
||||
* [Linkedin](https://www.linkedin.com/company/victoriametrics/)
|
||||
* [Reddit](https://www.reddit.com/r/VictoriaMetrics/)
|
||||
* [Telegram-en](https://t.me/VictoriaMetrics_en)
|
||||
* [Telegram-ru](https://t.me/VictoriaMetrics_ru1)
|
||||
* [Mastodon](https://mastodon.social/@victoriametrics/)
|
||||
* Telegram:
|
||||
* [VictoriaMetrics_en](https://t.me/VictoriaMetrics_en)
|
||||
* [VictoriaMetrics_ru](https://t.me/VictoriaMetrics_ru1)
|
||||
* [VictoriaLogs_ru](https://t.me/victorialogs_ru)
|
||||
* [VictoriaTraces_en](https://t.me/VictoriaTraces)
|
||||
|
||||
If you like VictoriaMetrics and want to contribute, then please [read these docs](https://docs.victoriametrics.com/victoriametrics/contributing/).
|
||||
|
||||
|
||||
@@ -13,250 +13,47 @@ aliases:
|
||||
- /release-guide/
|
||||
---
|
||||
|
||||
## PreRequisites
|
||||
> To maintainers:
|
||||
>
|
||||
> The detailed guidance document has been moved to [VictoriaMetrics/release](https://github.com/VictoriaMetrics/release/blob/main/README.md).
|
||||
> The new document contains up-to-date release process guidance.
|
||||
> Please refer to it instead.
|
||||
>
|
||||
> An archived version of this document is available at: [Release-Guide.md](https://github.com/VictoriaMetrics/release/blob/main/legacy_docs/Release-Guide.md).
|
||||
|
||||
1. Make sure you have these tools installed `golang`, `docker`, `make`, `git`, `gcc`, `ca-certificates`, `curl`, `zip`:
|
||||
1. Make sure you have `enterprise` remote configured
|
||||
|
||||
```sh
|
||||
git remote add enterprise <url>
|
||||
```
|
||||
## Release Process Overview
|
||||
|
||||
1. Replace `origin` remote with `opensource` to avoid accidentally pushing enterprise stuff to public repository.
|
||||
VictoriaMetrics releases follow a two-step process: a release candidate phase and a final release.
|
||||
Both the latest and LTS releases are typically done on a bi-weekly cadence.
|
||||
|
||||
```sh
|
||||
git remote remove origin
|
||||
git remote add opensource git@github.com:VictoriaMetrics/VictoriaMetrics.git
|
||||
```
|
||||
### Step 1 — Release Candidate (usually Friday)
|
||||
|
||||
1. Make sure you have singing key configured
|
||||
1. Make sure you have github token with at least `read:org, repo, write:packages` permissions exported under `GITHUB_TOKEN` env variable.
|
||||
You can create a token in your [profile settings](https://github.com/settings/tokens)
|
||||
1. Make sure you're [authorized](https://hub.docker.com/orgs/victoriametrics/settings/enforce-sign-in/windows) for pushing docker images to [docker.io](https://hub.docker.com/u/victoriametrics) and [quay.io](https://quay.io/organization/victoriametrics).
|
||||
A release candidate (RC) is prepared and published to VM sandbox for testing.
|
||||
|
||||
### For MacOS users
|
||||
Key steps:
|
||||
|
||||
Make sure you have GNU version of utilities `zip`, `tar`, `sha256sum`. To install them run the following commands:
|
||||
* Verify all branches are in sync.
|
||||
* Ensure relevant bug fixes are backported where needed (e.g., LTS versions).
|
||||
* Run tests and basic validation.
|
||||
* Check for known vulnerabilities (dependencies and base images).
|
||||
* Build release binaries and Docker images.
|
||||
* Publish **release candidate images** (with `-rc` suffix).
|
||||
* Create a **draft GitHub release** (not published yet).
|
||||
* Deploy the release candidate to a sandbox/testing environment.
|
||||
|
||||
```sh
|
||||
brew install coreutils
|
||||
brew install gnu-tar
|
||||
export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
|
||||
```
|
||||
The goal of this phase is to validate the release in a real environment before making it official.
|
||||
|
||||
Docker may need additional configuration changes:
|
||||
### Step 2 — Final Release (usually Monday)
|
||||
|
||||
```sh
|
||||
docker buildx create --use --name=qemu
|
||||
docker buildx inspect --bootstrap
|
||||
```
|
||||
If the release candidate performs well in testing, it is promoted to a final release.
|
||||
|
||||
By default, docker on MacOS has limited amount of resources (CPU, mem) to use.
|
||||
Bumping the limits may significantly improve build speed.
|
||||
Key steps:
|
||||
|
||||
## Release
|
||||
* Review stability and performance of the RC in the sandbox.
|
||||
* Publish final Docker images (without `-rc` suffix) and update `latest` tag.
|
||||
* Perform a quick smoke test on final images.
|
||||
* Publish the GitHub release.
|
||||
* Close issues included in the release.
|
||||
* Update version references in documentation and related projects (operator, helm-charts, ansible-playbook).
|
||||
|
||||
The release process is divided into two steps.
|
||||
|
||||
### Step 1
|
||||
|
||||
In the first step, which usually takes place on Friday, the binaries are compiled,
|
||||
release candidate images are built and pushed, a draft GitHub release is created,
|
||||
and the candidate is deployed to the sandbox environment.
|
||||
|
||||
1. Lock merges for 24h with `/mutex merge` in Slack.
|
||||
1. Make sure you get all changes fetched.
|
||||
|
||||
```sh
|
||||
git fetch opensource
|
||||
git fetch enterprise
|
||||
```
|
||||
|
||||
1. Make sure all the changes are synced between `master`, `cluster`, `enterprise-single-node` and `enterprise-cluster` branches.
|
||||
This serves as a double-check. The synchronization should already have been done by the person who merged a PR.
|
||||
You can use the following commands to gather the changes and then compare them using your favorite diff tool.
|
||||
|
||||
```sh
|
||||
git checkout master && git log v1.xx.y..HEAD --oneline > master_changelog.txt
|
||||
git checkout cluster && git log v1.xx.y-cluster..HEAD --oneline > cluster_changelog.txt
|
||||
git checkout enterprise-single-node && git log v1.xx.y-enterprise..HEAD --oneline > enterprise_changelog.txt
|
||||
git checkout enterprise-cluster && git log v1.xx.y-enterprise-cluster..HEAD --oneline > enterprise_cluster_changelog.txt
|
||||
```
|
||||
|
||||
1. Review bugfixes in the changelog to determine if they need to be backported to LTS versions.
|
||||
Cherry-pick bug fixes relevant for [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
|
||||
This serves as a double-check. The initial assessment should already have been done by the person who merged a PR.
|
||||
|
||||
1. Make sure tests pass on branches `master`, `cluster`, `enterprise-single-node` and `enterprise-cluster`.
|
||||
|
||||
```sh
|
||||
make test-full
|
||||
make check-all
|
||||
```
|
||||
|
||||
1. Verify no CVEs in Go code or base images according to the [CVE handling policy](https://docs.victoriametrics.com/victoriametrics/#cve-handling-policy).
|
||||
It’s sufficient to run `govulncheck` on the `master` branch since other branches are checked in CI on regular bases.
|
||||
For image scanning, build and check Alpine base image.
|
||||
|
||||
```sh
|
||||
make govulncheck
|
||||
|
||||
make package-base
|
||||
grype --only-fixed [base-image-tag]
|
||||
```
|
||||
|
||||
1. Re-build `vmui` static files. Static assets needs to be rebuilt separately for oss and enterprise branches (changes should not be cherry-picked between these branches). See [commit example](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/9dde5b8ee3fdc9d4cd495c8118e04ff4ee32e650).
|
||||
|
||||
```sh
|
||||
make vmui-update
|
||||
```
|
||||
|
||||
1. Make sure that the release branches have no security issues.
|
||||
1. Update release versions if needed in [SECURITY.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/SECURITY.md).
|
||||
1. Run `TAG=v1.xx.y make docs-update-version` command to update version help tooltips.
|
||||
1. Cut new version in [CHANGELOG.md](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md) and commit it. See example in this [commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b771152039d23b5ccd637a23ea748bc44a9511a7).
|
||||
1. Create the following release tags:
|
||||
* `git tag -s v1.xx.y` in `master` branch
|
||||
* `git tag -s v1.xx.y-cluster` in `cluster` branch
|
||||
* `git tag -s v1.xx.y-enterprise` in `enterprise-single-node` branch
|
||||
* `git tag -s v1.xx.y-enterprise-cluster` in `enterprise-cluster` branch
|
||||
1. Run `TAG=v1.xx.y EXTRA_DOCKER_TAG_SUFFIX=-rcY make publish-release`. This command performs the following tasks:
|
||||
|
||||
* a) Build and package binaries in `*.tar.gz` release archives with the corresponding `_checksums.txt` files inside `bin` directory.
|
||||
This step can be run manually with the command `make release` from the needed git tag.
|
||||
* b) Build and publish [multi-platform Docker images](https://docs.docker.com/build/buildx/multiplatform-images/)
|
||||
for the given `TAG`, `TAG-cluster`, `TAG-enterprise` and `TAG-enterprise-cluster`.
|
||||
The resulting docker images will have special release candidate suffix for the given `EXTRA_DOCKER_TAG_SUFFIX`.
|
||||
The multi-platform Docker image is built for the following platforms:
|
||||
* linux/amd64
|
||||
* linux/arm64
|
||||
* linux/arm
|
||||
* linux/ppc64le
|
||||
* linux/386
|
||||
This step can be run manually with the command `make publish` from the needed git tag.
|
||||
* c) [SPDX](https://spdx.dev/) SBOM attestations are
|
||||
generated automatically by BuildKit during
|
||||
`docker buildx build` (`--sbom=true`). SBOMs can
|
||||
be inspected with
|
||||
`docker buildx imagetools inspect <image> --format "{{ json .SBOM }}"`
|
||||
or consumed by vulnerability scanners such as
|
||||
[Trivy](https://github.com/aquasecurity/trivy) via
|
||||
`trivy image --sbom-sources oci <image-ref>`.
|
||||
|
||||
1. Run `TAG=v1.xx.y make github-create-release github-upload-assets`. This command performs the following tasks:
|
||||
|
||||
* a) Create draft GitHub release with the name `TAG`. This step can be run manually
|
||||
with the command `TAG=v1.xx.y make github-create-release`.
|
||||
The release id is stored at `/tmp/vm-github-release` file.
|
||||
* b) Upload all the binaries and checksums created at step `11a` to that release.
|
||||
This step can be run manually with the command `make github-upload-assets`.
|
||||
It is expected that the needed release id is stored at `/tmp/vm-github-release` file,
|
||||
which must be created at the step `a`.
|
||||
If the upload process is interrupted by any reason, then the following recovery steps must be performed:
|
||||
* To delete the created draft release by running the command `make github-delete-release`.
|
||||
This command expects that the id of the release to delete is located at `/tmp/vm-github-release`
|
||||
file created at the step `a`.
|
||||
* To run the command `TAG=v1.xx.y make github-create-release github-upload-assets`, so new release is created
|
||||
and all the needed assets are re-uploaded to it.
|
||||
|
||||
1. Run `TAG=v1.xx.y make github-verify-release` to verify that draft release with the name `TAG` has been created
|
||||
and this release contains all the needed binaries and checksums.
|
||||
1. Update the release description with the content of [CHANGELOG](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md) for this release. **Use "Save Draft" button, do not publish the release yet!**.
|
||||
1. Follow the instructions in [LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/enterprise-single-node/Release-Guide.md#lts-release).
|
||||
1. Deploy created images to sandbox environment. Make sure it is deployed and don't introduce any regressions. Follow instructions at [Testing Releases](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/enterprise-single-node/Release-Guide.md#testing-releases).
|
||||
|
||||
### Step 2
|
||||
|
||||
In the second step, typically on Monday, the release candidate's performance in the sandbox is reviewed.
|
||||
If no issues are found, the release candidate is promoted and officially published and announced to the community.
|
||||
Version numbers are updated in the codebase, documentation, and related projects.
|
||||
Issues included in the release are closed, with the comment.
|
||||
|
||||
1. Review the performance of the release candidate in the sandbox environment.
|
||||
If any issues are found, they must be addressed, and the release process restarted from [Step 1](#step-1) with an incremented release candidate version.
|
||||
1. Run `TAG=v1.xx.y EXTRA_DOCKER_TAG_SUFFIX=-rc1 make publish-final-images`. This command publishes the final release images from release candidate image for given `EXTRA_DOCKER_TAG_SUFFIX` and updates `latest` Docker image tag for the given `TAG`. SBOM attestations are preserved from the RC images by `imagetools create`.
|
||||
This command must be run only for the latest officially published release. It must be skipped when publishing other releases such as
|
||||
[LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) or some test releases.
|
||||
1. Deploy the final images to the sandbox environment and perform a quick smoke test to verify basic functionality works.
|
||||
1. Push the tags `v1.xx.y` and `v1.xx.y-cluster` created at previous steps to public GitHub repository at [https://github.com/VictoriaMetrics/VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics):
|
||||
|
||||
```shell
|
||||
git push opensource v1.xx.y
|
||||
git push opensource v1.xx.y-cluster
|
||||
```
|
||||
|
||||
Push the tags `v1.xx.y`, `v1.xx.y-cluster`, `v1.xx.y-enterprise` and `v1.xx.y-enterprise-cluster` to the corresponding
|
||||
branches in private repository:
|
||||
|
||||
```shell
|
||||
git push enterprise v1.xx.y
|
||||
git push enterprise v1.xx.y-cluster
|
||||
git push enterprise v1.xx.y-enterprise
|
||||
git push enterprise v1.xx.y-enterprise-cluster
|
||||
```
|
||||
|
||||
**Important note:** do not push enterprise tags to public GitHub repository - they must be pushed only to private repository.
|
||||
|
||||
1. Publish release by pressing "Publish release" green button on GitHub [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
|
||||
1. Update GitHub issues related to the new release. Usually, such issues have a label [waiting for release](https://github.com/VictoriaMetrics/VictoriaMetrics/issues?q=is%3Aopen+is%3Aissue+label%3A%22waiting+for+release%22). Close such issues by mentioning which release they were included into, and remove the label. See an example of a [closed issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6637#issuecomment-2390729511).
|
||||
1. Bump VictoriaMetrics version at `deployment/docker/*.yml`. For example:
|
||||
|
||||
```shell
|
||||
for f in $(grep "v1\.116\.0" -R deployment/docker/ -l); do sed -i 's/v1.116.0/v1.117.0/g' $f; done
|
||||
```
|
||||
|
||||
1. Bump VictoriaMetrics version mentioned in [docs](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7388).
|
||||
1. Run `TAG=v1.xx.y make docs-update-flags` command to update command-line flags in the documentation. [Commit example](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/4d42b291e55ac9211130efbd5a56aa819998516d).
|
||||
1. Follow the instructions in [release follow-up](https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/blob/enterprise-single-node/Release-Guide.md).
|
||||
|
||||
#### Operator
|
||||
|
||||
The operator repository [https://github.com/VictoriaMetrics/operator/](https://github.com/VictoriaMetrics/operator/)
|
||||
|
||||
1. Bump the VictoriaMetrics version in [file `internal/config/config.go`](https://github.com/VictoriaMetrics/operator/blob/master/internal/config/config.go) with new release version for:
|
||||
* `VM_METRICS_VERSION` key in `defaultEnvs` map.
|
||||
* `BaseOperatorConf.MetricsVersion` default value.
|
||||
1. Run `make docs`.
|
||||
1. Add the dependency to the new release to the tip section in `docs/CHANGELOG.md` ([example](https://github.com/VictoriaMetrics/operator/pull/1355/commits/1d7f4439c359b371b05a06e93f615dbcfb266cf5)).
|
||||
1. Commit and send a PR for review.
|
||||
|
||||
#### Helm Charts
|
||||
|
||||
The helm chart repository [https://github.com/VictoriaMetrics/helm-charts/](https://github.com/VictoriaMetrics/helm-charts/)
|
||||
|
||||
Bump the version of images
|
||||
|
||||
> Note that helm charts versioning uses its own versioning scheme. The version of the charts not tied to the version of VictoriaMetrics components.
|
||||
|
||||
Bump `appVersion` field in `Chart.yaml` with new release version.
|
||||
Add new line to "Next release" section in `CHANGELOG.md` about version update (the line must always start with "`-`"). Do **NOT** change headers in `CHANGELOG.md`.
|
||||
Bump `version` field in `Chart.yaml` with incremental semver version (based on the `CHANGELOG.md` analysis).
|
||||
|
||||
Do these updates to the following charts:
|
||||
|
||||
1. Update `vmagent` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-agent/Chart.yaml)
|
||||
1. Update `vmalert` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-alert/Chart.yaml)
|
||||
1. Update `vmauth` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-auth/Chart.yaml)
|
||||
1. Update `cluster` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-cluster/Chart.yaml)
|
||||
1. Update `distributed` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-distributed/Chart.yaml)
|
||||
1. Update `gateway` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-gateway/Chart.yaml)
|
||||
1. Update `k8s-stack` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-k8s-stack/Chart.yaml)
|
||||
1. Update `single` chart `version` and `appVersion` in [`Chart.yaml`](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-single/Chart.yaml)
|
||||
|
||||
See commit example of a [helm chart](https://github.com/VictoriaMetrics/helm-charts/commit/0ec3ab81795cb098d4741451b66886cc6d9be36c).
|
||||
|
||||
Once updated, run the following commands:
|
||||
|
||||
1. Commit and push changes to `master`.
|
||||
1. Run "Release" action on Github:
|
||||
|
||||
1. Merge new PRs *"Automatic update CHANGELOGs and READMEs"* and *"Synchronize docs"* after pipelines are complete.
|
||||
|
||||
#### Ansible Roles
|
||||
|
||||
> Note that ansible playbooks versioning uses its own versioning scheme. The version of the playbooks is not tied to the version of VictoriaMetrics components.
|
||||
|
||||
1. Update the version of VictoriaMetrics components at [https://github.com/VictoriaMetrics/ansible-playbooks](https://github.com/VictoriaMetrics/ansible-playbooks).
|
||||
1. Commit changes.
|
||||
1. Create a new tag with `git tag -sm <TAG> <TAG>`.
|
||||
1. Push the changes with the new tag. This automatically publishes the new versions to galaxy.ansible.com.
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 81 KiB |
@@ -26,10 +26,24 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly discover addresses of storage nodes with [enterprise automatic-vmstorage-discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery). Bug was introduced at [v1.141.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11410).
|
||||
|
||||
* FEATURE: all VictoriaMetrics components: suppress TCP health check errors when `-tls` flag is set. See [#10538](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10538).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-rule.stripFilePath` to support stripping rule file paths in logs and all API responses, including /metrics. See [#5625](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5625).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `formatTime` template function for formatting a Unix timestamp using the provided layout. For example, `{{ now | formatTime "2006-01-02T15:04:05Z07:00" }}` returns the current time in RFC3339 format. See issue [#10624](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10624). Thanks to @andriibeee for the contribution.
|
||||
|
||||
## [v1.142.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.142.0)
|
||||
|
||||
Released at 2026-04-28
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly add [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) `Unit` suffix to the metric name and metadata `Help` when `-opentelemetry.usePrometheusNaming` flag is enabled. Previously, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
## [v1.141.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.141.0)
|
||||
|
||||
Released at 2026-04-24
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.23.3 to 3.23.4. See [Alpine 3.23.4 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.10-3.21.7-3.22.4-3.23.4-released.html).
|
||||
|
||||
* FEATURE: all VictoriaMetrics components: add support for reading cpu/memory limits configured via [systemd slices](https://www.freedesktop.org/software/systemd/man/latest/systemd.slice.html). Previously, only limits set directly on the process's own cgroup were detected. See [#10635](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10635). Thanks to @andriibeee for the contribution.
|
||||
@@ -64,6 +78,8 @@ Released at 2026-04-10
|
||||
|
||||
**Update Note 2:** [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): Due to an ordering bug in binary operations, some queries may produce incorrect results. For example, `10 - (3 + 3 + 4)` is evaluated as `10 - 3 + 3 + 4`. The issue was introduced in versions v1.140.0, v1.136.4, v1.122.19, and is addressed in upcoming releases. It is strongly recommended to avoid these versions entirely. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
|
||||
|
||||
**Update Note 3:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.26.1 to Go1.26.2. See [the list of issues addressed in Go1.26.2](https://github.com/golang/go/issues?q=milestone%3AGo1.26.2%20label%3ACherryPickApproved).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add per-URL `-remoteWrite.disableMetadata` flag to disable metadata sending for specific remote storage URLs. See [#10711](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10711). Thanks to @evkuzin for the contribution.
|
||||
@@ -94,6 +110,8 @@ Released at 2026-04-10
|
||||
|
||||
Released at 2026-03-27
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show `seriesCountByMetricName` table when a label is in focus in the [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer). See [#10630](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10630). Thanks to @Roshan1299 for the contribution.
|
||||
* FEATURE: [dashboards/unused-metrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/unused-metrics.json): add a new dashboard for exploring stored metrics based on [Caridnality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) and [ingested metrics usage API](https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage). The dashboard requires [Infinity Grafana plugin](https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/) to be installed. See [#10617](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10617) for details.
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `search` parameter and pagination support in `/api/v1/rules` API. See [#10046](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10046).
|
||||
@@ -116,6 +134,8 @@ Released at 2026-03-27
|
||||
|
||||
Released at 2026-03-13
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.26.0 to Go1.26.1. See [the list of issues addressed in Go1.26.1](https://github.com/golang/go/issues?q=milestone%3AGo1.26.1%20label%3ACherryPickApproved).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `headers` field to `oauth2` scrape config for passing custom HTTP headers to `token_url`. Some services require different headers for the token endpoint and the scrape targets. See [#8939](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8939).
|
||||
@@ -143,6 +163,8 @@ Released at 2026-02-27
|
||||
**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): default value of the flag `-promscrape.dropOriginalLabels` changed from `true` to `false`.
|
||||
It enables back `Discovered targets` debug UI by default.
|
||||
|
||||
**Update Note 2:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup/): can now copy backups between different storage backends, such as from s3 to local disk or gcs to s3. See [#10401](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10401). Thanks to @BenNF for the contribution.
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add JWT token authentication support with signature verification based on provided `public_keys`. Read more about configuration in [JWT Token auth proxy](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) documentation. See [#10445](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10445).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): support dynamic rewriting of upstream URLs and request headers using placeholders populated from JWT `vm_access` claim fields. This allows routing requests to the correct tenant backend without maintaining a separate user config entry per tenant. Read more in [JWT claim-based request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) documentation. See [#10492](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10492).
|
||||
@@ -157,7 +179,7 @@ It enables back `Discovered targets` debug UI by default.
|
||||
* FEATURE: [dashboards/alert-statistics](https://grafana.com/grafana/dashboards/24553): add `job` and `instance` filters to the `VictoriaMetrics - Alert statistics` dashboard. This allows users running multiple independent [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) instances to filter and analyze alerts statistics per specific instance, making it easier to identify issues in a particular vmalert deployment. See [#10549](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10549).
|
||||
* FEATURE: [dashboards/alert-statistics](https://grafana.com/grafana/dashboards/24553): add a link to a specific alerting rule on the table of firing alerts. See [#10508](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10508). Thanks to @sias32 for the contribution.
|
||||
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): use `$externalURL` instead of `localhost` in the alerting rules. This should improve usability of the rules if `$externalURL` is correctly configured, without need to update rules annotations. See [#10508](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10508). Thanks to @sias32 for the contribution.
|
||||
* FEATURE: all VictoriaMetrics components: publish [SPDX](https://spdx.dev/) SBOM attestations for container images on `docker.io` and `quay.io`. See [SECURITY.md](https://docs.victoriametrics.com/victoriametrics/security/) and [#10474](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10474). Thanks to @smuda for the contribution.
|
||||
* FEATURE: all VictoriaMetrics components: publish [SPDX](https://spdx.dev/) SBOM attestations for container images on `docker.io` and `quay.io`. See [security](https://docs.victoriametrics.com/victoriametrics/#security) and [#10474](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10474). Thanks to @smuda for the contribution.
|
||||
|
||||
* BUGFIX: all VictoriaMetrics components: return gzip-compressed response instead of zstd-compressed response to the client if `Accept-Encoding` request header contains both `gzip` and `zstd`. This is needed because some clients and proxies improperly handle zstd-compressed responses. See [#10535](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10535).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check expired client certificate during mTLS requests. See [#10393](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10393).
|
||||
@@ -167,10 +189,58 @@ It enables back `Discovered targets` debug UI by default.
|
||||
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
|
||||
|
||||
## [v1.136.8](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.8)
|
||||
|
||||
Released at 2026-04-30
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
|
||||
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly discover addresses of storage nodes with [enterprise automatic-vmstorage-discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery). Bug was introduced at [v1.141.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11410).
|
||||
|
||||
## [v1.136.7](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.7)
|
||||
|
||||
Released at 2026-04-28
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly add [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) `Unit` suffix to the metric name and metadata `Help` when `-opentelemetry.usePrometheusNaming` flag is enabled. Previously, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
## [v1.136.6](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.6)
|
||||
|
||||
Released at 2026-04-24
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.4 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.10-3.21.7-3.22.4-3.23.4-released.html).
|
||||
|
||||
* BUGFIX: all VictoriaMetrics components: properly parse IPv6 source address when accepting connections with proxy protocol v2 enabled. See [#10839](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10839). Thanks to @andriibeee for the contribution.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly attach [tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) from `__tenant_id__` label to the scraped metadata. See [#10828](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10828).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): keep [tenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) ingested into vmagent via [prometheus remotewrite](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) endpoint. See [#10828](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10828).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): `-maxScrapeSize` is now correctly applied when reading response bodies, including non-OK scrape error responses. See [#10804](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10804).
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `ec2_sd_configs` returning 401 `AuthFailure` from AWS when credentials are obtained via IRSA, instance role or `AWS_CONTAINER_CREDENTIALS_*` env vars. The regression was introduced in [v1.140.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.140.0). See [#10815](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10815). Thanks to @andriibeee for the contribution.
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fix leak of backend TCP connections, file descriptors and goroutines when the client cancels the request after the backend response has been received. See [#10833](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10833). Thanks to @andriibeee for the contribution.
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fix a rare panic during config reload when a backend is marked as broken. See [#10806](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10806).
|
||||
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): stop logging warnings about failed handshakes when the `clusternative` port receives TCP healthchecks from load balancers. See [#10786](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10786). Thanks to @andriibeee for the contribution.
|
||||
* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix an issue where vmrestore could hang indefinitely when interrupted during backup download. See [#10794](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10794).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly execute graceful shutdown for vmsingle if `-maxIngestionRate` is configured. See [#10795](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10795).
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix time display on Alerting Rules page to use selected timezone. See [#10827](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10827).
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): use contrasting colors when displaying time series to improve visibility on light and dark themes. See [#10869](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10869).
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): delete labels from rule results if they are specified with an empty string value in rule or group labels. See [#10766](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10766).
|
||||
|
||||
## [v1.136.5](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.5)
|
||||
|
||||
Released at 2026-04-23
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
@@ -181,6 +251,8 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
|
||||
|
||||
**Update Note 1:** [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): Due to an ordering bug in binary operations, some queries may produce incorrect results. For example, `10 - (3 + 3 + 4)` is evaluated as `10 - 3 + 3 + 4`. The issue was introduced in versions v1.140.0, v1.136.4, v1.122.19, and is addressed in upcoming releases. It is strongly recommended to avoid these versions entirely. See [#10856](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10856).
|
||||
|
||||
**Update Note 2:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
Released at 2026-04-10
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
@@ -200,6 +272,8 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
|
||||
|
||||
Released at 2026-03-27
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
@@ -214,6 +288,8 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
|
||||
|
||||
Released at 2026-03-13
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
@@ -235,6 +311,8 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
|
||||
|
||||
Released at 2026-03-02
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
|
||||
@@ -251,6 +329,8 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
|
||||
|
||||
Released at 2026-02-13
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.25.6 to Go1.26.0. See [Go 1.26 release notes](https://go.dev/doc/go1.26).
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.9-3.21.6-3.22.3-3.23.3-released.html).
|
||||
|
||||
@@ -276,6 +356,8 @@ Released at 2026-01-30
|
||||
|
||||
**Update Note 1:** `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): has a bug affecting the `/select/multitenant/*` APIs. Due to an issue in the tenant search logic [#10422](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10422), these endpoints may return incorrect results. The bug has been fixed and the correction will be included in `v1.137.0`.
|
||||
|
||||
**Update Note 2:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): improved scrape size display. Sizes below 1024 bytes are now shown in `B`, and larger sizes are shown as whole `KiB` (rounded up). This prevents confusion where values like 123.456 KiB were interpreted as 123456 KiB, while the actual size was only 123 KiB. See [#10307](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10307).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow buffering request bodies before proxying them to backends. This reduces load on backends when processing requests from slow clients such as IoT devices connected to `vmauth` via slow networks. See [#10309](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10309) and [request body buffering docs](https://docs.victoriametrics.com/victoriametrics/vmauth/#request-body-buffering).
|
||||
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): allow completely disabling scheduled backups by using `-disableScheduledBackups` command-line flag. This is useful to run `vmbackupmanager` only for on-demand backups and restores triggered via API. See [#10364](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10364).
|
||||
@@ -303,6 +385,8 @@ Released at 2026-01-30
|
||||
|
||||
Released at 2026-01-16
|
||||
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.25.5 to Go1.25.6. See [the list of issues addressed in Go1.25.6](https://github.com/golang/go/issues?q=milestone%3AGo1.25.6%20label%3ACherryPickApproved).
|
||||
|
||||
* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): refine `VictoriaMetrics - single` dashboard and aligned it with the [VictoriaMetrics - cluster](https://grafana.com/grafana/dashboards/11176) dashboard. For the full list of changes see [#10132-comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10187#issuecomment-3696769466) and [#10260](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10260).
|
||||
@@ -339,6 +423,8 @@ Released at 2026-01-02
|
||||
|
||||
**Update Note 5:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `indexdb/tagFiltersToMetricIDs`, `indexdb/metricID` and `indexdb/date_metricID` cache metrics are not reported properly. See [10275](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10275). Addressed in `v1.135.0`.
|
||||
|
||||
**Update Note 6:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.22.2 to 3.23.2. See [Alpine 3.23.2 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.2-released.html).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add support for global `sampleLimit` setting. This allows users to efficiently limit the number of samples accepted per scrape target. This also ensures target-level `sample_limit` can correctly override the global setting. See [#10145](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10145). Thanks to @kobylyanskiy for the contribution.
|
||||
@@ -403,6 +489,25 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
|
||||
|
||||
See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
|
||||
|
||||
## [v1.122.21](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.21)
|
||||
|
||||
Released at 2026-04-24
|
||||
|
||||
**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
|
||||
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
|
||||
The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.25.9 to Go1.26.2. See [the list of issues addressed in Go1.26.2](https://github.com/golang/go/issues?q=milestone%3AGo1.26.2%20label%3ACherryPickApproved).
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.4 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.10-3.21.7-3.22.4-3.23.4-released.html).
|
||||
|
||||
* BUGFIX: all VictoriaMetrics components: properly parse IPv6 source address when accepting connections with proxy protocol v2 enabled. See [#10839](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10839). Thanks to @andriibeee for the contribution.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): `-maxScrapeSize` is now correctly applied when reading response bodies, including non-OK scrape error responses. See [#10804](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10804).
|
||||
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly allocate vminsert buffer size per vmstorage node based on available CPU, memory and storage node count to reduce OOM risk. See [#10725](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10725).
|
||||
* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix an issue where vmrestore could hang indefinitely when interrupted during backup download. See [#10794](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10794).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly execute graceful shutdown for vmsingle if `-maxIngestionRate` is configured. See [#10795](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10795).
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): delete labels from rule results if they are specified with an empty string value in rule or group labels. See [#10766](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10766).
|
||||
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): use contrasting colors when displaying time series to improve visibility on light and dark themes. See [#10869](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10869).
|
||||
|
||||
## [v1.122.20](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.20)
|
||||
|
||||
Released at 2026-04-23
|
||||
@@ -1566,4 +1671,4 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
|
||||
|
||||
## Previous releases
|
||||
|
||||
See [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
|
||||
See [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases).
|
||||
|
||||
@@ -16,69 +16,21 @@ aliases:
|
||||
- /changelog/changelog_2025/index.html
|
||||
- /changelog/changelog_2025/
|
||||
---
|
||||
The following `tip` changes can be tested by building VictoriaMetrics components from the latest commits according to the following docs:
|
||||
|
||||
* [How to build single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-build-from-sources)
|
||||
* [How to build cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#building-from-sources)
|
||||
* [How to build vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/#how-to-build-from-sources)
|
||||
* [How to build vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/#how-to-build-from-sources)
|
||||
* [How to build vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/#how-to-build-from-sources)
|
||||
* [How to build vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/#how-to-build-from-sources)
|
||||
* [How to build vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/#how-to-build-from-sources)
|
||||
* [How to build vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/#how-to-build)
|
||||
|
||||
Metrics of the latest version of VictoriaMetrics cluster are available for viewing at our
|
||||
[sandbox](https://play-grafana.victoriametrics.com/d/oS7Bi_0Wz_vm/victoriametrics-cluster-vm).
|
||||
The sandbox cluster installation runs under the constant load generated by
|
||||
[prometheus-benchmark](https://github.com/VictoriaMetrics/prometheus-benchmark) and is used for testing latest releases.
|
||||
|
||||
See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/).
|
||||
|
||||
## tip
|
||||
|
||||
* FEATURE: [dashboards/single](https://grafana.com/grafana/dashboards/10229): refine `VictoriaMetrics - single` dashboard and aligned it with the [VictoriaMetrics - cluster](https://grafana.com/grafana/dashboards/11176) dashboard. For the full list of changes see [#10132-comment](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10187#issuecomment-3696769466) and [#10260](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10260).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/): add `vm_persistentqueue_free_disk_space_bytes` metric for vmagent's persistentqueue capacity. See [#10193](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10193).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `vm_rollup_result_cache_requests_total` which tracks the number of requests to the query rollup cache. See [#10117](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10117).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `localStorage` availability checks with error reporting. See [#10085](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10085).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add `VMUI:`-prefixed `localStorage` keys and legacy key migration.
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix configuration reloading for `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` when vmagent is launched with empty files. Previously, if vmagent started with an empty config, subsequent config reloads were ignored. See [#10211](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10211).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Fixed a missing path error for `http://<victoriametrics-addr>:8428/zabbixconnector/api/v1/history`. See PR [10214](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10214)
|
||||
|
||||
## [v1.133.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.133.0)
|
||||
|
||||
Released at 2026-01-02
|
||||
|
||||
**Update Note 1:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Upgrading to per-partition index requires registering all active time series. Expect slow down of data ingestion and queries during upgrade roll-out. This is a one-time operation. Additionally, for users with retention periods shorter than 1 month the disk usage may increase.
|
||||
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.22.2 to 3.23.2. See [Alpine 3.23.2 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.2-released.html).
|
||||
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add support for global `sampleLimit` setting. This allows users to efficiently limit the number of samples accepted per scrape target. This also ensures target-level `sample_limit` can correctly override the global setting. See [#10145](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10145). Thanks to @kobylyanskiy for the contribution.
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): expose `vmauth_user_request_backend_requests_total` and `vmauth_unauthorized_user_request_backend_requests_total` [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/#monitoring), which track the number of requests sent to backends. These counts may exceed `vmauth_user_requests_total` and `vmauth_unauthorized_user_requests_total` when requests are retried across multiple backends. See [#10171](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10171).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): expose `vmauth_user_request_errors_total` and `vmauth_unauthorized_user_request_errors_total` [metrics](https://docs.victoriametrics.com/victoriametrics/vmauth/#monitoring), which track the number of user request errors. See [#10188](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10188).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `-maxQueueDuration` command-line flag for waiting until the incoming request could be executed if `-maxConcurrentRequests` are already exceeded. This should help with graceful handling of a short spike in the number of concurrent requests without leading to a retry storm. See [#10078](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce per-partition index. This should reduce disk space occupied by indexDBs as they get deleted along with the corresponding partitions once those partitions become outside the retention window. Read more about the motivation behind this feature at [#7599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7599) and what to expect at [#8134](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8134).
|
||||
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly add `exported_` prefix to scraped metrics that have the same names as [auto-generated metrics](https://docs.victoriametrics.com/victoriametrics/vmagent/#automatically-generated-metrics). Previously, some auto-generated metrics were not recognized, so scraped metrics with the same names could overwrite them. See [#10197](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10197). Thanks to @fxrlv for the contribution.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix `vmagent_rows_inserted_total{type="newrelic"}` metric to correctly count samples. See [#10191](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10191). Thanks to @fxrlv for the contribution.
|
||||
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fix `vmauth_user_request_backend_errors_total` and `vmauth_unauthorized_user_request_backend_errors_total` to only reflect backend request errors. Previously, these counters could be overcounted with user request error. See [#10177](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10177).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): rotate `dateMetricIDCache` instead of resetting it. This should make the eviction less aggressive. Since the cache does not have fixed max size anymore the `-storage.cacheSizeIndexDBDateMetricID` flag has been removed. See [#10064](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10064) and PR [#10169](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10169).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly add metrics metadata scraped with `promscrape.config` and `selfScrapeInterval`. See [#10175](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10175).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix stats collection for `indexdb/tagFiltersToMetricIDs`, `indexdb/metricID`, and `indexdb/date_metricID` caches. As per PR [#10131](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10131), the stats is collected for most utilized instance only, but if the size of all instances is 0 then the stats won't be collected at all. This may result in max cache size alternating between the actual value and 0. See [#10204](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10204) and [#10230](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10230).
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly proxy requests to [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) when `-vmalert.proxyURL` flag is set. See [#10178](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10178).
|
||||
|
||||
## [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0)
|
||||
|
||||
Released at 2025-12-12
|
||||
|
||||
**Known issue: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): may leak memory when ingesting data via the [OpenTelemetry protocol](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/).
|
||||
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): may leak memory when ingesting data via the [OpenTelemetry protocol](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/).
|
||||
The problem introduced in [293d809](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/293d80910ce14c247e943c63cd19467df5767c3c), and is already fixed in commits [fastjson#18c81211](https://github.com/valyala/fastjson/commit/18c812114b638d460f0fc6d8e2b86b719e171389) and [19009836](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/19009836c704a75a295c11b5d55a171c206646bd).
|
||||
If you rely on OpenTelemetry ingestion, skip this version or [build from master](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-build-from-sources) to avoid the leak.
|
||||
Read [VictoriaLogs#869](https://github.com/VictoriaMetrics/VictoriaLogs/issues/869) for more details.**
|
||||
Read [VictoriaLogs#869](https://github.com/VictoriaMetrics/VictoriaLogs/issues/869) for more details.
|
||||
|
||||
**Known issue: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): may shutdown ungracefully (data, indexes, and caches are not flushed to disk) in Kubernetes deployments if the number of `vminserts` is > 1 and `terminationGracePeriodSeconds` < 60s (30s by default). The issue was introduced in [9487](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9487) and [10136](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10136) and was fixed in [10224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10224). The fix will be included into `v1.133.0`. Even with this fix, ungraceful shutdowns are possible if flushing in-memory contents takes more than 5s. In this case decrease `-storage.vminsertConnsShutdownDuration` and/or increase [terminationGracePeriodSeconds](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination-flow).**
|
||||
**Update Note 2:** `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): may shutdown ungracefully (data, indexes, and caches are not flushed to disk) in Kubernetes deployments if the number of `vminserts` is > 1 and `terminationGracePeriodSeconds` < 60s (30s by default). The issue was introduced in [9487](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9487) and [10136](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10136) and was fixed in [10224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10224). The fix will be included into `v1.133.0`. Even with this fix, ungraceful shutdowns are possible if flushing in-memory contents takes more than 5s. In this case decrease `-storage.vminsertConnsShutdownDuration` and/or increase [terminationGracePeriodSeconds](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination-flow).
|
||||
|
||||
**Known issue:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Certain data and query patterns may cause high CPU utilization due to [10154](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10154). See [10297](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10297).
|
||||
**Update Note 3:** [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Certain data and query patterns may cause high CPU utilization due to [10154](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10154). See [10297](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10297).
|
||||
|
||||
**Update Note 4:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.25.4 to Go1.25.5. See [the list of issues addressed in Go1.25.5](https://github.com/golang/go/issues?q=milestone%3AGo1.25.5%20label%3ACherryPickApproved).
|
||||
|
||||
|
||||
@@ -117,7 +117,7 @@ It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [
|
||||
|
||||
Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
|
||||
and [the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest).
|
||||
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz`.
|
||||
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz`.
|
||||
|
||||
In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
|
||||
from the corresponding releases page and unpack it. Then run the unpacked binary.
|
||||
@@ -135,8 +135,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
|
||||
obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
|
||||
|
||||
```sh
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz
|
||||
tar -xzf victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz
|
||||
tar -xzf victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz
|
||||
./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
|
||||
```
|
||||
|
||||
@@ -151,7 +151,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
|
||||
It is allowed to run VictoriaMetrics and VictoriaLogs Enterprise components in [cases listed here](#valid-cases-for-victoriametrics-enterprise).
|
||||
|
||||
Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
|
||||
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.140.0-enterprise`.
|
||||
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.142.0-enterprise`.
|
||||
|
||||
In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
|
||||
flag as described in the [binary-releases](#binary-releases) section.
|
||||
@@ -161,13 +161,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
|
||||
For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
|
||||
|
||||
```sh
|
||||
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.140.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
|
||||
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.142.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
|
||||
```
|
||||
|
||||
Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
|
||||
|
||||
```sh
|
||||
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.140.0-enterprise -licenseFile=/path/to/vm-license
|
||||
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.142.0-enterprise -licenseFile=/path/to/vm-license
|
||||
```
|
||||
|
||||
Example docker-compose configuration:
|
||||
@@ -177,7 +177,7 @@ version: "3.5"
|
||||
services:
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.140.0
|
||||
image: victoriametrics/victoria-metrics:v1.142.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -209,7 +209,7 @@ is used to provide the license key in plain-text:
|
||||
```yaml
|
||||
server:
|
||||
image:
|
||||
tag: v1.140.0-enterprise
|
||||
tag: v1.142.0-enterprise
|
||||
|
||||
license:
|
||||
key: {BASE64_ENCODED_LICENSE_KEY}
|
||||
@@ -220,7 +220,7 @@ In order to provide the license key via existing secret, the following values fi
|
||||
```yaml
|
||||
server:
|
||||
image:
|
||||
tag: v1.140.0-enterprise
|
||||
tag: v1.142.0-enterprise
|
||||
|
||||
license:
|
||||
secret:
|
||||
@@ -270,7 +270,7 @@ spec:
|
||||
license:
|
||||
key: {BASE64_ENCODED_LICENSE_KEY}
|
||||
image:
|
||||
tag: v1.140.0-enterprise
|
||||
tag: v1.142.0-enterprise
|
||||
```
|
||||
|
||||
In order to provide the license key via an existing secret, the following custom resource is used:
|
||||
@@ -287,7 +287,7 @@ spec:
|
||||
name: vm-license
|
||||
key: license
|
||||
image:
|
||||
tag: v1.140.0-enterprise
|
||||
tag: v1.142.0-enterprise
|
||||
```
|
||||
|
||||
Example secret with license key:
|
||||
@@ -338,7 +338,7 @@ Builds are available for amd64 and arm64 architectures.
|
||||
|
||||
Example archive:
|
||||
|
||||
`victoria-metrics-linux-amd64-v1.140.0-enterprise.tar.gz`
|
||||
`victoria-metrics-linux-amd64-v1.142.0-enterprise.tar.gz`
|
||||
|
||||
Includes:
|
||||
|
||||
@@ -347,7 +347,7 @@ Includes:
|
||||
|
||||
Example Docker image:
|
||||
|
||||
`victoriametrics/victoria-metrics:v1.140.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
|
||||
`victoriametrics/victoria-metrics:v1.142.0-enterprise-fips` – uses the FIPS-compatible binary and based on `scratch` image.
|
||||
|
||||
## Monitoring license expiration
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ Let's assume vmagent is running with command line flags:
|
||||
* `-zabbixconnector.addEmptyTagsValue=exists`
|
||||
* `-zabbixconnector.addDuplicateTagsSeparator=,`
|
||||
|
||||
Let's fetch the ingested data via [data export API](#how-to-export-data-in-json-line-format):
|
||||
Let's fetch the ingested data via [data export API](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-export-data-in-json-line-format):
|
||||
|
||||
```sh
|
||||
curl http://localhost:8428/api/v1/export -d 'match={host="Zabbix server"}'
|
||||
|
||||
@@ -426,7 +426,7 @@ target.
|
||||
### Service Discovery Relabeling Cheatsheet
|
||||
|
||||
**Target-level relabeling** is applied during
|
||||
[service discovery](https://docs.victoriametrics.com/victoriametrics/sd_configs/#prometheus-service-discovery)
|
||||
[service discovery](https://docs.victoriametrics.com/victoriametrics/sd_configs/)
|
||||
and affects the targets (which will be scraped), their labels and all the
|
||||
metrics scraped from them:
|
||||
|
||||
|
||||
@@ -35,8 +35,8 @@ scrape_configs:
|
||||
After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
|
||||
|
||||
```sh
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.140.0.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.142.0.tar.gz
|
||||
```
|
||||
|
||||
Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
|
||||
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
|
||||
|
||||
```yaml
|
||||
# Download and unpack single-node VictoriaMetrics
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/victoria-metrics-linux-amd64-v1.140.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.140.0.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/victoria-metrics-linux-amd64-v1.142.0.tar.gz
|
||||
tar xzf victoria-metrics-linux-amd64-v1.142.0.tar.gz
|
||||
|
||||
# Run single-node VictoriaMetrics with the given scrape.yaml
|
||||
./victoria-metrics-prod -promscrape.config=scrape.yaml
|
||||
|
||||
@@ -1184,10 +1184,10 @@ One of the following `role` types can be configured to discover targets:
|
||||
Available meta labels for `role: endpoints` during [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/):
|
||||
|
||||
* `__meta_kubernetes_namespace`: The namespace of the endpoints object.
|
||||
* `__meta_kubernetes_namespace_annotation_<annotationname>`: Each annotation from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_annotationpresent_<annotationname>`: "true" for each annotation from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_label_<labelname>`: Each label from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_labelpresent_<labelname>`: "true" for each label from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_annotation_<annotationname>`: Each annotation from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_annotationpresent_<annotationname>`: "true" for each annotation from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_label_<labelname>`: Each label from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_labelpresent_<labelname>`: "true" for each label from the namespace of the endpoints object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_endpoints_name`: The names of the endpoints object.
|
||||
* `__meta_kubernetes_endpoints_label_<labelname>`: Each label from the endpoints object.
|
||||
* `__meta_kubernetes_endpoints_labelpresent_<labelname>`: "true" for each label from the endpoints object.
|
||||
@@ -1217,10 +1217,10 @@ One of the following `role` types can be configured to discover targets:
|
||||
Available meta labels for `role: endpointslice` during [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/):
|
||||
|
||||
* `__meta_kubernetes_namespace`: The namespace of the endpointslice object.
|
||||
* `__meta_kubernetes_namespace_annotation_<annotationname>`: Each annotation from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_annotationpresent_<annotationname>`: "true" for each annotation from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_label_<labelname>`: Each label from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_labelpresent_<labelname>`: "true" for each label from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_annotation_<annotationname>`: Each annotation from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_annotationpresent_<annotationname>`: "true" for each annotation from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_label_<labelname>`: Each label from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_labelpresent_<labelname>`: "true" for each label from the namespace of the endpointslice object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_endpointslice_name`: The name of endpointslice object.
|
||||
|
||||
For all targets discovered directly from the endpointslice list (those not additionally inferred from underlying pods), the following labels are attached:
|
||||
@@ -1251,10 +1251,10 @@ One of the following `role` types can be configured to discover targets:
|
||||
Available meta labels for `role: ingress` during [relabeling](https://docs.victoriametrics.com/victoriametrics/relabeling/):
|
||||
|
||||
* `__meta_kubernetes_namespace`: The namespace of the ingress object.
|
||||
* `__meta_kubernetes_namespace_annotation_<annotationname>`: Each annotation from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_annotationpresent_<annotationname>`: "true" for each annotation from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_label_<labelname>`: Each label from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_labelpresent_<labelname>`: "true" for each label from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`. See [attach_metadata](#attach_metadata).
|
||||
* `__meta_kubernetes_namespace_annotation_<annotationname>`: Each annotation from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_annotationpresent_<annotationname>`: "true" for each annotation from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_label_<labelname>`: Each label from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_namespace_labelpresent_<labelname>`: "true" for each label from the namespace of the ingress object when `attach_metadata.namespace` is set to `true`.
|
||||
* `__meta_kubernetes_ingress_name`: The name of the ingress object.
|
||||
* `__meta_kubernetes_ingress_label_<labelname>`: Each label from the ingress object.
|
||||
* `__meta_kubernetes_ingress_labelpresent_<labelname>`: "true" for each label from the ingress object.
|
||||
|
||||
@@ -24,13 +24,12 @@ Stream aggregation has the following features:
|
||||
- It can calculate [aggregates](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#aggregation-outputs) on ingested samples before they're sent to remote destination;
|
||||
- It is applied to all the metric samples received via any [supported data ingestion protocol](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data)
|
||||
and/or scraped from [Prometheus-compatible targets](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter)
|
||||
- It can filter out raw samples matched by aggregation rules, so raw data will never reach the remote destination. See `--streamAggr.keepInput` and `-streamAggr.dropInput` in [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/);
|
||||
- It can filter out raw samples matched by aggregation rules, so raw data will never reach the remote destination. See `-streamAggr.keepInput` and `-streamAggr.dropInput` in [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/);
|
||||
- It allows building [flexible processing pipelines](#routing);
|
||||
|
||||
# Limitations
|
||||
|
||||
- Stream aggregation **ignores timestamps associated with the input [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)**.
|
||||
It expects that the ingested samples have timestamps close to the current time. See [how to ignore old samples](#ignoring-old-samples).
|
||||
- By default, stream aggregation ignores timestamps of the input [samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) and processes samples based on their ingestion time. See [how to ignore old samples](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#ignoring-old-samples).
|
||||
- Aggregation state is held in the process memory and will be lost on process restart.
|
||||
|
||||
# Use cases
|
||||
@@ -340,7 +339,7 @@ Please note, histograms can be aggregated if their `le` labels are configured id
|
||||
have no such requirement.
|
||||
|
||||
Stream aggregation of histogram buckets is very sensitive to sample delays. Histogram is logical group of independent time series (buckets) that are supposed to be updated uniformly.
|
||||
Aggregation can't guarantee that within one aggregation intervals all samples belonging to one histogram were updated. Situations like this can cause [accuracy issues](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580).
|
||||
Aggregation cannot guarantee that all samples belonging to a single histogram are updated within the same aggregation interval. Situations like this can cause [accuracy issues](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4580).
|
||||
The recommended ways for improving accuracy are:
|
||||
- enable [aggregation windows](#aggregation-windows);
|
||||
- increase `interval` ;
|
||||
@@ -394,11 +393,9 @@ before sending them to the configured `-remoteWrite.url`. The deduplication can
|
||||
|
||||
- By specifying `dedup_interval` option individually per each [stream aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config) at `-streamAggr.config`.
|
||||
|
||||
It is possible to drop the given labels before applying the deduplication. See [these docs](#dropping-unneeded-labels).
|
||||
Labels can be dropped before deduplication is applied. See [these docs](#dropping-unneeded-labels).
|
||||
|
||||
The online deduplication uses the same logic as [`-dedup.minScrapeInterval` command-line flag](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#deduplication) at VictoriaMetrics.
|
||||
|
||||
Deduplication is applied before stream aggregation rules and can drop samples before they get matched for aggregation.
|
||||
Stream aggregation deduplication is applied before aggregation rules, so duplicate samples are dropped before aggregation.
|
||||
|
||||
# Relabeling
|
||||
|
||||
@@ -444,6 +441,10 @@ outside the current [aggregation interval](https://docs.victoriametrics.com/vict
|
||||
- To set `ignore_old_samples: true` option at the particular [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
|
||||
This enables ignoring old samples for that particular aggregation config.
|
||||
|
||||
- To enable [aggregation windows](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#aggregation-windows).
|
||||
|
||||
The dropped old samples can be tracked with the `vm_streamaggr_ignored_samples_total{reason="too_old"}` metric.
|
||||
|
||||
## Ignore aggregation intervals on start
|
||||
|
||||
Streaming aggregation results may be incorrect for some time after the restart of [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/)
|
||||
@@ -617,10 +618,9 @@ deduplication results, which are consistent within the margin of error. But for
|
||||
like [histograms](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#histogram), such inaccuracy leads to invalid aggregation results.
|
||||
|
||||
For this case, streaming aggregation and deduplication support mode with aggregation windows for the current and previous state.
|
||||
With this mode, flush doesn't happen immediately but is shifted by a calculated sample lag that improves correctness for delayed data. {{% available_from "v1.112.0" %}}
|
||||
In this mode, flush doesn't happen immediately but is delayed by a calculated sample lag, which should significantly improve the accuracy of calculations when samples arrive with a delay. {{% available_from "v1.112.0" %}}
|
||||
|
||||
Enabling this mode has increased resource usage: memory usage is expected to double as aggregation will store two states
|
||||
instead of one. However, this significantly improves the accuracy of calculations. Aggregation windows can be enabled via
|
||||
This mode doubles the memory used for the aggregation state, since two states(for the current and previous intervals) are stored per series simultaneously. Aggregation windows can be enabled via
|
||||
the following settings:
|
||||
|
||||
- `-streamAggr.enableWindows` at [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)
|
||||
@@ -671,7 +671,7 @@ The following solutions can help reduce memory usage and CPU usage during stream
|
||||
[raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples) are aggregated.
|
||||
- To increase the aggregation interval by specifying a bigger duration for the `interval` option at [streaming aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config).
|
||||
- To generate a lower number of output time series by using less specific [`by` list](#aggregating-by-labels) or more specific [`without` list](#aggregating-by-labels).
|
||||
- To drop unneeded long labels in input samples via [input_relabel_configs](#relabeling).
|
||||
- To drop unneeded labels from input samples via [input_relabel_configs](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#relabeling) or [dropInputLabels](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/#dropping-unneeded-labels).
|
||||
|
||||
## Cluster mode
|
||||
|
||||
|
||||
@@ -74,6 +74,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
|
||||
Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
|
||||
-fs.maxConcurrency int
|
||||
The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph (default fsutil.getDefaultConcurrency())
|
||||
-futureRetention value
|
||||
Data with timestamps bigger than now+futureRetention is automatically deleted. The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention
|
||||
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 2d)
|
||||
-graphite.sanitizeMetricName
|
||||
Sanitize metric names for the ingested Graphite data. See https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting
|
||||
-graphiteListenAddr string
|
||||
|
||||
@@ -365,6 +365,8 @@ Additionally, `vmalert` provides some extra templating functions listed in [temp
|
||||
* `title` - converts the first letters of every input word to uppercase.
|
||||
* `toLower` - converts all the chars in the input string to lowercase.
|
||||
* `toTime` - converts the input unix timestamp to [time.Time](https://pkg.go.dev/time#Time).
|
||||
* `formatTime layout` - formats a Unix timestamp time using the given [time layout](https://pkg.go.dev/time#Layout).
|
||||
For example, `{{ now | formatTime "2006-01-02T15:04:05Z07:00" }}` returns the current time formatted as RFC3339.
|
||||
* `toUpper` - converts all the chars in the input string to uppercase.
|
||||
* `value` - returns the numeric value from the input query result.
|
||||
|
||||
@@ -532,9 +534,7 @@ The multitenant endpoint in vmselect is less efficient than [specifying tenants
|
||||
|
||||
For security considerations, it is recommended restricting access to multitenant endpoints only to trusted sources, since untrusted source may break per-tenant data by writing unwanted samples or get access to data of arbitrary tenants.
|
||||
|
||||
* To specify `tenant` parameter per each alerting and recording group if
|
||||
[enterprise version of vmalert](https://docs.victoriametrics.com/victoriametrics/enterprise/) is used
|
||||
with `-clusterMode` command-line flag. For example:
|
||||
* To specify the `tenant` parameter for each rule group when `-clusterMode` is enabled in the [enterprise version of vmalert](https://docs.victoriametrics.com/victoriametrics/enterprise/). In this mode, `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must contain only the hostname without tenant information, such as `-datasource.url=http://vmselect:8481`, and `vmalert` will automatically append the specified tenant to the URLs for querying and writing:
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
@@ -553,10 +553,6 @@ The results of alerting and recording rules contain `vm_account_id` and `vm_proj
|
||||
if `-clusterMode` is enabled. These labels can be used during [templating](https://docs.victoriametrics.com/victoriametrics/vmalert/#templating),
|
||||
and help to identify to which account or project the triggered alert or produced recording belongs.
|
||||
|
||||
If `-clusterMode` is enabled, then `-datasource.url`, `-remoteRead.url` and `-remoteWrite.url` must
|
||||
contain only the hostname without tenant id. For example: `-datasource.url=http://vmselect:8481`.
|
||||
`vmalert` automatically adds the specified tenant to urls per each recording rule in this case.
|
||||
|
||||
If `-clusterMode` is enabled and the `tenant` in a particular group is missing, then the tenant value
|
||||
is obtained from `-defaultTenant.prometheus` or `-defaultTenant.graphite` depending on the `type` of the group.
|
||||
|
||||
@@ -1144,10 +1140,16 @@ It is possible to specify custom TLS Root CA via `-mtlsCAFile` command-line flag
|
||||
|
||||
See general recommendations regarding [security](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#security).
|
||||
|
||||
vmalert [web UI](#web) exposes configuration details such as list of [Groups](#groups), active alerts,
|
||||
[alerts state](#alerts-state-on-restarts), [notifiers](#notifier-configuration-file). Notifier addresses (sanitized) are attached
|
||||
as labels to metrics `vmalert_alerts_sent_.*` on `http://<vmalert>/metrics` page. Consider limiting user's access
|
||||
to the web UI or `/metrics` page if this information is sensitive.
|
||||
vmalert [web UI](https://docs.victoriametrics.com/victoriametrics/vmalert/#web), logs, and exported metrics contain details such as [group](https://docs.victoriametrics.com/victoriametrics/vmalert/#groups) configurations, active alerts,
|
||||
[alerts state](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerts-state-on-restarts), [notifiers configuration](https://docs.victoriametrics.com/victoriametrics/vmalert/#notifier-configuration-file). Consider limiting user's access
|
||||
to them if this information is sensitive.
|
||||
Specifically:
|
||||
* Log messages, the UI, and exported metrics contain full path to the configured rule files. These file paths can be stripped
|
||||
by enabling `-rule.stripFilePath` command-line flag {{% available_from "#" %}};
|
||||
* Datasource address is sanitized in log messages, UI and exported metrics, can be shown by enabling ` -datasource.showURL`;
|
||||
* Notifier addresses are sanitized in log messages, UI and exported metrics, can be shown by enabling `-notifier.showURL`;
|
||||
* Remote read address is sanitized in log messages, UI and exported metrics, can be shown by enabling ` --remoteRead.showURL`;
|
||||
* Remote write address is sanitized in log messages, UI and exported metrics, can be shown by enabling ` -remoteWrite.showURL`.
|
||||
|
||||
[Alerts state](#alerts-state-on-restarts) page or [debug mode](#debug-mode) could emit additional information about configured
|
||||
datasource URL, GET params and headers. Sensitive information such as passwords or auth tokens is stripped by default.
|
||||
|
||||
@@ -434,6 +434,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
|
||||
Minimum amount of time to wait before resending an alert to notifier.
|
||||
-rule.resultsLimit int
|
||||
Limits the number of alerts or recording results a single rule can produce. Can be overridden by the limit option under group if specified. If exceeded, the rule will be marked with an error and all its results will be discarded. 0 means no limit.
|
||||
-rule.stripFilePath
|
||||
Whether to strip rule file paths in logs and all API responses, including /metrics. For example, file path '/path/to/tenant_id/rules.yml' will be stripped to 'groupHashID/rules.yml'. This flag may be useful for hiding sensitive information in file paths, such as S3 bucket details.
|
||||
-rule.templates array
|
||||
Path or glob pattern to location with go template definitions for rules annotations templating. Flag can be specified multiple times.
|
||||
Examples:
|
||||
|
||||
@@ -32,8 +32,6 @@ sitemap:
|
||||
Optional path to TLS Root CA for verifying client certificates at the corresponding -httpListenAddr when -mtls is enabled. By default the host system TLS Root CA is used for client certificate verification. This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
|
||||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
|
||||
-rule.stripFilePath
|
||||
Whether to strip file path in responses from the api/v1/rules API for files configured via -rule cmd-line flag. For example, the file path '/path/to/tenant_id/rules.yml' will be stripped to just 'rules.yml'. This flag might be useful to hide sensitive information in file path such as tenant ID. This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
|
||||
-s3.configFilePath string
|
||||
Path to file with S3 configs. Configs are loaded from default location if not set.
|
||||
See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html . This flag is available only in Enterprise binaries. See https://docs.victoriametrics.com/victoriametrics/enterprise/
|
||||
|
||||
@@ -55,7 +55,7 @@ Feel free to [contact us](mailto:info@victoriametrics.com) if you need a customi
|
||||
* [Per-tenant authorization](#per-tenant-authorization)
|
||||
* [mTLS-based request routing](#mtls-based-request-routing)
|
||||
* [Enforcing query args](#enforcing-query-args)
|
||||
* [OIDC authorization](#oidc-authorization)
|
||||
* [OIDC discovery](#oidc-discovery)
|
||||
|
||||
### Simple HTTP proxy
|
||||
|
||||
|
||||
@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
|
||||
|
||||
Download and unpack vmctl:
|
||||
```sh
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.140.0/vmutils-darwin-arm64-v1.140.0.tar.gz
|
||||
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.142.0/vmutils-darwin-arm64-v1.142.0.tar.gz
|
||||
|
||||
tar xzf vmutils-darwin-arm64-v1.140.0.tar.gz
|
||||
tar xzf vmutils-darwin-arm64-v1.142.0.tar.gz
|
||||
```
|
||||
|
||||
Once binary is unpacked, see the full list of supported modes by running the following command:
|
||||
|
||||
@@ -29,7 +29,7 @@ OPTIONS:
|
||||
--prom-filter-time-end value The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'
|
||||
--prom-filter-label value Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.
|
||||
--prom-filter-label-value value Prometheus regular expression to filter label from "prom-filter-label" flag. (default: ".*")
|
||||
--prom-tmp-dir-path value Path to directory to be used for temporary files. (default: "/var/folders/9b/rj_f77q52w57vnxx236qh51m0000gn/T/")
|
||||
--prom-tmp-dir-path value Path to directory to be used for temporary files. (default: "/var/folders/ds/3kj5p3v17ll0hsyvq380ryvm0000gn/T/")
|
||||
--vm-addr value VictoriaMetrics address to perform import requests.
|
||||
Should be the same as --httpListenAddr value for single-node version or vminsert component.
|
||||
When importing into the clustered version do not forget to set additionally --vm-account-id flag.
|
||||
|
||||
@@ -55,6 +55,9 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
|
||||
Whether to use pread() instead of mmap() for reading data files. By default, mmap() is used for 64-bit arches and pread() is used for 32-bit arches, since they cannot read data files bigger than 2^32 bytes in memory. mmap() is usually faster for reading small data chunks than pread()
|
||||
-fs.maxConcurrency int
|
||||
The maximum number of concurrent goroutines to work with files; smaller values may help reducing Go scheduling latency on systems with small number of CPU cores; higher values may help reducing data ingestion latency on systems with high-latency storage such as NFS or Ceph (default fsutil.getDefaultConcurrency())
|
||||
-futureRetention value
|
||||
Data with timestamps bigger than now+futureRetention is automatically deleted. The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention
|
||||
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 2d)
|
||||
-http.connTimeout duration
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpserver
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
_ "embed"
|
||||
@@ -165,7 +166,7 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler, disableB
|
||||
// Do not set ReadTimeout and WriteTimeout here,
|
||||
// since these timeouts must be controlled by request handlers.
|
||||
|
||||
ErrorLog: logger.StdErrorLogger(),
|
||||
ErrorLog: log.New(&tlsErrorSkipLogger{}, "", 0),
|
||||
}
|
||||
s.s.SetKeepAlivesEnabled(!*disableKeepAlive)
|
||||
if *connTimeout > 0 {
|
||||
@@ -806,3 +807,14 @@ func LogError(req *http.Request, errStr string) {
|
||||
remoteAddr := GetQuotedRemoteAddr(req)
|
||||
logger.Errorf("uri: %s, remote address: %q: %s", uri, remoteAddr, errStr)
|
||||
}
|
||||
|
||||
type tlsErrorSkipLogger struct{}
|
||||
|
||||
func (*tlsErrorSkipLogger) Write(p []byte) (int, error) {
|
||||
// skip common health check errors produced by Kubernetes and other tools
|
||||
if bytes.Contains(p, []byte("TLS handshake error")) &&
|
||||
(bytes.Contains(p, []byte("EOF")) || bytes.Contains(p, []byte("connection reset by peer"))) {
|
||||
return len(p), nil
|
||||
}
|
||||
return logger.StdErrorLogger().Writer().Write(p)
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ type MetricMetadata struct {
|
||||
func (mm *MetricMetadata) reset() {
|
||||
mm.Name = ""
|
||||
mm.Unit = ""
|
||||
mm.Description = ""
|
||||
mm.Type = 0
|
||||
}
|
||||
|
||||
@@ -583,8 +584,12 @@ type Metric struct {
|
||||
|
||||
func (m *Metric) marshalProtobuf(mm *easyproto.MessageMarshaler) {
|
||||
mm.AppendString(1, m.Name)
|
||||
mm.AppendString(2, m.Description)
|
||||
mm.AppendString(3, m.Unit)
|
||||
if m.Description != "" {
|
||||
mm.AppendString(2, m.Description)
|
||||
}
|
||||
if m.Unit != "" {
|
||||
mm.AppendString(3, m.Unit)
|
||||
}
|
||||
switch {
|
||||
case m.Gauge != nil:
|
||||
m.Gauge.marshalProtobuf(mm.AppendMessage(5))
|
||||
@@ -619,6 +624,8 @@ func (dctx *decoderContext) decodeMetric(src []byte) error {
|
||||
// repeated opentelemetry.proto.common.v1.KeyValue metadata = 12;
|
||||
// }
|
||||
|
||||
dctx.mm.reset()
|
||||
|
||||
metricName, ok, err := easyproto.GetString(src, 1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read metric name: %w", err)
|
||||
|
||||
@@ -248,6 +248,30 @@ func TestParseStream(t *testing.T) {
|
||||
},
|
||||
},
|
||||
)
|
||||
gaugeWithoutMetadata := generateGauge("my-gauge-without-meta", "")
|
||||
gaugeWithoutMetadata.Description = ""
|
||||
f(
|
||||
[]*pb.Metric{
|
||||
generateGauge("my-gauge-milliseconds", "ms"),
|
||||
gaugeWithoutMetadata,
|
||||
},
|
||||
[]prompb.TimeSeries{
|
||||
newTimeSeries("my_gauge_milliseconds", 15000, 15.0, jobLabelValue, kvLabel("label1", "value1")),
|
||||
newTimeSeries("my_gauge_without_meta", 15000, 15.0, jobLabelValue, kvLabel("label1", "value1")),
|
||||
},
|
||||
[]prompb.MetricMetadata{
|
||||
{
|
||||
MetricFamilyName: "my_gauge_milliseconds",
|
||||
Help: "I'm a gauge",
|
||||
Type: prompb.MetricTypeGauge,
|
||||
Unit: "ms",
|
||||
},
|
||||
{
|
||||
MetricFamilyName: "my_gauge_without_meta",
|
||||
Type: prompb.MetricTypeGauge,
|
||||
},
|
||||
},
|
||||
)
|
||||
*usePrometheusNaming = prevPromNaming
|
||||
|
||||
// Test gauge with ratio suffix
|
||||
|
||||
Reference in New Issue
Block a user