Compare commits

..

2 Commits

Author SHA1 Message Date
func25
b4dca01ede update 2026-07-16 20:50:32 +07:00
func25
9636f9e171 update 2026-07-15 21:18:37 +07:00
91 changed files with 1745 additions and 3612 deletions

View File

@@ -65,7 +65,7 @@ jobs:
arch: amd64
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false

View File

@@ -13,7 +13,7 @@ jobs:
contents: read
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
# needed for proper diff
fetch-depth: 0

View File

@@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0 # we need full history for commit verification
persist-credentials: false

View File

@@ -17,7 +17,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false

View File

@@ -31,7 +31,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
@@ -54,14 +54,14 @@ jobs:
restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
- name: Initialize CodeQL
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
languages: go
- name: Autobuild
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
category: 'language:go'

View File

@@ -18,13 +18,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
path: __vm
persist-credentials: false
- name: Checkout private code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: VictoriaMetrics/vmdocs
token: ${{ secrets.VM_BOT_GH_TOKEN }}

View File

@@ -34,7 +34,7 @@ jobs:
runs-on: 'vm-runner'
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
@@ -79,7 +79,7 @@ jobs:
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
@@ -106,7 +106,7 @@ jobs:
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false

View File

@@ -26,11 +26,13 @@ jobs:
vmui-checks:
name: VMUI Checks (lint, test, typecheck)
permissions:
checks: write
contents: read
pull-requests: read
runs-on: ubuntu-latest
steps:
- name: Code checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
@@ -68,6 +70,12 @@ jobs:
env:
VMUI_SKIP_INSTALL: true
- name: Annotate Code Linting Results
uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9 # 3.0.0
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
report-json: app/vmui/packages/vmui/vmui-lint-report.json
- name: Check overall status
run: |
echo "Lint status: ${{ steps.lint.outcome }}"

View File

@@ -462,9 +462,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
return true
case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
promscrapeMetricRelabelDebugRequests.Inc()
rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
promscrape.WriteMetricRelabelDebug(w, r)
return true
case "/prometheus/target-relabel-debug", "/target-relabel-debug":
promscrapeTargetRelabelDebugRequests.Inc()

View File

@@ -12,7 +12,6 @@ import (
"github.com/VictoriaMetrics/metrics"
"gopkg.in/yaml.v2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -83,16 +82,6 @@ func WriteRelabelConfigData(w io.Writer) {
_, _ = w.Write(*p)
}
// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
func GetRemoteWriteRelabelConfigString() string {
var bb bytesutil.ByteBuffer
WriteRelabelConfigData(&bb)
if bb.Len() == 0 {
return ""
}
return string(bb.B)
}
// WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
func WriteURLRelabelConfigData(w io.Writer) {
p := remoteWriteURLRelabelConfigData.Load()
@@ -119,24 +108,6 @@ func WriteURLRelabelConfigData(w io.Writer) {
_, _ = w.Write(d)
}
// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
func GetURLRelabelConfigString() []string {
p := remoteWriteURLRelabelConfigData.Load()
if p == nil {
return nil
}
var ss []string
for i := range *remoteWriteURLs {
cfgData := (*p)[i]
var cfgDataBytes []byte
if cfgData != nil {
cfgDataBytes, _ = yaml.Marshal(cfgData)
}
ss = append(ss, string(cfgDataBytes))
}
return ss
}
func reloadRelabelConfigs() {
rcs := allRelabelConfigs.Load()
if !rcs.isSet() {

View File

@@ -1064,7 +1064,7 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
copyTimeSeriesIfNeeded := func() {
if v == nil {
v = tssPool.Get().(*[]prompb.TimeSeries)
v := tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
}
}

View File

@@ -386,7 +386,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf("reject request from the user %s because the request body couldn't be read in -maxQueueDuration=%s; read %d bytes in %s",
userName, *maxQueueDuration, len(buf), d.Truncate(time.Second)),
StatusCode: http.StatusRequestTimeout,
StatusCode: http.StatusBadRequest,
}
}

View File

@@ -1,54 +0,0 @@
//go:build synctest
package main
import (
"context"
"net/http"
"testing"
"testing/synctest"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
)
func TestBufferRequestBody_Timeout(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
defaultMaxQueueDuration := *maxQueueDuration
defer func() {
*maxQueueDuration = defaultMaxQueueDuration
}()
*maxQueueDuration = 100 * time.Millisecond
ctx, cancel := context.WithTimeout(context.Background(), *maxQueueDuration)
defer cancel()
_, err := bufferRequestBody(ctx, &timeoutBody{delay: *maxQueueDuration + 100*time.Millisecond}, "foo")
if err == nil {
t.Fatalf("expecting non-nil error")
}
esc, ok := err.(*httpserver.ErrorWithStatusCode)
if !ok {
t.Fatalf("unexpected error type: %s", err)
}
if esc.StatusCode != http.StatusRequestTimeout {
t.Fatalf("read request body timeout meet unexpected status code; got %d; want %d", esc.StatusCode, http.StatusRequestTimeout)
}
})
}
type timeoutBody struct {
delay time.Duration
}
func (r *timeoutBody) Read(_ []byte) (int, error) {
time.Sleep(r.delay)
return 0, context.DeadlineExceeded
}
func (r *timeoutBody) Close() error {
return nil
}

View File

@@ -541,7 +541,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
return true
case "/metric-relabel-debug":
promscrapeMetricRelabelDebugRequests.Inc()
promscrape.WriteMetricRelabelDebug(w, r, "", nil)
promscrape.WriteMetricRelabelDebug(w, r)
return true
case "/target-relabel-debug":
promscrapeTargetRelabelDebugRequests.Inc()

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1 +0,0 @@
var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t,n)=>()=>{if(n)throw n[0];try{return e&&(t=e(e=0)),t}catch(e){throw n=[e],e}},s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

View File

@@ -0,0 +1 @@
var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,t)=>()=>(e&&(t=e(e=0)),t),s=(e,t)=>()=>(t||(e((t={exports:{}}).exports,t),e=null),t.exports),c=(e,n)=>{let r={};for(var i in e)t(r,i,{get:e[i],enumerable:!0});return n||t(r,Symbol.toStringTag,{value:`Module`}),r},l=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},u=(n,r,a)=>(a=n==null?{}:e(i(n)),l(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n)),d=e=>a.call(e,`module.exports`)?e[`module.exports`]:l(t({},`__esModule`,{value:!0}),e);export{u as a,d as i,o as n,c as r,s as t};

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -37,11 +37,11 @@
<meta property="og:title" content="UI for VictoriaMetrics">
<meta property="og:url" content="https://victoriametrics.com/">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<script type="module" crossorigin src="./assets/index-D5egN2id.js"></script>
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-CNC7AqOf.js">
<link rel="modulepreload" crossorigin href="./assets/vendor-DwJYpOdw.js">
<script type="module" crossorigin src="./assets/index-xYKUiOTH.js"></script>
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-Cyuzqnbw.js">
<link rel="modulepreload" crossorigin href="./assets/vendor-B83wxFqK.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">
<link rel="stylesheet" crossorigin href="./assets/index-BJqoElx2.css">
<link rel="stylesheet" crossorigin href="./assets/index-BBUnmLOr.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>

View File

@@ -35,10 +35,7 @@ var (
"By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. "+
"This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. "+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
vmselectAddr = flag.String("vmselectAddr", "", "TCP address to listen for incoming connections from vmselect. "+
"When set, the node will be able to accept cluster-native vmselect RPC requests as if it were vmstorage. "+
"The tenant ID assigned to this node's data is controlled by -accountID and -projectID flags. "+
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy")
vmselectAddr = flag.String("vmselectAddr", "", "TCP address to accept connections from vmselect services")
vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
"This reduces CPU usage at the cost of higher network bandwidth usage")
snapshotAuthKey = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")

View File

@@ -1,4 +1,4 @@
FROM golang:1.26.5 AS build-web-stage
FROM golang:1.26.4 AS build-web-stage
COPY build /build
WORKDIR /build

File diff suppressed because it is too large Load Diff

View File

@@ -9,7 +9,7 @@
"start": "vite",
"start:playground": "cross-env PLAYGROUND=true npm run start",
"build": "vite build",
"lint": "eslint --format stylish 'src/**/*.{ts,tsx}'",
"lint": "eslint --output-file vmui-lint-report.json --format json 'src/**/*.{ts,tsx}'",
"lint:local": "eslint --ext .ts,.tsx -f stylish src",
"lint:fix": "eslint 'src/**/*.{ts,tsx}' --fix",
"copy-metricsql-docs": "cp ../../../../docs/MetricsQL.md src/assets/MetricsQL.md || true",
@@ -23,40 +23,40 @@
"classnames": "^2.5.1",
"dayjs": "^1.11.21",
"lodash.debounce": "^4.0.8",
"marked": "^18.0.6",
"preact": "^10.29.7",
"qs": "^6.15.3",
"marked": "^18.0.5",
"preact": "^10.29.2",
"qs": "^6.15.2",
"react-input-mask": "^2.0.4",
"react-router-dom": "^7.18.1",
"react-router-dom": "^7.17.0",
"uplot": "^1.6.32",
"vite": "^8.1.4",
"vite": "^8.0.16",
"web-vitals": "^5.3.0"
},
"devDependencies": {
"@eslint/eslintrc": "^3.3.6",
"@eslint/eslintrc": "^3.3.5",
"@eslint/js": "^9.39.2",
"@preact/preset-vite": "^2.10.5",
"@testing-library/jest-dom": "^6.9.1",
"@testing-library/preact": "^3.2.4",
"@types/lodash.debounce": "^4.0.9",
"@types/node": "^26.1.1",
"@types/node": "^25.9.2",
"@types/qs": "^6.15.1",
"@types/react": "^19.2.17",
"@types/react-input-mask": "^3.0.6",
"@types/react-router-dom": "^5.3.3",
"@typescript-eslint/eslint-plugin": "^8.64.0",
"@typescript-eslint/parser": "^8.64.0",
"@typescript-eslint/eslint-plugin": "^8.61.0",
"@typescript-eslint/parser": "^8.61.0",
"cross-env": "^10.1.0",
"eslint": "^9.39.2",
"eslint-plugin-react": "^7.37.5",
"eslint-plugin-unused-imports": "^4.4.1",
"globals": "^17.7.0",
"http-proxy-middleware": "^4.2.0",
"globals": "^17.6.0",
"http-proxy-middleware": "^4.1.0",
"jsdom": "^29.1.1",
"postcss": "^8.5.19",
"postcss": "^8.5.15",
"sass-embedded": "^1.100.0",
"typescript": "^6.0.3",
"vitest": "^4.1.10"
"vitest": "^4.1.8"
},
"browserslist": {
"production": [

View File

@@ -98,7 +98,7 @@ const BaseAlert = ({ item, group }: BaseAlertProps) => {
{!!Object.keys(item.annotations || {}).length && (
<>
<span className="vm-alerts-title">Annotations</span>
<table className="vm-annotations-table">
<table>
<colgroup>
<col className="vm-col-md"/>
<col/>

View File

@@ -121,7 +121,7 @@ const BaseRule = ({ item, group }: BaseRuleProps) => {
{!!Object.keys(item?.annotations || {}).length && (
<>
<span className="vm-alerts-title">Annotations</span>
<table className="vm-annotations-table">
<table>
<colgroup>
<col className="vm-col-md"/>
<col/>

View File

@@ -32,7 +32,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
const match = searchParams.get("match");
const focusLabel = searchParams.get("focusLabel");
const isMetric = /__name__/.test(match || "");
const showMetricNameStats = !(match || focusLabel);
const progress = totalSeries / totalSeriesAll * 100;
const diff = totalSeries - totalSeriesPrev;
const dynamic = Math.abs(diff) / totalSeriesPrev * 100;
@@ -56,7 +56,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
},
].filter(t => t.display);
if (!totals.length && !showMetricNameStats) {
if (!totals.length) {
return null;
}
@@ -75,7 +75,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
<h4 className="vm-cardinality-totals-card__title">
{info && (
<Tooltip title={<p className="vm-cardinality-totals-card__tooltip">{info}</p>}>
<div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon /></div>
<div className="vm-cardinality-totals-card__info-icon"><InfoOutlinedIcon/></div>
</Tooltip>
)}
{title}
@@ -99,10 +99,7 @@ const CardinalityTotals: FC<CardinalityTotalsProps> = ({
)}
</div>
))}
{
showMetricNameStats &&
<CardinalityMetricNameStats metricNameStats={metricNameStats} />
}
<CardinalityMetricNameStats metricNameStats={metricNameStats}/>
</div>
);
};

View File

@@ -62,13 +62,6 @@
padding-right: 40px;
}
.vm-annotations-table {
tbody > tr > td {
vertical-align: top;
white-space: pre-line;
}
}
display: flex;
flex-direction: column;
align-items: stretch;

View File

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.24.1
ROOT_IMAGE_SCRATCH ?= scratch
CERTS_IMAGE := alpine:3.24.1
GO_BUILDER_IMAGE := golang:1.26.5
GO_BUILDER_IMAGE := golang:1.26.4
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

View File

@@ -3,7 +3,7 @@ services:
# It scrapes targets defined in --promscrape.config
# And forward them to --remoteWrite.url
vmagent:
image: victoriametrics/vmagent:v1.148.0
image: victoriametrics/vmagent:v1.147.0
depends_on:
- "vmauth"
ports:
@@ -42,14 +42,14 @@ services:
# vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
# where N is number of vmstorages (2 in this case).
vmstorage-1:
image: victoriametrics/vmstorage:v1.148.0-cluster
image: victoriametrics/vmstorage:v1.147.0-cluster
volumes:
- strgdata-1:/storage
command:
- "--storageDataPath=/storage"
restart: always
vmstorage-2:
image: victoriametrics/vmstorage:v1.148.0-cluster
image: victoriametrics/vmstorage:v1.147.0-cluster
volumes:
- strgdata-2:/storage
command:
@@ -59,7 +59,7 @@ services:
# vminsert is ingestion frontend. It receives metrics pushed by vmagent,
# pre-process them and distributes across configured vmstorage shards.
vminsert-1:
image: victoriametrics/vminsert:v1.148.0-cluster
image: victoriametrics/vminsert:v1.147.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -68,7 +68,7 @@ services:
- "--storageNode=vmstorage-2:8400"
restart: always
vminsert-2:
image: victoriametrics/vminsert:v1.148.0-cluster
image: victoriametrics/vminsert:v1.147.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -80,7 +80,7 @@ services:
# vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
# vmselect collects results from configured `--storageNode` shards.
vmselect-1:
image: victoriametrics/vmselect:v1.148.0-cluster
image: victoriametrics/vmselect:v1.147.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -90,7 +90,7 @@ services:
- "--vmalert.proxyURL=http://vmalert:8880"
restart: always
vmselect-2:
image: victoriametrics/vmselect:v1.148.0-cluster
image: victoriametrics/vmselect:v1.147.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -105,7 +105,7 @@ services:
# read requests from Grafana, vmui, vmalert among vmselects.
# It can be used as an authentication proxy.
vmauth:
image: victoriametrics/vmauth:v1.148.0
image: victoriametrics/vmauth:v1.147.0
depends_on:
- "vmselect-1"
- "vmselect-2"
@@ -119,7 +119,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
image: victoriametrics/vmalert:v1.148.0
image: victoriametrics/vmalert:v1.147.0
depends_on:
- "vmauth"
ports:

View File

@@ -3,7 +3,7 @@ services:
# It scrapes targets defined in --promscrape.config
# And forward them to --remoteWrite.url
vmagent:
image: victoriametrics/vmagent:v1.148.0
image: victoriametrics/vmagent:v1.147.0
depends_on:
- "victoriametrics"
ports:
@@ -18,7 +18,7 @@ services:
# VictoriaMetrics instance, a single process responsible for
# storing metrics and serve read requests.
victoriametrics:
image: victoriametrics/victoria-metrics:v1.148.0
image: victoriametrics/victoria-metrics:v1.147.0
ports:
- 8428:8428
- 8089:8089
@@ -59,7 +59,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
image: victoriametrics/vmalert:v1.148.0
image: victoriametrics/vmalert:v1.147.0
depends_on:
- "victoriametrics"
- "alertmanager"

View File

@@ -30,35 +30,6 @@ groups:
summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5m"
- alert: SchedulerWorkerDown
expr: >
(vmanomaly_scheduler_alive{job=~".*vmanomaly.*"} == 0)
and on (job, instance, scheduler_alias, preset)
(sum(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"})
by (job, instance, scheduler_alias, preset) >= 0)
for: 2m
labels:
severity: critical
annotations:
summary: "Scheduler {{ $labels.scheduler_alias }} is down in {{ $labels.job }} ({{ $labels.instance }})"
description: |
Periodic scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} has remained down for more than 2 minutes
after entering automatic recovery. The restart policy may be exhausted.
Check vmanomaly logs and the vmanomaly_scheduler_restarts_total metric for restart failures.
- alert: FrequentSchedulerRestarts
expr: >
sum(increase(vmanomaly_scheduler_restarts_total{job=~".*vmanomaly.*"}[15m]))
by (job, instance, scheduler_alias, preset) > 2
labels:
severity: warning
annotations:
summary: "Scheduler {{ $labels.scheduler_alias }} is restarting frequently in {{ $labels.job }} ({{ $labels.instance }})"
description: |
Scheduler {{ $labels.scheduler_alias }} from preset {{ $labels.preset }} had more than two restart attempts
during the last 15 minutes. The service may be self-healing, but repeated restarts indicate an unstable
scheduler, configuration, datasource, or runtime dependency. Check vmanomaly logs for the original failure.
# default value of 900 Should be changed to the scrape_interval for pull metrics. For push metrics this should be the lowest fit_every or infer_every in your vmanomaly config.
- alert: NoSelfMonitoringMetrics
expr: >

View File

@@ -1,6 +1,6 @@
services:
vmagent:
image: victoriametrics/vmagent:v1.148.0
image: victoriametrics/vmagent:v1.147.0
depends_on:
- "victoriametrics"
ports:
@@ -14,7 +14,7 @@ services:
restart: always
victoriametrics:
image: victoriametrics/victoria-metrics:v1.148.0
image: victoriametrics/victoria-metrics:v1.147.0
ports:
- 8428:8428
volumes:
@@ -40,7 +40,7 @@ services:
restart: always
vmalert:
image: victoriametrics/vmalert:v1.148.0
image: victoriametrics/vmalert:v1.147.0
depends_on:
- "victoriametrics"
ports:
@@ -59,7 +59,7 @@ services:
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
restart: always
vmanomaly:
image: victoriametrics/vmanomaly:v1.30.0
image: victoriametrics/vmanomaly:v1.29.7
depends_on:
- "victoriametrics"
ports:

View File

@@ -1,18 +1,16 @@
schedulers:
periodic:
infer_every: "1m"
fit_every: "100w" # the online model keeps learning during inference
fit_window: "2w"
fit_every: "1h"
fit_window: "2d" # 2d-14d based on the presence of weekly seasonality in your data
models:
temporal_envelope:
class: "temporal_envelope"
queries: "all"
schedulers: "all"
seasonalities: ["hod_smooth", "dow_smooth"]
alpha: 0.005
loss_reactivity: 5
iqr_threshold: 2
prophet:
class: "prophet"
args:
interval_width: 0.98
weekly_seasonality: False # comment it if your data has weekly seasonality
yearly_seasonality: False
reader:
datasource_url: "http://victoriametrics:8428/"
@@ -28,4 +26,4 @@ writer:
monitoring:
pull: # Enable /metrics endpoint.
addr: "0.0.0.0"
port: 8490
port: 8490

View File

@@ -69,21 +69,18 @@ See more details at [VictoriaMetrics/mcp-victoriatraces](https://github.com/Vict
REST API and documentation for AI-assisted anomaly detection, model management, and observability insights.
Capabilities include:
- Check `vmanomaly` health, build information, compatibility, and self-monitoring metrics
- Inspect model schemas and validate model or complete service configurations
- Profile sampled query results for trends, seasonalities, changepoints, gaps, and intermittent behavior
- Run asynchronous autotune tasks and turn their results into data-driven model recommendations
- Generate complete `vmanomaly` YAML configurations and [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules)
- Search embedded `vmanomaly` documentation with fuzzy matching
- Health Monitoring: Check `vmanomaly` server health and build information
- Model Management: List, validate, and configure anomaly detection models (like `zscore_online`, `prophet`, and more)
- Configuration Generation: Generate complete `vmanomaly` YAML configurations
- Alert Rule Generation: Generate [`vmalert`](https://docs.victoriametrics.com/victoriametrics/vmalert/) [alerting rules](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules) based on [anomaly score metrics](https://docs.victoriametrics.com/anomaly-detection/faq/#what-is-anomaly-score) to simplify alerting setup
- Documentation Search: Full-text search across embedded `vmanomaly` documentation with fuzzy matching
See more details at [VictoriaMetrics/mcp-vmanomaly](https://github.com/VictoriaMetrics/mcp-vmanomaly).
# UI Copilot
vmanomaly UI includes an [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) that can assist users with anomaly detection tasks, model configuration, and troubleshooting, changing the UI state based on user queries and providing actionable suggestions through automated data profiling and validation. The AI Copilot is powered by respective [MCP Server](#vmanomaly-mcp-server) and [Agent Skills](#agent-skills), enabling it to understand the context of the user's actions and provide relevant guidance.
# Agent Skills
[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
and troubleshoot VictoriaMetrics observability components, including metrics, logs, traces, and [`vmanomaly`](https://docs.victoriametrics.com/anomaly detection/).
[Agent skills](https://github.com/VictoriaMetrics/skills) help AI agents and automation tools understand, operate,
and troubleshoot VictoriaMetrics observability components, including metrics, logs, and traces.
These skills provide predefined workflows and capabilities such as:
* Query metrics, logs, traces and alerts
@@ -92,9 +89,6 @@ These skills provide predefined workflows and capabilities such as:
* Cardinality optimization
* Unused metric detection
* Stream aggregation configuration
* Build validated `vmanomaly` configurations from measured time-series characteristics (e.g., seasonality, changepoints, trends)
* Query and operate the `vmanomaly` API
* Review existing anomaly detection configurations against real data and identify false-positive or model-data fit issues
To install the available skills for AI agents, run:
```sh

View File

@@ -9,7 +9,6 @@ tags:
- metrics
- logs
- traces
- anomaly detection
- AI
- AI integration
- agent

View File

@@ -14,29 +14,6 @@ aliases:
---
Please find the changelog for VictoriaMetrics Anomaly Detection below.
## v1.30.0
Released: 23-07-2026
- FEATURE: Added univariate and multivariate online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) models for complex time-series profiles. They combine robust causal trend, compact calendar and learned holiday behavior, changepoint adaptation, residual prediction intervals, future-horizon forecasts, and optional cross-series dependency detection in bounded state.
- FEATURE: Added asynchronous [`/api/v1/autotune/tasks`](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow) endpoints and improved unsupervised tuning with model-capability-aware objectives, optional causal exact validation for online models, constrained alert-volume selection, fitted-state complexity tie-breaking, and frozen model-specific parameters.
- FEATURE: Added `/api/v1/timeseries/characteristics` for bounded analysis of trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior across sampled query results. This improves automated model selection and search-space suggestions in both agentic workflows and [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance)-backed [UI](https://docs.victoriametrics.com/anomaly-detection/ui/) experiments.
- UI: Updated [vmanomaly UI](https://docs.victoriametrics.com/anomaly-detection/ui/) from [v1.7.2](https://docs.victoriametrics.com/anomaly-detection/ui/#v172) to [v1.8.0](https://docs.victoriametrics.com/anomaly-detection/ui/#v180). Notable mentions are model and query configuration, query prettification, fullscreen charts, out-of-date results alert and improved [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistance) suggestion and cancellation behavior.
- FEATURE: Made AI-assisted configuration more reliable with reusable [vmanomaly workflow skills](https://github.com/VictoriaMetrics/skills), data-aware model suggestions, protocol-safe cancellation and synchronized query, model, and anomaly-setting updates in [UI](https://docs.victoriametrics.com/anomaly-detection/ui/).
- IMPROVEMENT: Fitted history can be retained as a stronger prior through `history_strength` argument, requiring a reduced number of observations (say 2 weeks for weekly seasonality) to be effective if the history window is a good indicator of "normal" data patterns. Models supported: `mad_online`, `zscore_online`, `quantile_online`.
- IMPROVEMENT: Improved bounded datasource operation with optional ad-hoc series limits, stale-series lookback caps, and propagation of configured per-query sampling periods to models and autotune trials.
- IMPROVEMENT: Added `reader.fetch_timeout` and `reader.processing_timeout` for [`VmReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) and [`VLogsReader`](https://docs.victoriametrics.com/anomaly-detection/components/reader/#victorialogs-reader), allowing datasource reads and post-fetch processing to be tuned *independently* while preserving `reader.timeout` as the backward-compatible default for both phases.
- BUGFIX: Fixed [backtesting runs](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#backtesting-scheduler) across multiple fit cycles and for auto-tuned online wrappers, preventing duplicate or missing predictions while retaining causal model updates and restoration of compatible legacy auto-tuned state.
- BUGFIX: Prevented service shutdown after a [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) worker dies by adding bounded restart attempts with backoff and exposing restart health through [startup metrics](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics).
## v1.29.7
Released: 2026-06-25

View File

@@ -143,15 +143,9 @@ For information on migrating between different versions of `vmanomaly`, please r
> {{% available_from "v1.28.3" anomaly %}} Try our [MCP Server](https://github.com/VictoriaMetrics/mcp-vmanomaly) to get AI-assisted recommendations on selecting the best model and its configuration for your use case. See [installation guide](https://github.com/VictoriaMetrics/mcp-vmanomaly#installation) for more details.
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect:
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet).
- Use [Online MAD](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) for simple, mostly stationary data with no-to-slow trend, when robustness to outliers is important.
- Use [Online Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) for simple, light-tailed data where standard-deviation units are meaningful.
- Use [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} for complex data with trends, calendar patterns, holidays, or persistent shifts. It is the preferred *online* alternative to Prophet (which will be deprecated in the future releases).
- Use multivariate Temporal Envelope when normal relationships between aligned metrics matter. This should replace [Isolation Forest](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate) used in previous versions of `vmanomaly`, which will be deprecated in future releases.
- Use [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) when Prophet-specific decomposition outputs, or offline batch behavior are required. Consider using Temporal Envelope instead, as it is more efficient and provides better results in most cases.
There is also an option to auto-tune the most important parameters of a selected model class {{% available_from "v1.12.0" anomaly %}}. {{% available_from "v1.30.0" anomaly %}} The asynchronous autotune API can first profile a bounded sample through `/api/v1/timeseries/characteristics`, then tune a shared concrete configuration through `/api/v1/autotune/tasks`. See the [autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow).
Also, there is an option to auto-tune the most important (hyper)parameters of selected model class {{% available_from "v1.12.0" anomaly %}}, find [the details here](https://docs.victoriametrics.com/anomaly-detection/components/models/#autotuned).
Please refer to [respective blogpost on anomaly types and alerting heuristics](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/) for more details.
@@ -304,7 +298,7 @@ Configuration above will produce N intervals of full length (`fit_window`=14d +
## Forecasting
`vmanomaly` can generate future forecasts using [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) {{% available_from "v1.30.0" anomaly %}} or [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}. This is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
`vmanomaly` can generate future forecasts (e.g. using [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) {{% available_from "v1.25.3" anomaly %}}), which is helpful for capacity planning, resource allocation, or trend analysis when the underlying data is complex and exceeds what inline MetricsQL queries, including [predict_linear](https://docs.victoriametrics.com/victoriametrics/metricsql/#predict_linear), can handle.
> However, please note that this mode should be used with care, as the model will produce `yhat_{h}` (and probably `yhat_lower_{h}`, and `yhat_upper_{h}`) time series **for each timeseries returned by input queries and for each forecasting horizon specified in `forecast_at` argument, which can lead to a significant increase in the number of active timeseries in VictoriaMetrics TSDB**.
@@ -315,12 +309,12 @@ Here's an example of how to produce forecasts using `vmanomaly` and combine it w
schedulers:
periodic_5m: # this scheduler will be used to produce anomaly scores each 5 minutes using "regular" simple model
class: 'periodic'
fit_every: '100w'
fit_every: '30d'
fit_window: '3d'
infer_every: '5m'
periodic_forecast: # this scheduler will be used to produce forecasts each 24h using "daily" model
class: 'periodic'
fit_every: '1000w'
fit_every: '7d'
fit_window: '730d' # to fit the model on 2 years of data to account for seasonality and holidays
infer_every: '24h'
# https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader
@@ -362,17 +356,15 @@ models:
quantiles: [0.25, 0.5, 0.75] # to produce median and upper quartiles
iqr_threshold: 2.0
envelope_1d:
class: 'temporal_envelope'
prophet_1d:
class: 'prophet'
queries: ['disk_usage_perc_1d']
schedulers: ['periodic_forecast']
clip_predictions: True
detection_direction: 'above_expected' # as we are interested in spikes in capacity planning
forecast_at: ['3d', '7d'] # this will produce forecasts for 3 and 7 days ahead
provide_series: ['yhat', 'yhat_upper'] # to write forecasts back to VictoriaMetrics, omitting `yhat_lower` as it is not needed in this example
# other model params
seasonalities:
# other model params, yearly_seasonality may stay
# https://facebook.github.io/prophet/docs/quick_start#python-api
args:
@@ -431,7 +423,7 @@ services:
# ...
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.30.0
image: victoriametrics/vmanomaly:v1.29.7
# ...
restart: always
volumes:
@@ -468,8 +460,6 @@ With the introduction of [online models](https://docs.victoriametrics.com/anomal
- **Optimized resource utilization**: By spreading the computational load over time and reducing peak demands, online models make more efficient use of resources and inducing less data transfer from VictoriaMetrics TSDB, improving overall system performance.
- **Faster convergence**: Online models can adapt {{% available_from "v1.23.0" anomaly %}} to changes in data patterns more quickly, which is particularly beneficial in dynamic environments where data characteristics may shift frequently. See `decay` argument description [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay).
{{% available_from "v1.30.0" anomaly %}} Online models expose a **common** `min_n_samples_seen` warmup: they continue learning, but emit `anomaly_score: 0` until enough observations have been seen. Online MAD, Z-score, and Seasonal Quantile also support `history_strength`, which retains fitted history as a stronger prior while inference continues to adapt the state.
> {{% available_from "v1.24.0" anomaly %}} Online models are best used in conjunction with [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) to preserve the model state across service restarts. This allows the model to continue adapting to new data without losing previously learned patterns, thus avoiding the need for a full `fit` stage to start working again. {{% available_from "v1.28.1" anomaly %}} Additionally, setting [retention policies](https://docs.victoriametrics.com/anomaly-detection/components/settings/#retention) helps manage disk space or RAM used by periodical cleanup of old model instances.
Here's an example of how we can switch from (offline) [Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score) to [Online Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score):
@@ -651,7 +641,7 @@ options:
Heres an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
```sh
docker pull victoriametrics/vmanomaly:v1.30.0 && docker image tag victoriametrics/vmanomaly:v1.30.0 vmanomaly
docker pull victoriametrics/vmanomaly:v1.29.7 && docker image tag victoriametrics/vmanomaly:v1.29.7 vmanomaly
```
```sh

View File

@@ -45,7 +45,7 @@ There are 2 types of compatibility to consider when migrating in stateful mode:
| Group start | Group end | Compatibility | Notes |
|---------|--------- |------------|-------|
| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300) | Fully Compatible | v1.30.0 adds new [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model state without changing the compatibility of existing model and data artifacts. |
| [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) | [v1.29.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1297) | Fully Compatible | - |
| [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) | Partially compatible* | Dumped models of class [prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) and [seasonal quantile](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile) have problems with loading to [v1.29.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1290) due to dropped `pytz` library. **Upgrading directly from v1.28.7 to [v1.29.1](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1291) with a fix is suggested** |
| [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262) | [v1.28.7](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1287) | Fully Compatible | [v1.28.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1280) introduced [rolling](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) model class drop in favor of [online](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) models (`rolling_quantile` and `std` models), however, it does not impact compatibility, as artifacts were not produced by default for rolling models. Also, offline `mad` and `zscore` models are redirecting to their respective online counterparts since [v1.28.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1284). |
| [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) | [v1.26.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270) | Partially Compatible* | [v1.25.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1253) introduced `forecast_at` argument for base [univariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) and `Prophet` [models](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), however, itself remains backward-reversible from newer states like [v1.26.2](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1262), [v1.27.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1270). (All models except `isolation_forest_multivariate` class will be dropped) |

View File

@@ -132,7 +132,7 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
1. Pull Docker image:
```sh
docker pull victoriametrics/vmanomaly:v1.30.0
docker pull victoriametrics/vmanomaly:v1.29.7
```
2. Create the license file with your license key.
@@ -152,7 +152,7 @@ docker run -it \
-v ./license:/license \
-v ./config.yaml:/config.yaml \
-p 8490:8490 \
victoriametrics/vmanomaly:v1.30.0 \
victoriametrics/vmanomaly:v1.29.7 \
/config.yaml \
--licenseFile=/license \
--loggerLevel=INFO \
@@ -169,7 +169,7 @@ docker run -it \
-e VMANOMALY_DATA_DUMPS_DIR=/tmp/vmanomaly/data \
-e VMANOMALY_MODEL_DUMPS_DIR=/tmp/vmanomaly/models \
-p 8490:8490 \
victoriametrics/vmanomaly:v1.30.0 \
victoriametrics/vmanomaly:v1.29.7 \
/config.yaml \
--licenseFile=/license \
--loggerLevel=INFO \
@@ -182,7 +182,7 @@ services:
# ...
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.30.0
image: victoriametrics/vmanomaly:v1.29.7
# ...
restart: always
volumes:
@@ -245,7 +245,7 @@ Before deploying, check the correctness of your configuration validate config fi
### Example
Here is an example of a config file that runs the online [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope) model on a CPU metric. The scheduler runs inference every five minutes and performs a full refit only every 100 weeks; between refits the model updates causally from each inference batch. The initial fit uses four weeks of data. The model produces `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging, and its hour-of-day and day-of-week profiles follow the query timezone and daylight-saving-time changes.
Here is an example of a config file that will run the [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) model on `vm_cache_entries` metric, with periodic scheduler that runs inference every minute and fits the model every day. The model will be trained on the last 2 weeks of data each time it is (re)fitted. The model will produce `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging purposes. The model will be timezone-aware and will use cyclical encoding for the hour of the day and day of the week seasonality.
```yaml
settings:
@@ -260,30 +260,38 @@ settings:
# scheduler: INFO
# reader: INFO
# writer: INFO
model.online.temporal_envelope: WARNING
model.prophet: WARNING
schedulers:
100w_5m:
1d_5m:
# https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler
class: 'periodic'
infer_every: '5m'
scatter_infer_jobs: true
# Temporal Envelope learns online between full refits.
fit_every: '100w'
fit_every: '1d'
fit_window: '4w'
models:
# https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope
temporal_envelope_model:
class: 'temporal_envelope'
queries: ['cpu_user']
schedulers: ['100w_5m']
# https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet
prophet_model:
class: 'prophet'
provide_series: ['anomaly_score', 'yhat', 'yhat_lower', 'yhat_upper'] # for debugging
seasonalities: ['hod_smooth', 'dow_smooth']
alpha: 0.005 # trend reactivity; try 0.0025-0.02
loss_reactivity: 5 # try 1-5; lower values reduce the influence of spikes
iqr_threshold: 2 # try 1-4 to adjust data-driven interval width
min_n_samples_seen: 16
tz_aware: True # set to True if your data is timezone-aware, to deal with DST changes correctly
tz_use_cyclical_encoding: True
tz_seasonalities: # intra-day + intra-week seasonality
- name: 'hod' # intra-day seasonality, hour of the day
fourier_order: 4 # keep it 3-8 based on intraday pattern complexity
prior_scale: 10
- name: 'dow' # intra-week seasonality, time of the week
fourier_order: 2 # keep it 2-4, as dependencies are learned separately for each weekday
compression: # available since v1.28.1
window: "30m" # downsample 5m data into 30m intervals before fitting
agg_method: "mean" # use mean aggregation within each window
adjust_boundaries: true # adjust confidence intervals after downsampling
# inner model args (key-value pairs) accepted by
# https://facebook.github.io/prophet/docs/quick_start#python-api
args:
interval_width: 0.98 # see https://facebook.github.io/prophet/docs/uncertainty_intervals
reader:
class: 'vm' # use VictoriaMetrics as a data source
@@ -291,7 +299,6 @@ reader:
datasource_url: "https://play.victoriametrics.com/" # [YOUR_DATASOURCE_URL]
tenant_id: '0:0'
sampling_period: "5m"
tz: 'UTC' # set the IANA timezone that defines local calendar patterns, e.g. 'America/New_York'
series_processing_batch_size: 8 # number of time series to process together while preparing data for fit or infer stages
queries:
# define your queries with MetricsQL - https://docs.victoriametrics.com/victoriametrics/metricsql/

View File

@@ -135,8 +135,6 @@ These alerting rules complements the [dashboard](#grafana-dashboard) to monitor
`vmanomaly-health` alerting group:
- **`TooManyRestarts`**: Triggers if an instance restarts more than twice within 15 minutes, suggesting the process might be crashlooping and needs investigation.
- **`ServiceDown`**: Alerts if an instance is down for more than 5 minutes, indicating a service outage.
- **`SchedulerWorkerDown`**: {{% available_from "v1.30.0" anomaly %}} Alerts when an individual periodic scheduler remains down for more than 2 minutes after automatic recovery attempts. This catches partial service failures that a process-level `ServiceDown` alert cannot detect.
- **`FrequentSchedulerRestarts`**: {{% available_from "v1.30.0" anomaly %}} Warns when an individual scheduler has more than two restart attempts within 15 minutes. A single successful self-heal does not alert, while repeated attempts indicate a flapping scheduler or dependency.
- **`ProcessNearFDLimits`**: Alerts when the number of available file descriptors falls below 100, which could lead to severe degradation if the limit is exhausted.
- **`TooHighCPUUsage`**: Alerts when CPU usage exceeds 90% for a continuous 5-minute period, indicating possible resource exhaustion and the need to adjust resource allocation or load.
- **`TooHighMemoryUsage`**: Alerts when RAM usage exceeds 85% for a continuous 5-minute period and the need to adjust resource allocation or load.

View File

@@ -193,10 +193,9 @@ The best applications of this mode are:
### What you can do with Copilot
- **Ask questions** about any model (e.g. [Temporal Envelope](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
- **Ask questions** about any model (e.g. [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) - parameters, trade-offs, when to use each)
- **Improve detection quality** - describe what's wrong ("too many false positives", "missing spikes") and Copilot reads the config, searches the docs, and proposes a validated configuration change to fix the issue.
- **Get config suggestions inline** - suggestions appear as interactive cards with an explanation and a YAML diff; click **Apply** to write the change directly to your current settings, or **Decline** to keep the conversation going.
- {{% available_from "v1.30.0" anomaly %}} **Profile and tune the real query** - with [mcp-vmanomaly](#mcp-tools-server) connected, Copilot can inspect bounded time-series characteristics, recommend an online model, start an asynchronous autotune task, and apply its validated query and model suggestions.
### How it works
@@ -213,7 +212,7 @@ AI Assistant is disabled by default; enable it with `VMANOMALY_COPILOT_ENABLED=t
Supported providers and model formats:
- **Anthropic** - set `ANTHROPIC_API_KEY`; model format: `anthropic:<model>`
- Examples: `claude-haiku-4-5`, `claude-sonnet-5`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
- Examples: `claude-haiku-4-5`, `claude-sonnet-4-6`; see [full list](https://platform.claude.com/docs/en/about-claude/models/overview#latest-models-comparison)
- **OpenAI** - set `OPENAI_API_KEY`; model format: `openai:<model>` or `openai-responses:<model>`
- Examples: `gpt-5-mini`, `gpt-5.2`; see [full list](https://platform.openai.com/docs/models)
- {{% available_from "v1.29.1" anomaly %}} OpenAI-compatible non-OpenAI providers are supported through `OPENAI_BASE_URL` + `OPENAI_API_KEY`
@@ -316,7 +315,7 @@ docker run -it --rm \
-e VMANOMALY_MCP_SERVER_URL=http://mcp-vmanomaly:8081/mcp \
-p 8080:8080 \
-p 8490:8490 \
victoriametrics/vmanomaly:v1.30.0 \
victoriametrics/vmanomaly:v1.29.7 \
vmanomaly_config.yaml
```
@@ -641,29 +640,6 @@ If the **results** look good and the **model configuration should be deployed in
## Changelog
### v1.8.0
Released: 23-07-2026
vmanomaly version: [v1.30.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1300)
- FEATURE: UX Improvements, refreshed interface with clearer model and query forms, inline configuration previews, and fullscreen chart controls.
- IMPROVEMENT: Boosted [AI Copilot](#ai-assistance) stability and suggestions quality, aligned with MCP/skills toolset, and proper handling of canceled or incomplete tool calls.
- Server (production-running) models and queries can be accessed and selected from the UI, with a new "Queries" button, while model selection now includes a drop-down for server-configured scheduled models in model wizard.
- IMPROVEMENT: Added query prettification and stable value formatting, including platform-aware keyboard-shortcut hints.
- IMPROVEMENT: Results now visibly switch to an out-of-date state after the query, time range, or model configuration changes. The action updates to rerun detection and remains visible in both expanded and collapsed model views.
- IMPROVEMENT: Chart range navigation commits one query on interaction completion instead of issuing many intermediate requests.
- IMPROVEMENT: Compact consecutive identical [AI Copilot](#ai-assistance) tool calls, keep query/model/anomaly suggestions synchronized, and recover cleanly from canceled or incomplete tool calls.
- BUGFIX: Fixed exact UI backtesting across multiple fit cycles and for auto-tuned online wrappers, preventing *duplicate or missing predictions* while retaining causal model updates and restoration of compatible legacy auto-tuned state.
- BUGFIX: Kept automatic trailing-slash redirects for configured path prefixes and `/vmui` relative to the public origin, preventing internal backend hostnames from leaking through reverse proxies such as `vmauth`.
### v1.7.2
Released: 2026-06-25

View File

@@ -551,6 +551,7 @@ Main differences between offline and online:
**Limitations**:
- Until the online model sees enough data (especially if the data shows strong seasonality), its predictions might be unstable, producing more [false positives](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-positive) (`anomaly_score > 1`) or making [false negative predictions, skipping real anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#false-negative).
- Not all models (e.g., complex ones like [Prophet](#prophet)) have a direct online alternative, thus their applicability can be somewhat limited.
Each of the ([built-in](#built-in-models) or [custom](#custom-model-guide)) online models (like [`OnlineZscoreModel`](#online-z-score)) shares the following common parameters and properties:
- `n_samples_seen_` (int) - this model *property* refers to the number of datapoints the model was trained on and increases from 0 (before the first `fit`) with each consecutive `infer` call.
@@ -567,24 +568,23 @@ Every other model that isn't [online](#online-models). Offline models are comple
### Overview
Built-in models support 2 groups of arguments:
- **Model-specific arguments** - please refer to the collapsed *Model-specific arguments* section for each model below.
- **`vmanomaly`-specific** arguments - please refer to *Parameters specific for vmanomaly* and *Default model parameters* subsections for each of the models below.
- Arguments to **inner model** (say, [Facebook's Prophet](https://facebook.github.io/prophet/docs/quick_start#python-api)), passed inside `args` argument as key-value pairs, that will be directly given to the model during initialization to allow granular control. Optional.
> For users who may not be familiar with Python data types such as `list[dict]`, a [dictionary](https://www.w3schools.com/python/python_dictionaries.asp) in Python is a data structure that stores data values in key-value pairs. This structure allows for efficient data retrieval and management.
**Models**:
- [AutoTuned](#autotuned) - designed to take the cognitive load off the user, allowing any of built-in models below to be re-tuned for best hyperparameters on data seen during each `fit` phase of the algorithm. Tradeoff is between increased computational time and optimized results / simpler maintenance.
- [Temporal Envelope](#temporal-envelope) - the preferred **online model for complex operational data** with trends, changepoints, multiple calendar patterns, holidays, capable of [forecasting](https://docs.victoriametrics.com/anomaly-detection/components/faq/#forecasting). Its multivariate form also learns cross-series relationships.
- [Prophet](#prophet) - an offline forecasting alternative when Prophet-specific decomposition outputs are required. Favor `Temporal Envelope` for online adaptation and multivariate support.
- [Online Z-score](#online-z-score) - useful for initial testing and for simpler data ([de-trended](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) data without strict [seasonality](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality) and with anomalies of similar magnitude as your "normal" data)
- [MAD](#online-mad) - similarly to [Z-score](#online-z-score), is effective for **identifying outliers in relatively consistent data**. Useful for detecting sudden, stark deviations from the median, being less prone to outlier's magnitude than z-score.
- [Rolling Quantile](#rolling-quantile) - best for **data with evolving patterns**, as it adapts to changes over a rolling window.
- [Online Seasonal Quantile](#online-seasonal-quantile) - best used on **[de-trended](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) data with strong (possibly multiple) [seasonalities](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality)**. Can act as a (slightly less powerful) [online](#online-models) replacement to [`ProphetModel`](#prophet).
- [Seasonal Trend Decomposition](#seasonal-trend-decomposition) - similarly to Holt-Winters, is best for **data with pronounced [seasonal](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality) and [trend](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) components**
- [Isolation forest (Multivariate)](#isolation-forest-multivariate) - an offline alternative for **metrics data interaction** (several queries/metrics -> single anomaly score) and high-dimensional feature-space outliers. Prefer multivariate Temporal Envelope when temporal profiles and *online* adaptation matter.
- [Holt-Winters](#holt-winters) - well-suited for **data with moderate complexity**, exhibiting distinct [trends](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) and/or [single seasonal pattern](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality).
- [Custom model](#custom-model-guide) - benefit from your own models and expertise to better support your **unique use case**.
* [AutoTuned](#autotuned) - designed to take the cognitive load off the user, allowing any of built-in models below to be re-tuned for best hyperparameters on data seen during each `fit` phase of the algorithm. Tradeoff is between increased computational time and optimized results / simpler maintenance.
* [Prophet](#prophet) - the most versatile one for production usage, especially for complex data ([trends](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend), [change points](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#novelties), [multi-seasonality](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality))
* [Online Z-score](#online-z-score) - useful for initial testing and for simpler data ([de-trended](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) data without strict [seasonality](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality) and with anomalies of similar magnitude as your "normal" data)
* [MAD](#online-mad) - similarly to [Z-score](#online-z-score), is effective for **identifying outliers in relatively consistent data**. Useful for detecting sudden, stark deviations from the median, being less prone to outlier's magnitude than z-score.
* [Rolling Quantile](#rolling-quantile) - best for **data with evolving patterns**, as it adapts to changes over a rolling window.
* [Online Seasonal Quantile](#online-seasonal-quantile) - best used on **[de-trended](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) data with strong (possibly multiple) [seasonalities](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality)**. Can act as a (slightly less powerful) [online](#online-models) replacement to [`ProphetModel`](#prophet).
* [Seasonal Trend Decomposition](#seasonal-trend-decomposition) - similarly to Holt-Winters, is best for **data with pronounced [seasonal](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality) and [trend](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) components**
* [Isolation forest (Multivariate)](#isolation-forest-multivariate) - useful for **metrics data interaction** (several queries/metrics -> single anomaly score) and **efficient in detecting anomalies in high-dimensional datasets**
* [Holt-Winters](#holt-winters) - well-suited for **data with moderate complexity**, exhibiting distinct [trends](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) and/or [single seasonal pattern](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality).
* [Custom model](#custom-model-guide) - benefit from your own models and expertise to better support your **unique use case**.
### AutoTuned
@@ -613,13 +613,11 @@ Tuning [hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_(machine_l
> ```
> will produce **one model per each timeseries** returned by `your_query`, with **the same** hyperparameters, such as `z_threshold`, but different parameters, such as mean, std, etc.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
- `class` (string) - model class name `"model.auto.AutoTunedModel"` (or `auto` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `tuned_class_name` (string) - [Built-in model class](#built-in-models) to wrap, i.e. `zscore_online`
- `optimization_params` (dict) - Optimization parameters for *unsupervised* model tuning. Control percentage of found anomalies, as well as a tradeoff between time spent and the accuracy. The higher `timeout` and `n_trials` are, the better model configuration can be found for `tuned_class_name`, but the longer it takes and vice versa. Set `n_jobs` to `-1` to use all the CPUs available, it makes sense if only you have a big dataset to train on during `fit` calls, otherwise overhead isn't worth it.
* `class` (string) - model class name `"model.auto.AutoTunedModel"` (or `auto` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `tuned_class_name` (string) - [Built-in model class](#built-in-models) to wrap, i.e. `zscore_online`
* `optimization_params` (dict) - Optimization parameters for *unsupervised* model tuning. Control percentage of found anomalies, as well as a tradeoff between time spent and the accuracy. The higher `timeout` and `n_trials` are, the better model configuration can be found for `tuned_class_name`, but the longer it takes and vice versa. Set `n_jobs` to `-1` to use all the CPUs available, it makes sense if only you have a big dataset to train on during `fit` calls, otherwise overhead isn't worth it.
- `anomaly_percentage` (float) - Expected percentage of anomalies that can be seen in training data, from `[0, 0.5)` interval (i.e. 0.01 means it's expected ~ 1% of anomalies to be present in training data). This is a *required* parameter.
- `optimized_business_params` (list[string]) - {{% available_from "v1.15.0" anomaly %}} this argument allows particular [business-specific parameters](#common-args) such as [`detection_direction`](https://docs.victoriametrics.com/anomaly-detection/components/models/#detection-direction) or [`min_dev_from_expected`](https://docs.victoriametrics.com/anomaly-detection/components/models/#minimal-deviation-from-expected) to remain **unchanged during optimizations, retaining their initial values**. I.e. setting `optimized_business_params` to `['detection_direction']` will allow to optimize only `detection_direction` business-specific arg, while `min_dev_from_expected` will retain its default value of (e.g. [1, 2] if set to that value in model config). By default and if not set, will be equal to `[]` (empty list), meaning no business params will be optimized. **A recommended option is to leave it empty** as this feature is still experimental and may lead to unexpected results.
- `seed` (int) - Random seed for reproducibility and deterministic nature of underlying optimizations.
@@ -628,13 +626,9 @@ Tuning [hyperparameters](https://en.wikipedia.org/wiki/Hyperparameter_(machine_l
- `train_val_ratio` (float) - {{% available_from "v1.25.1" anomaly %}} the ratio of training to validation data size when constructing folds, e.g. setting it to 2 will result in 2/3 of the data being used for training and 1/3 for validation in each of the splits. Defaults to 3 (3/4 of the data for training and 1/4 for validation). Isn't used when `validation_scheme` is set to `leaky`.
- `n_trials` (int) - How many trials to sample from hyperparameter search space. The higher, the longer it takes but the better the results can be. Defaults to 128.
- `timeout` (float) - How many seconds in total can be spent on each model to tune hyperparameters. The higher, the longer it takes, allowing to test more trials out of defined `n_trials`, but the better the results can be.
- `exact` (boolean) - {{% available_from "v1.30.0" anomaly %}} evaluate online models causally during validation: predict from the current state, then update it with newly observed data. Set this to `true` when production uses exact online inference. Defaults to `false` for backward compatibility.
- `optimize_complexity` (boolean) - {{% available_from "v1.30.0" anomaly %}} prefer a smaller fitted model state when validation quality is effectively tied. Defaults to `true`.
- `frozen_params` (dict) - {{% available_from "v1.30.0" anomaly %}} model-specific parameters that **must remain fixed** while the remaining search space is tuned. Nested dictionaries are merged recursively. Use this for non-optimizable context such as configured holidays or multivariate `groupby`; `class` and `class_name` cannot be frozen.
{{% /collapse %}}
![vmanomaly-autotune-schema](autotune.webp)
{{% collapse name="Configuration example" %}}
```yaml
# ...
@@ -653,153 +647,15 @@ models:
timeout: 10 # how many seconds to spend on optimization for each trained model during `fit` phase call
n_jobs: 1 # how many jobs in parallel to launch. Consider making it > 1 only if you have fit window containing > 10000 datapoints for each series
optimized_business_params: [] # business-specific params to include in optimization, if not set - defaults to empty list, meaning no business params will be optimized, which is a recommended option as business arguments are better set by stakeholders rather than algorithms
exact: true # causally validate online models the same way they run in production
optimize_complexity: true # prefer smaller fitted state when quality is tied
frozen_params:
history_strength: 2
# ...
```
{{% /collapse %}}
</div>
![vmanomaly-autotune-schema](autotune.webp)
#### Shared asynchronous autotune workflow
{{% available_from "v1.30.0" anomaly %}} Agents, the UI, and external automation can tune one shared configuration across a bounded sample of query results without adding an `auto` wrapper to the production configuration:
1. Inspect the query with `GET /api/v1/timeseries/characteristics` to identify trend, calendar seasonality, changepoints, gaps, and intermittent behavior.
2. Start a task with `POST /api/v1/autotune/tasks`. Provide the actual query, the candidate model class, the same query `step` used in production, and a bounded `limit`.
3. Poll `GET /api/v1/autotune/tasks/{task_id}` until `status` is `done`; cancel unnecessary work with `DELETE` on the same path.
4. Validate and deploy `result_data.data.modelConfig`, which is a concrete configuration for the selected model class.
> [!TIP]
> Use [skills](https://docs.victoriametrics.com/ai-tools/#agent-skills) where abovementioned workflow is automated. Also, [AI Copilot](https://docs.victoriametrics.com/anomaly-detection/ui/#ai-assistant) can generate a tuned model configuration to interactively backtest in UI, based on the query data characteristics and user's anomaly expectations.
Example request for an online model:
```json
{
"query": "sum(rate(http_requests_total[5m])) by (service)",
"tuned_class_name": "temporal_envelope",
"anomaly_percentage": 0.01,
"step": "5m",
"limit": 100,
"use_profile_hints": true,
"optimization_params": {
"exact": true,
"n_splits": 3,
"n_trials": 64,
"timeout": 60,
"optimize_complexity": true
},
"frozen_params": {
"holidays": {"countries": ["US"], "group": true}
}
}
```
The requested anomaly percentage is treated as an alert-volume constraint rather than a target that must be reached in every validation fold. Model-specific search ranges may be narrowed using the sampled characteristics, while `frozen_params` preserves operator-supplied context. For online models, `exact: true` usually gives the most representative choice when production inference is causal.
> There are some expected limitations of Autotune mode:
> - It can't be made on your [custom model](#custom-model-guide).
> - It can't be applied to itself (like `tuned_class_name: 'model.auto.AutoTunedModel'`)
> - `AutoTunedModel` can't be used on [rolling models](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-models) like [`RollingQuantile`](https://docs.victoriametrics.com/anomaly-detection/components/models/#rolling-quantile) in combination with [on-disk model storage mode](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode), as the rolling models exists only during `infer` calls and aren't persisted neither in RAM, nor on disk.
### Temporal Envelope
{{% available_from "v1.30.0" anomaly %}} Temporal Envelope is the preferred online model for complex operational and business metrics. It learns an evolving expected range from robust trend, calendar and holiday patterns, persistent level shifts, uncertainty, and optional future forecasts. The model adapts during inference while limiting the lasting influence of short-lived spikes.
> `TemporalEnvelopeModel` is [univariate](#univariate-models) and [online](#online-models). `TemporalEnvelopeMultivariateModel` also learns normal cross-series relationships as a [multivariate](#multivariate-models) model.
Use it for:
- infrastructure, application, or business metrics with trends and daily, weekly, monthly, or holiday behavior;
- deployments, traffic migrations, and capacity changes that create persistent shifts, including short-horizon forecasts through `forecast_at`;
- aligned related metrics where each channel keeps its own temporal pattern while their joint behavior contributes to one anomaly score.
For simple profiles without strong trend or seasonality, prefer [Online MAD](#online-mad) or [Online Z-score](#online-z-score). [Prophet](#prophet) and [Isolation Forest](#isolation-forest-multivariate) remain offline alternatives when their distinct capabilities are required or validation favors them.
<div class="model-details">
{{% collapse name="Model-specific arguments" %}}
- `class` (string) - `temporal_envelope` or the full class name `model.online.TemporalEnvelopeModel`.
- `quantiles` (list[float], default `[0.25, 0.75]`) - two ordered probabilities used as the base lower and upper residual boundaries. Values must be within `[0, 1]`.
- `iqr_threshold` (float, default `2.0`) - primary data-derived interval-width control. Higher values widen the ordinary residual envelope and produce fewer detections. Values from `1` to `4` are a practical experimentation range.
- `alpha` (float, default `0.005`) - **Trend Reactivity**. Higher values follow persistent level and trend changes faster; lower values produce steadier expectations. Values from `0.0025` to `0.02` are a practical experimentation range.
- `loss_reactivity` (float, default `5.0`) - controls how readily new deviations influence the model. Higher values adapt more readily; lower values reduce the influence of spikes. Values from `1` to `5` are a good starting range.
- `min_n_samples_seen` (integer, default `16`) - observations each model or group must see before anomaly scores are trusted. State and forecasts continue learning during warmup, while anomaly scores remain zero. Approximate warmup duration is this value multiplied by the query step.
- `changepoint_window` (integer, default `16`) - consecutive same-direction residuals required before treating a deviation as a persistent level shift. Use a value longer than ordinary anomaly bursts; approximate confirmation time is this value multiplied by the query step.
- `seasonalities` (list[string], default `['hod_smooth', 'dow_smooth']`) - recurring calendar patterns to model. Use an empty list when calendar patterns are known to be irrelevant. Supported presets are:
- hour of day: `hod_smooth`, `hod_spiky`, `hod_plateau`;
- day of week: `dow_smooth`, `dow_spiky`, `dow_plateau`;
- weekday versus weekend: `weekpart_plateau`;
- month of year: `month_smooth`, `month_plateau`.
- `holidays` (object, default `{}`) - known country holidays and special events. Supported keys are `countries` (country codes or names), `special_events` (recurring `MM-DD` or one-off `YYYY-MM-DD` dates), and `group` (whether configured events share one learned response). Holiday response shape is learned from data rather than configured publicly.
- `forecast_at` (list[string], default `[]`) - positive future offsets such as `['1h', '1d']`. Each offset adds point and, when requested through `provide_series`, lower and upper forecast series.
Preset suffixes describe expected profile shape: `smooth` represents gradual recurring curves, `spiky` represents narrow phase peaks, and `plateau` represents sustained calendar levels. Choose only profiles supported by the data. Calendar and holiday features use civil time from the configured query timezone, so hour/day profiles remain aligned across daylight-saving-time transitions.
Temporal Envelope also supports the [common model arguments](#common-args), including `queries`, `schedulers`, `provide_series`, `detection_direction`, `scale`, `clip_predictions`, `min_dev_from_expected`, and `min_rel_dev_from_expected`. Input `data_range` and query timezone are configured on the [reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#config-parameters).
The multivariate variant uses `class: temporal_envelope_multivariate` or `model.online.TemporalEnvelopeMultivariateModel` and adds:
- `dependency_rank` (integer, default `8`) - compact rank used to represent cross-channel dependencies. Set it to `0` to disable dependency features.
- `score_aggregation` (`max`, `mean`, or `l2`; default `l2`) - combines per-channel boundary deviations into one anomaly score. `l2` uses their root-mean-square magnitude.
- `groupby` (list[string], optional) - metric label names used to fit a separate multivariate model for each unique label-value combination.
- `max_channels` (integer, default `1000`) - hard safety limit for input time series in one model instance.
- `recommended_max_channels` (integer, default `100`) - advisory limit; larger inputs up to `max_channels` are accepted with a warning because high-dimensional estimates may be less stable and more expensive.
- `random_state` (integer, default `42`) - seed used for reproducible dependency features.
The multivariate model emits only the joint `anomaly_score` by default to keep output cardinality bounded. Explicitly request channel-level `y`, `yhat`, `yhat_lower`, and `yhat_upper` families through `provide_series` only when those diagnostics are needed and the downstream cardinality is acceptable.
{{% /collapse %}}
{{% collapse name="Configuration examples" %}}
Univariate monitoring with calendar and holiday behavior:
```yaml
models:
operations_envelope:
class: temporal_envelope
queries: [request_rate, latency]
seasonalities: [hod_smooth, dow_smooth, weekpart_plateau]
alpha: 0.005
loss_reactivity: 5
iqr_threshold: 2
changepoint_window: 16
holidays:
countries: [US]
special_events: ['12-24']
group: true
forecast_at: [1h, 1d]
provide_series: [anomaly_score, yhat, yhat_lower, yhat_upper]
```
Multivariate monitoring with one dependency model per cluster:
```yaml
models:
service_dependency_envelope:
class: temporal_envelope_multivariate
queries: [request_rate, error_rate, latency]
groupby: [cluster]
dependency_rank: 8
score_aggregation: l2
seasonalities: [hod_smooth, dow_smooth]
provide_series: [anomaly_score]
```
For independent per-series detection, use `temporal_envelope`. Use `temporal_envelope_multivariate` only when the relationship between aligned series is itself meaningful; each channel still learns its own trend and calendar patterns.
{{% /collapse %}}
</div>
### [Prophet](https://facebook.github.io/prophet/)
`vmanomaly` uses the Facebook Prophet implementation for time series forecasting, with detailed usage provided in the [Prophet library documentation](https://facebook.github.io/prophet/docs/quick_start#python-api). All original Prophet parameters are supported and can be directly passed to the model via `args` argument.
@@ -807,9 +663,7 @@ For independent per-series detection, use `temporal_envelope`. Use `temporal_env
> {{% available_from "v1.25.3" anomaly %}} Producing forecasts for future timestamps is now supported. To enable this, set the `forecast_at` argument to a list of relative future offsets (e.g., `['1h', '1d']`). The model will then generate forecasts for these future timestamps, which can be useful for planning and resource allocation. Output series are affected by [provide_series](#provide-series) argument, which need to include at least `yhat` for point-wise forecasts (and `yhat_lower` or/and `yhat_upper` for respective confidence intervals). See the example below for more details.
<div class="model-details">
{{% collapse name="Model-specific arguments" %}}
*Parameters specific for vmanomaly*:
- `class` (string) - model class name `"model.prophet.ProphetModel"` (or `prophet` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `seasonalities` (list[dict], optional): Additional seasonal components to include in Prophet. See Prophets [`add_seasonality()`](https://facebook.github.io/prophet/docs/seasonality,_holiday_effects,_and_regressors#modeling-holidays-and-special-events:~:text=modeling%20the%20cycle-,Specifying,-Custom%20Seasonalities) documentation for details.
@@ -827,9 +681,19 @@ For independent per-series detection, use `temporal_envelope`. Use `temporal_env
- `agg_method` (str, optional, default="mean"): Aggregation function to apply within each window. Supported values: "mean", "median".
- `adjust_boundaries` (bool, optional, default=true): Whether to adjust confidence interval boundaries after downsampling. If true, `yhat_lower` and `yhat_upper` will be adjusted based on the aggregated vs original data variability.
{{% /collapse %}}
> Apart from standard [`vmanomaly` output](#vmanomaly-output), Prophet model can provide additional metrics.
{{% collapse name="Configuration examples" %}}
**Additional output metrics produced by FB Prophet**
Depending on chosen `seasonality` parameter FB Prophet can return additional metrics such as:
- `trend`, `trend_lower`, `trend_upper`
- `additive_terms`, `additive_terms_lower`, `additive_terms_upper`,
- `multiplicative_terms`, `multiplicative_terms_lower`, `multiplicative_terms_upper`,
- `daily`, `daily_lower`, `daily_upper`,
- `hourly`, `hourly_lower`, `hourly_upper`,
- `holidays`, `holidays_lower`, `holidays_upper`,
- and a number of columns for each holiday if `holidays` param is set
*Config Example*
Timezone-unaware example:
@@ -902,21 +766,6 @@ models:
country_holidays: 'US'
```
{{% /collapse %}}
</div>
> Apart from standard [`vmanomaly` output](#vmanomaly-output), Prophet model can provide additional metrics.
**Additional output metrics produced by FB Prophet**
Depending on chosen `seasonality` parameter FB Prophet can return additional metrics such as:
- `trend`, `trend_lower`, `trend_upper`
- `additive_terms`, `additive_terms_lower`, `additive_terms_upper`,
- `multiplicative_terms`, `multiplicative_terms_lower`, `multiplicative_terms_upper`,
- `daily`, `daily_lower`, `daily_upper`,
- `hourly`, `hourly_lower`, `hourly_upper`,
- `holidays`, `holidays_lower`, `holidays_upper`,
- and a number of columns for each holiday if `holidays` param is set
Resulting metrics of the model are described [here](#vmanomaly-output)
@@ -926,18 +775,13 @@ Resulting metrics of the model are described [here](#vmanomaly-output)
Online version of existing [Z-score](#z-score) implementation with the same exact behavior and implications {{% available_from "v1.15.0" anomaly %}}.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.online.OnlineZscoreModel"` (or `zscore_online`with class alias support{{% available_from "v1.13.0" anomaly %}})
* `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculation boundaries and anomaly score. Defaults to `2.5`.
* `min_n_samples_seen` (int, optional) - the minimum number of samples to be seen (`n_samples_seen_` property) before computing the anomaly score. Otherwise, the **anomaly score will be 0**, as there is not enough data to trust the model's predictions. Defaults to 16.
- `class` (string) - model class name `"model.online.OnlineZscoreModel"` (or `zscore_online`with class alias support{{% available_from "v1.13.0" anomaly %}})
- `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculation boundaries and anomaly score. Defaults to `2.5`.
- `min_n_samples_seen` (int, optional) - the minimum number of samples to be seen (`n_samples_seen_` property) before computing the anomaly score. Otherwise, the **anomaly score will be 0**, as there is not enough data to trust the model's predictions. Defaults to 16.
- `history_strength` (float, optional) - {{% available_from "v1.30.0" anomaly %}} strength of the initial history learned by `fit`. Values above `1` keep fitted mean and variance unchanged initially but reduce the leverage of subsequent updates. Defaults to `1`.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
models:
@@ -945,7 +789,6 @@ models:
class: "zscore_online" # or 'model.online.OnlineZscoreModel'
z_threshold: 3.5
min_n_samples_seen: 128 # i.e. calculate it as full seasonality / data freq
history_strength: 2 # retain fitted history as a stronger prior
provide_series: ['anomaly_score', 'yhat'] # common arg example
# Common arguments for built-in model, if not set, default to
# See https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args
@@ -960,10 +803,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
@@ -973,19 +812,14 @@ Resulting metrics of the model are described [here](#vmanomaly-output).
The MAD model is a robust method for anomaly detection that is *less sensitive* to outliers in data compared to standard deviation-based models. It considers a point as an anomaly if the absolute deviation from the median is significantly large. This is the online approximate version, based on [t-digests](https://www.sciencedirect.com/science/article/pii/S2665963820300403) for online quantile estimation{{% available_from "v1.15.0" anomaly %}}.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.online.OnlineMADModel"` (or `mad_online` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `threshold` (float, optional) - The threshold multiplier for the MAD to determine anomalies. Defaults to `2.5`. Higher values will identify fewer points as anomalies.
* `min_n_samples_seen` (int, optional) - the minimum number of samples to be seen (`n_samples_seen_` property) before computing the anomaly score. Otherwise, the **anomaly score will be 0**, as there is not enough data to trust the model's predictions. Defaults to 16.
* `compression` (int, optional) - the compression parameter for underlying [t-digest](https://www.sciencedirect.com/science/article/pii/S2665963820300403). Higher values mean higher accuracy but higher memory usage. By default 100.
- `class` (string) - model class name `"model.online.OnlineMADModel"` (or `mad_online` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `threshold` (float, optional) - The threshold multiplier for the MAD to determine anomalies. Defaults to `2.5`. Higher values will identify fewer points as anomalies.
- `min_n_samples_seen` (int, optional) - the minimum number of samples to be seen (`n_samples_seen_` property) before computing the anomaly score. Otherwise, the **anomaly score will be 0**, as there is not enough data to trust the model's predictions. Defaults to 16.
- `history_strength` (float, optional) - {{% available_from "v1.30.0" anomaly %}} strength of the initial history learned by `fit`. Values above `1` preserve fitted quantiles initially but reduce the leverage of subsequent updates. Defaults to `1`.
- `compression` (int, optional) - the compression parameter for underlying [t-digest](https://www.sciencedirect.com/science/article/pii/S2665963820300403). Higher values mean higher accuracy but higher memory usage. By default 100.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
@@ -994,7 +828,6 @@ models:
class: "mad_online" # or 'model.online.OnlineMADModel'
threshold: 2.5
min_n_samples_seen: 128 # i.e. calculate it as full seasonality / data freq
history_strength: 2 # retain fitted history as a stronger prior
compression: 100 # higher values mean higher accuracy but higher memory usage
provide_series: ['anomaly_score', 'yhat'] # common arg example
# Common arguments for built-in model, if not set, default to
@@ -1010,10 +843,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
@@ -1023,18 +852,13 @@ Resulting metrics of the model are described [here](#vmanomaly-output).
This model is best used on **data with short evolving patterns** (i.e. 10-100 datapoints of particular frequency), as it adapts to changes over a rolling window.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.rolling_quantile.RollingQuantileModel"` (or `rolling_quantile` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `quantile` (float) - quantile value, from 0.5 to 1.0. This constraint is implied by 2-sided confidence interval.
* `window_steps` (integer) - size of the moving window. (see 'sampling_period')
- `class` (string) - model class name `"model.rolling_quantile.RollingQuantileModel"` (or `rolling_quantile` with class alias support {{% available_from "v1.13.0" anomaly %}})
- `quantile` (float) - quantile value, from 0.5 to 1.0. This constraint is implied by 2-sided confidence interval.
- `window_steps` (integer) - size of the moving window. (see 'sampling_period')
- `iqr_threshold` (float, optional) - {{% available_from "v1.25.0" anomaly %}} The [interquartile range (IQR)](https://en.wikipedia.org/wiki/Interquartile_range) multiplier to increase the width of the prediction intervals. Defaults to 0 (no adjustment) for backward compatibility. If set > 0, the model will add half IQR * `iqr_threshold` to `yhat_lower` and `yhat_upper`. This is useful for data with high variance or outliers, as it helps to avoid false positives in anomaly detection.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
models:
@@ -1042,7 +866,6 @@ models:
class: "rolling_quantile"
quantile: 0.9
window_steps: 96
iqr_threshold: 1
# Common arguments for built-in model, if not set, default to
# See https://docs.victoriametrics.com/anomaly-detection/components/models/#common-args
#
@@ -1056,10 +879,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
@@ -1073,26 +892,21 @@ Best used on **[de-trended](https://victoriametrics.com/blog/victoriametrics-ano
It uses the `quantiles` triplet to calculate `yhat_lower`, `yhat`, and `yhat_upper` [output](#vmanomaly-output), respectively, for each of the `min_subseasons` sub-intervals contained in `seasonal_interval`. For example, with '4d' + '2h' seasonality patterns (multiple), it will hold and update 24*4 / 2 = 48 consecutive estimates (each 2 hours long).
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.online.OnlineQuantileModel"` (or `quantile_online` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `quantiles` (list[float], optional) - The quantiles to estimate. `yhat_lower`, `yhat`, `yhat_upper` are the quantile order. By default (0.01, 0.5, 0.99).
* `iqr_threshold` (float, optional) - {{% available_from "v1.25.0" anomaly %}} The [interquartile range (IQR)](https://en.wikipedia.org/wiki/Interquartile_range) multiplier to increase the width of the prediction intervals. Defaults to 0 (no adjustment) for backward compatibility. If set > 0, the model will add IQR * `iqr_threshold` to `yhat_lower` and `yhat_upper` (respecting `min_subseason` seasonal buckets). This is useful for data with high variance or outliers, as it helps to avoid false positives in anomaly detection. Best used with **robust** `quantiles` set to (0.25, 0.5, 0.75) or similar.
* `seasonal_interval` (string, optional) - the interval for the seasonal adjustment. If not set, the model will equal to a simple online quantile model. By default not set.
* `min_subseason` (str, optional) - the minimum interval to estimate quantiles for. By default not set. Note that the minimum interval should be a multiple of the seasonal interval, i.e. if seasonal_interval='2h', then min_subseason='15m' is valid, but '37m' is not.
* `use_transform` (bool, optional) - whether to internally apply a `log1p(abs(x)) * sign(x)` transformation to the data to stabilize internal quantile estimation. Does not affect the scale of produced output (i.e. `yhat`) By default False.
* `global_smoothing` (float, optional) - the smoothing parameter for the global quantiles. i.e. the output is a weighted average of the global and seasonal quantiles (if `seasonal_interval` and `min_subseason` args are set). Should be from `[0, 1]` interval, where 0 means no smoothing and 1 means using only global quantile values.
* `scale` (float, optional) - Is used to adjust the margins between `yhat` and [`yhat_lower`, `yhat_upper`]. New margin = `|yhat_* - yhat_lower| * scale`. Defaults to 1 (no scaling is applied). See `scale`[common arg](https://docs.victoriametrics.com/anomaly-detection/components/models/#scale) section for detailed instructions and 2-sided option.
* `season_starts_from` (str, optional) - the start date for the seasonal adjustment, as a reference point to start counting the intervals. By default '1970-01-01'.
* `min_n_samples_seen` (int, optional) - the minimum number of samples to be seen (`n_samples_seen_` property) before computing the anomaly score. Otherwise, the **anomaly score will be 0**, as there is not enough data to trust the model's predictions. Defaults to 16.
* `compression` (int, optional) - the compression parameter for the underlying [t-digests](https://www.sciencedirect.com/science/article/pii/S2665963820300403). Higher values mean higher accuracy but higher memory usage. By default 100.
- `class` (string) - model class name `"model.online.OnlineQuantileModel"` (or `quantile_online` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `quantiles` (list[float], optional) - The quantiles to estimate. `yhat_lower`, `yhat`, `yhat_upper` are the quantile order. By default (0.01, 0.5, 0.99).
- `iqr_threshold` (float, optional) - {{% available_from "v1.25.0" anomaly %}} The [interquartile range (IQR)](https://en.wikipedia.org/wiki/Interquartile_range) multiplier to increase the width of the prediction intervals. Defaults to 0 (no adjustment) for backward compatibility. If set > 0, the model will add IQR * `iqr_threshold` to `yhat_lower` and `yhat_upper` (respecting `min_subseason` seasonal buckets). This is useful for data with high variance or outliers, as it helps to avoid false positives in anomaly detection. Best used with **robust** `quantiles` set to (0.25, 0.5, 0.75) or similar.
- `seasonal_interval` (string, optional) - the interval for the seasonal adjustment. If not set, the model will equal to a simple online quantile model. By default not set.
- `min_subseason` (str, optional) - the minimum interval to estimate quantiles for. By default not set. Note that the minimum interval should be a multiple of the seasonal interval, i.e. if seasonal_interval='2h', then min_subseason='15m' is valid, but '37m' is not.
- `use_transform` (bool, optional) - whether to internally apply a `log1p(abs(x)) * sign(x)` transformation to the data to stabilize internal quantile estimation. Does not affect the scale of produced output (i.e. `yhat`) By default False.
- `global_smoothing` (float, optional) - the smoothing parameter for the global quantiles. i.e. the output is a weighted average of the global and seasonal quantiles (if `seasonal_interval` and `min_subseason` args are set). Should be from `[0, 1]` interval, where 0 means no smoothing and 1 means using only global quantile values.
- `scale` (float, optional) - Is used to adjust the margins between `yhat` and [`yhat_lower`, `yhat_upper`]. New margin = `|yhat_* - yhat_lower| * scale`. Defaults to 1 (no scaling is applied). See `scale`[common arg](https://docs.victoriametrics.com/anomaly-detection/components/models/#scale) section for detailed instructions and 2-sided option.
- `season_starts_from` (str, optional) - the start date for the seasonal adjustment, as a reference point to start counting the intervals. By default '1970-01-01'.
- `min_n_samples_seen` (int, optional) - the minimum number of samples to be seen (`n_samples_seen_` property) before computing the anomaly score. Otherwise, the **anomaly score will be 0**, as there is not enough data to trust the model's predictions. Defaults to 16.
- `history_strength` (float, optional) - {{% available_from "v1.30.0" anomaly %}} strength of the initial history learned by `fit`. Values above `1` preserve fitted quantiles initially but reduce the leverage of subsequent updates. Defaults to `1`.
- `compression` (int, optional) - the compression parameter for the underlying [t-digests](https://www.sciencedirect.com/science/article/pii/S2665963820300403). Higher values mean higher accuracy but higher memory usage. By default 100.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
Suppose we have a data with strong intra-day (hourly) and intra-week (daily) seasonality, data granularity is '5m' with up to 5% expected outliers present in data. Then you can apply similar config:
@@ -1105,7 +919,6 @@ models:
seasonal_interval: '7d' # longest seasonality (week, day) = week, starting from `season_starts_from`
min_subseason: '1h' # smallest seasonality (week, day, hour) = hour, will have its own quantile estimates
min_n_samples_seen: 288 # 1440 / 5 - at least 1 full day, ideal = 1440 / 5 * 7 - one full week (seasonal_interval)
history_strength: 2 # retain fitted history as a stronger prior
scale: 1.1 # to compensate lowered quantile boundaries with wider intervals
season_starts_from: '2024-01-01' # interval calculation starting point, especially for uncommon seasonalities like '36h' or '12d'
compression: 100 # higher values mean higher accuracy but higher memory usage
@@ -1123,10 +936,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
@@ -1136,17 +945,14 @@ Resulting metrics of the model are described [here](#vmanomaly-output).
Here we use Seasonal Decompose implementation from `statsmodels` [library](https://www.statsmodels.org/dev/generated/statsmodels.tsa.seasonal.seasonal_decompose). Parameters from this library can be passed to the model. Some parameters are specifically predefined in `vmanomaly` and can't be changed by user (`model`='additive', `two_sided`=False).
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.std.StdModel"` (or `std` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `period` (integer) - Number of datapoints in one season.
* `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculating boundaries to define anomaly score. Defaults to `2.5`.
- `class` (string) - model class name `"model.std.StdModel"` (or `std` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `period` (integer) - Number of datapoints in one season.
- `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculating boundaries to define anomaly score. Defaults to `2.5`.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
@@ -1167,16 +973,13 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
**Additional output metrics produced by Seasonal Trend Decomposition model**
- `resid` - The residual component of the data series.
- `trend` - The trend component of the data series.
- `seasonal` - The seasonal component of the data series.
* `resid` - The residual component of the data series.
* `trend` - The trend component of the data series.
* `seasonal` - The seasonal component of the data series.
### [Isolation forest](https://en.wikipedia.org/wiki/Isolation_forest) (Multivariate)
@@ -1191,15 +994,13 @@ Detects anomalies using binary trees. The algorithm has a linear time complexity
Here we use Isolation Forest implementation from `scikit-learn` [library](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest). All parameters from this library can be passed to the model.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.isolation_forest.IsolationForestMultivariateModel"` (or `isolation_forest_multivariate` with class alias support {{% available_from "v1.13.0" anomaly %}})
- `class` (string) - model class name `"model.isolation_forest.IsolationForestMultivariateModel"` (or `isolation_forest_multivariate` with class alias support {{% available_from "v1.13.0" anomaly %}})
* `contamination` (float or string, optional) - The amount of contamination of the data set, i.e. the proportion of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. Default value - "auto". Should be either `"auto"` or be in the range (0.0, 0.5]. {{% available_from "v1.29.5" anomaly %}} Numeric strings, such as `"0.01"`, are accepted, while invalid non-finite values, such as `nan`, `inf`, and `-inf`, are rejected during config validation.
- `contamination` (float or string, optional) - The amount of contamination of the data set, i.e. the proportion of outliers in the data set. Used when fitting to define the threshold on the scores of the samples. Default value - "auto". Should be either `"auto"` or be in the range (0.0, 0.5]. {{% available_from "v1.29.5" anomaly %}} Numeric strings, such as `"0.01"`, are accepted, while invalid non-finite values, such as `nan`, `inf`, and `-inf`, are rejected during config validation.
- `seasonal_features` (list of string) - List of seasonality to encode through [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca), i.e. `dow` (day of week). **Introduced in [1.12.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1120)**.
* `seasonal_features` (list of string) - List of seasonality to encode through [cyclical encoding](https://towardsdatascience.com/cyclical-features-encoding-its-about-time-ce23581845ca), i.e. `dow` (day of week). **Introduced in [1.12.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1120)**.
- Empty by default for backward compatibility.
- Example: `seasonal_features: ['dow', 'hod']`.
- Supported seasonalities:
@@ -1208,11 +1009,9 @@ Here we use Isolation Forest implementation from `scikit-learn` [library](https:
- "dow" - day of week (1-7)
- "month" - month of year (1-12)
- `args` (dict, optional) - Inner model args (key-value pairs). See accepted params in [model documentation](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest). Defaults to empty (not provided). Example: {"random_state": 42, "n_estimators": 100}
* `args` (dict, optional) - Inner model args (key-value pairs). See accepted params in [model documentation](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest). Defaults to empty (not provided). Example: {"random_state": 42, "n_estimators": 100}
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
@@ -1236,10 +1035,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
### [Holt-Winters](https://en.wikipedia.org/wiki/Exponential_smoothing)
@@ -1248,33 +1043,29 @@ Resulting metrics of the model are described [here](#vmanomaly-output).
Here we use Holt-Winters Exponential Smoothing implementation from `statsmodels` [library](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing). All parameters from this library can be passed to the model.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.holtwinters.HoltWinters"` (or `holtwinters` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `class` (string) - model class name `"model.holtwinters.HoltWinters"` (or `holtwinters` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `frequency` (string) - Must be set equal to sampling_period. Model needs to know expected data-points frequency (e.g. '10m'). If omitted, frequency is guessed during fitting as **the median of intervals between fitting data timestamps**. During inference, if incoming data doesn't have the same frequency, then it will be interpolated. E.g. data comes at 15 seconds resolution, and our resample_freq is '1m'. Then fitting data will be downsampled to '1m' and internal model is trained at '1m' intervals. So, during inference, prediction data would be produced at '1m' intervals, but interpolated to "15s" to match with expected output, as output data must have the same timestamps. As accepted by pandas.Timedelta (e.g. '5m').
- `frequency` (string) - Must be set equal to sampling_period. Model needs to know expected data-points frequency (e.g. '10m'). If omitted, frequency is guessed during fitting as **the median of intervals between fitting data timestamps**. During inference, if incoming data doesn't have the same frequency, then it will be interpolated. E.g. data comes at 15 seconds resolution, and our resample_freq is '1m'. Then fitting data will be downsampled to '1m' and internal model is trained at '1m' intervals. So, during inference, prediction data would be produced at '1m' intervals, but interpolated to "15s" to match with expected output, as output data must have the same timestamps. As accepted by pandas.Timedelta (e.g. '5m').
* `seasonality` (string, optional) - As accepted by pandas.Timedelta.
- `seasonality` (string, optional) - As accepted by pandas.Timedelta.
- If `seasonal_periods` is not specified, it is calculated as `seasonality` / `frequency`
* If `seasonal_periods` is not specified, it is calculated as `seasonality` / `frequency`
Used to compute "seasonal_periods" param for the model (e.g. '1D' or '1W').
- `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculating boundaries to define anomaly score. Defaults to 2.5.
* `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculating boundaries to define anomaly score. Defaults to 2.5.
*Default model parameters*:
- If [parameter](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing#statsmodels.tsa.holtwinters.ExponentialSmoothing-parameters) `seasonal` is not specified, default value will be `add`.
* If [parameter](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing#statsmodels.tsa.holtwinters.ExponentialSmoothing-parameters) `seasonal` is not specified, default value will be `add`.
- If [parameter](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing#statsmodels.tsa.holtwinters.ExponentialSmoothing-parameters) `initialization_method` is not specified, default value will be `estimated`.
* If [parameter](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing#statsmodels.tsa.holtwinters.ExponentialSmoothing-parameters) `initialization_method` is not specified, default value will be `estimated`.
- `args` (dict, optional) - Inner model args (key-value pairs). See accepted params in [model documentation](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing#statsmodels.tsa.holtwinters.ExponentialSmoothing-parameters). Defaults to empty (not provided). Example: {"seasonal": "add", "initialization_method": "estimated"}
* `args` (dict, optional) - Inner model args (key-value pairs). See accepted params in [model documentation](https://www.statsmodels.org/dev/generated/statsmodels.tsa.holtwinters.ExponentialSmoothing#statsmodels.tsa.holtwinters.ExponentialSmoothing-parameters). Defaults to empty (not provided). Example: {"seasonal": "add", "initialization_method": "estimated"}
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
models:
@@ -1299,10 +1090,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
## vmanomaly output
@@ -1350,7 +1137,7 @@ Here in this guide, we will
We'll create `custom_model.py` file with `CustomModel` class that will inherit from `vmanomaly`'s `Model` base class.
In the `CustomModel` class, the following methods are required: - `__init__`, `fit`, `infer`, `serialize` and `deserialize`:
- `__init__` method should initiate parameters for the model.
* `__init__` method should initiate parameters for the model.
if your model relies on configs that have `arg` [key-value pair argument, like Prophet](#prophet), do not forget to use Python's `**kwargs` in method's signature and to explicitly call
@@ -1358,10 +1145,10 @@ In the `CustomModel` class, the following methods are required: - `__init__`, `f
super().__init__(**kwargs)
```
to initialize the base class each model derives from
- `fit` method should contain the model training process.
- `infer` should return Pandas.DataFrame object with model's inferences.
- `serialize` method that saves the model on disk.
- `deserialize` load the saved model from disk.
* `fit` method should contain the model training process.
* `infer` should return Pandas.DataFrame object with model's inferences.
* `serialize` method that saves the model on disk.
* `deserialize` load the saved model from disk.
For the sake of simplicity, the model in this example will return one of two values of `anomaly_score` - 0 or 1 depending on input parameter `percentage`.
@@ -1478,7 +1265,7 @@ monitoring:
Let's pull the docker image for `vmanomaly`:
```sh
docker pull victoriametrics/vmanomaly:v1.30.0
docker pull victoriametrics/vmanomaly:v1.29.7
```
Now we can run the docker container putting as volumes both config and model file:
@@ -1492,7 +1279,7 @@ docker run -it \
-v $(PWD)/license:/license \
-v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
-v $(PWD)/custom.yaml:/config.yaml \
victoriametrics/vmanomaly:v1.30.0 /config.yaml \
victoriametrics/vmanomaly:v1.29.7 /config.yaml \
--licenseFile=/license
--watch
```
@@ -1561,16 +1348,12 @@ Produced model instances are **stored in-memory** between consecutive re-fit cal
Model is useful for initial testing and for simpler data ([de-trended](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#trend) data without strict [seasonality](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-1/#seasonality) and with anomalies of similar magnitude as your "normal" data).
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.zscore.ZscoreModel"` (or `zscore` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculation boundaries and anomaly score. Defaults to `2.5`.
- `class` (string) - model class name `"model.zscore.ZscoreModel"` (or `zscore` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `z_threshold` (float, optional) - [standard score](https://en.wikipedia.org/wiki/Standard_score) for calculation boundaries and anomaly score. Defaults to `2.5`.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
models:
@@ -1590,10 +1373,6 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).
@@ -1603,16 +1382,12 @@ Resulting metrics of the model are described [here](#vmanomaly-output).
The MAD model is a robust method for anomaly detection that is *less sensitive* to outliers in data compared to standard deviation-based models. It considers a point as an anomaly if the absolute deviation from the median is significantly large.
<div class="model-details">
*Parameters specific for vmanomaly*:
{{% collapse name="Model-specific arguments" %}}
* `class` (string) - model class name `"model.mad.MADModel"` (or `mad` with class alias support{{% available_from "v1.13.0" anomaly %}})
* `threshold` (float, optional) - The threshold multiplier for the MAD to determine anomalies. Defaults to `2.5`. Higher values will identify fewer points as anomalies.
- `class` (string) - model class name `"model.mad.MADModel"` (or `mad` with class alias support{{% available_from "v1.13.0" anomaly %}})
- `threshold` (float, optional) - The threshold multiplier for the MAD to determine anomalies. Defaults to `2.5`. Higher values will identify fewer points as anomalies.
{{% /collapse %}}
{{% collapse name="Configuration example" %}}
*Config Example*
```yaml
@@ -1633,8 +1408,4 @@ models:
# anomaly_score_outside_data_range: 1.01 # auto anomaly score (1.01) if `y` (real value) is outside of data_range, if set
```
{{% /collapse %}}
</div>
Resulting metrics of the model are described [here](#vmanomaly-output).

View File

@@ -352,20 +352,6 @@ For detailed guidance on configuring mTLS parameters such as `verify_tls`, `tls_
<td>Gauge</td>
<td>Timestamp of the last successful config [hot-reload](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) in seconds since epoch {{% available_from "v1.25.1" anomaly %}}</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`vmanomaly_scheduler_alive`</span>
</td>
<td>Gauge</td>
<td>Whether the scheduler worker thread identified by `scheduler_alias` and `preset` is alive (`1`) or not (`0`) {{% available_from "v1.30.0" anomaly %}}.</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`vmanomaly_scheduler_restarts_total`</span>
</td>
<td>Counter</td>
<td>Number of bounded scheduler restart attempts {{% available_from "v1.30.0" anomaly %}}, labeled by `scheduler_alias`, `preset`, and `status` (`success` or `failure`).</td>
</tr>
</tbody>
</table>

View File

@@ -263,33 +263,7 @@ BasicAuth password. If set, it will be used to authenticate the request.
`30s`
</td>
<td>
Backward-compatible timeout used for both datasource fetches and post-fetch processing when `fetch_timeout` or `processing_timeout` are not set.
</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`fetch_timeout`</span>
</td>
<td>
Not set (`timeout` fallback)
</td>
<td>
Optional timeout {{% available_from "v1.30.0" anomaly %}} for each datasource read request. Use values such as `5s`, `30s`, or `1m`.
</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`processing_timeout`</span>
</td>
<td>
Not set (`timeout` fallback)
</td>
<td>
Optional timeout {{% available_from "v1.30.0" anomaly %}} for post-fetch processing that prepares returned data for fit or inference. High-cardinality queries may need a larger processing timeout than their datasource fetch timeout.
Timeout for the requests, passed as a string
</td>
</tr>
<tr>
@@ -388,19 +362,6 @@ If True, then query will be performed from the last seen timestamp for a given s
<tr>
<td>
<span style="white-space: nowrap;">`query_last_seen_max_lookback`</span>
</td>
<td>
`None`
</td>
<td>
Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far `query_from_last_seen_timestamp` may move a query start into the past to recover skipped inference intervals. When configured below the query step, the effective cap is raised to one step. Examples: `5m`, `1h`.
</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`latency_offset`</span>
</td>
<td>
@@ -434,7 +395,7 @@ Optional arg{{% available_from "v1.17.0" anomaly %}} overrides how `search.maxPo
`UTC`
</td>
<td>
Optional argument {{% available_from "v1.18.0" anomaly %}} specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`TemporalEnvelopeModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#temporal-envelope), [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet), or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
Optional argument{{% available_from "v1.18.0" anomaly %}} specifies the [IANA](https://nodatime.org/TimeZones) timezone to account for local shifts, like [DST](https://en.wikipedia.org/wiki/Daylight_saving_time), in models sensitive to seasonal patterns (e.g., [`ProphetModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) or [`OnlineQuantileModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-seasonal-quantile)). Defaults to `UTC` if not set and can be overridden on a [per-query basis](#per-query-parameters).
</td>
</tr>
<tr>
@@ -500,9 +461,6 @@ reader:
# tenant_id: '1:0' # if set, overrides reader-level tenant_id
# offset: '-15s' # if set, overrides reader-level offset
sampling_period: '1m'
timeout: '30s' # backward-compatible default for both phases
fetch_timeout: '30s' # timeout for each datasource request, overrides `timeout` if set
processing_timeout: '1m' # timeout for preparing fetched series for fit/infer, overrides `timeout` if set
query_from_last_seen_timestamp: True # false by default
latency_offset: '1ms'
series_processing_batch_size: 8
@@ -842,33 +800,7 @@ Frequency of the points returned. Will be converted to `/select/stats_query_rang
`30s`
</td>
<td>
(Optional) Backward-compatible timeout used for both datasource fetches and post-fetch processing when `fetch_timeout` or `processing_timeout` are not set.
</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`fetch_timeout`</span>
</td>
<td>
Not set (`timeout` fallback)
</td>
<td>
Optional timeout {{% available_from "v1.30.0" anomaly %}} for each VictoriaLogs datasource read request. Use values such as `5s`, `30s`, or `1m`.
</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`processing_timeout`</span>
</td>
<td>
Not set (`timeout` fallback)
</td>
<td>
Optional timeout {{% available_from "v1.30.0" anomaly %}} for post-fetch processing that prepares returned data for fit or inference. High-cardinality results may need a larger processing timeout than their datasource fetch timeout.
(Optional) Specifies the maximum duration to wait for a query to complete before timing out. Can be set on a [per-query basis](#per-query-parameters-1) to override the reader-level setting.
</td>
</tr>
<tr>
@@ -974,19 +906,6 @@ If a path to a CA bundle file (like `ca.crt`), it will verify the certificate us
Optional argument {{% available_from "v1.29.7" anomaly %}}, allows specifying the number of time series to process together while preparing data for fit or infer stages. Defaults to `8`. Suggested values are 4-16 for high-cardinality queries.
</td>
</tr>
<tr>
<td>
<span style="white-space: nowrap;">`query_last_seen_max_lookback`</span>
</td>
<td>
`None`
</td>
<td>
Optional hard cap {{% available_from "v1.30.0" anomaly %}} for how far last-seen recovery may move a query start into the past. Examples: `5m`, `1h`.
</td>
</tr>
</tbody>
</table>
@@ -1008,9 +927,7 @@ reader:
series_processing_batch_size: 8
data_range: [0, 'inf'] # reader-level
offset: '0s' # reader-level
timeout: '30s' # backward-compatible default for both phases
fetch_timeout: '30s' # timeout for each datasource request, overrides `timeout` if set
processing_timeout: '1m' # timeout for preparing fetched series for fit/infer, overrides `timeout` if set
timeout: '30s'
queries:
# one query returning 1 result fields (avg_duration), it will have __name__ label (series name) as `duration_30m__avg`
duration_avg_30m:

View File

@@ -72,8 +72,6 @@ options={`"scheduler.periodic.PeriodicScheduler"`, `"scheduler.oneoff.OneoffSche
> If `start_from` [parameter](#parameters-1) is used, it's suggested to also set `restore_state: true` in the [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) of a config, so that the scheduler can restore its state from the previous run **if terminated or restarted in between scheduled runs** and continue producing anomaly scores without interruptions, otherwise the service will be idle until future `start_from` time is reached. E.g. if `start_from` is set to `20:00` and the service is started and then terminated and restarted at `20:30`, it will not produce any anomaly scores until the next day's `20:00` is reached (+23:30 of being idle), which introduces inconvenience for the users.
> {{% available_from "v1.30.0" anomaly %}} If a periodic scheduler worker exits unexpectedly, the service attempts bounded restarts with exponential backoff instead of shutting down unrelated schedulers. Monitor [`vmanomaly_scheduler_alive`](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/#startup-metrics) and `vmanomaly_scheduler_restarts_total` to alert on persistent failures.
### Parameters
For periodic scheduler parameters are defined as differences in times, expressed in difference units, e.g. days, hours, minutes, seconds. Time granularity is defined by the last characters of a string. Examples: `"50s"` (seconds), `"4m"` (minutes), `"3h"` (hours), `"2d"` (days), `"1w"` (weeks).

View File

@@ -64,15 +64,3 @@ reader:
After starting the `vmanomaly` server with the above configuration, UI can be accessed at `<vmanomaly-host>:8490/vmanomaly/vmui/` (e.g. `http://localhost:8490/vmanomaly/vmui/`).
Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).
### Time-series analysis and autotune API
{{% available_from "v1.30.0" anomaly %}} The server exposes bounded endpoints for UI, MCP, and automation workflows:
- `GET /api/v1/timeseries/characteristics` samples the supplied query and summarizes trend, calendar seasonality, changepoints, gaps, and intermittent or spiky behavior. Use `limit` (default 100) to cap sampled series and pass the production `step` and timezone.
- `POST /api/v1/autotune/tasks` starts asynchronous shared-model tuning. The request contains the query, candidate `tuned_class_name`, expected `anomaly_percentage`, data-source settings, and optimization parameters.
- `GET /api/v1/autotune/tasks/{task_id}` returns progress and the concrete suggested `modelConfig` when complete.
- `DELETE /api/v1/autotune/tasks/{task_id}` cancels pending work cooperatively.
> [!TIP]
> For a complete request and recommended workflow, see [Shared asynchronous autotune workflow](https://docs.victoriametrics.com/anomaly-detection/components/models/#shared-asynchronous-autotune-workflow). OpenAPI schemas for the running version are available at `/docs` endpoint of a running `vmanomaly` instance.

View File

@@ -10,9 +10,9 @@ sitemap:
- To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
- In the tutorial, we'll be using the following VictoriaMetrics components:
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.148.0)
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.148.0)
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.148.0)
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.147.0)
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.147.0)
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.147.0)
- [Grafana](https://grafana.com/) (v12.2.0)
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.9.1) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.28.1)
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.148.0
image: victoriametrics/vmagent:v1.147.0
depends_on:
- "victoriametrics"
ports:
@@ -340,7 +340,7 @@ services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.148.0
image: victoriametrics/victoria-metrics:v1.147.0
ports:
- 8428:8428
volumes:
@@ -373,7 +373,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.148.0
image: victoriametrics/vmalert:v1.147.0
depends_on:
- "victoriametrics"
ports:
@@ -395,7 +395,7 @@ services:
restart: always
vmanomaly:
container_name: vmanomaly
image: victoriametrics/vmanomaly:v1.30.0
image: victoriametrics/vmanomaly:v1.29.7
depends_on:
- "victoriametrics"
ports:

View File

@@ -1,426 +0,0 @@
Several VictoriaMetrics components can connect to cloud storage to read or write object data.
The following table shows the supported types of storage for each component:
| Component | AWS S3 and S3-compatible | Google Cloud Storage | Azure Blob Storage |
|-----------|----|----------------------|--------------------|
| [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) | ✅ | ✅ | ✅ |
| [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/) | ✅ | ✅ | ✅ |
| [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) | ✅ | ✅ | ✅ |
| [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) | ✅ | ✅ | ❌ |
All these components use the same underlying libraries, so the authentication setup is largely the same. The main difference is in the command-line flags:
- vmalert uses `-s3.*` prefixed flags (e.g., `-s3.credsFilePath`)
- backup and restore tools use unprefixed flags (e.g., `-credsFilePath`)
See the [component reference](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/#per-component-flag-reference) for details.
## Obtaining credentials
You need to supply credentials so the component can connect to the cloud storage. The setup differs by provider; the sections below cover AWS S3, S3compatible systems, GCS, and Azure Blob Storage.
### AWS S3
1. In AWS, [create an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) or role with the minimum permissions for the target bucket (see table below).
1. [Create an access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) for that IAM identity.
1. Copy the **Access key ID** and **Secret access key** values. You will use them in the credentials file or environment variables.
| Component | S3 usage | Minimum S3 permissions |
|-----------------|-------------------------------------------------------------|------------------------|
| vmalert (Enterprise) | Reads alerting/recording rules from bucket. | `s3:GetObject`, `s3:ListBucket` on the rules bucket/prefix.|
| vmrestore | Restores backups from S3 buckets | `s3:GetObject`, `s3:ListBucket` on the backup bucket/prefix.|
| vmbackup | Uploads backups to S3 and may delete old backup objects during maintenance.| `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject` on the backup bucket/prefix.|
| vmbackupmanager (Enterprise) | Automates backups using vmbackup behavior. | Same as vmbackup. |
### S3-compatible storage (MinIO, Ceph)
Generate access keys using your storage system's admin interface or CLI. The credentials follow the same format as AWS S3.
### Google Cloud Storage
1. Open the Google Cloud Console and go to **IAM & Admin > Service Accounts**.
1. Click **Create service account**, enter a name, and assign a Storage role (for example, Storage Object Admin).
1. Open the service account, go to **Keys**, then click **Add key > Create new key**.
1. Choose **JSON** as the key type and click **Create**.
1. Store the JSON file on the machine running the VictoriaMetrics component.
### Azure Blob Storage
> Azure does not use credential files.
1. Log in to the Azure Portal.
1. Search for and select **Storage accounts**.
1. Click on your specific storage account name. (This is your `AZURE_STORAGE_ACCOUNT_NAME`).
1. In the left menu under **Security + networking**, click **Access keys**.
1. Copy the key value from either **key1** or **key2** (this is your `AZURE_STORAGE_ACCOUNT_KEY`).
1. Define the access keys as environment variables.
```sh
export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
```
## Authenticating with the cloud provider
Provide the credentials as a file or with environment variables, along with the path to the cloud storage bucket. The syntax for the bucket name depends on the cloud provider:
- `s3://`: for AWS S3 and self-hosted S3-compatible storage (MinIO, Ceph)
- `gs://`: Google Cloud Storage
- `azblob://`: Azure Blob Storage
### vmbackup and vmrestore
The following example backups to an AWS S3 bucket using a credentials file:
```sh
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=s3://victoriametrics-backup/backup01 \
-credsFilePath=/etc/credentials
```
In order to restore from the same backup from AWS S3:
```sh
vmrestore \
-src=s3://victoriametrics-backup/backup01 \
-storageDataPath=/data \
-credsFilePath=/etc/credentials
```
> To use non-AWS S3 buckets such as MinIO or Ceph, you must [supply the `-customS3Endpoint` argument](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/#s3-compatible).
Alternatively, you can set the access keys as environment variables instead of using a credential file:
```sh
export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=s3://victoriametrics-backup/backup01
vmrestore \
-src=s3://victoriametrics-backup/backup01 \
-storageDataPath=/data
```
Backups on Google Cloud Storage use the `gs://` prefix in the destination:
```sh
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=gs://victoriametrics-backup/backup01 \
-credsFilePath=/etc/credentials
```
You can restore this backup with:
```sh
vmrestore \
-src=gs://victoriametrics-backup/backup01 \
-storageDataPath=/data \
-credsFilePath=/etc/credentials
```
On Google Cloud, you can define the path to the JSON credential file with `GOOGLE_APPLICATION_CREDENTIALS`. For example:
```sh
export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=gs://victoriametrics-backup/backup01
vmrestore \
-src=gs://victoriametrics-backup/backup01 \
-storageDataPath=/data
```
For Azure Blob Storage, use the `azblob://` prefix and rely on environment variables instead of `-credsFilePath`.
```sh
export AZURE_STORAGE_ACCOUNT_NAME=myaccount
export AZURE_STORAGE_ACCOUNT_KEY=mykey
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=azblob://victoriametrics-backup/backup01
vmrestore \
-src=azblob://victoriametrics-backup/backup01 \
-storageDataPath=/data
```
### vmbackupmanager
> vmbackupmanager only works in the [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) edition.
To manage backups with vmbackupmanager on AWS S3, add the credentials with the `-credsFilePath` flag:
```sh
vmbackupmanager \
-dst=s3://vmstorage-data/backups \
-credsFilePath=/etc/credentials \
-storageDataPath=/vmstorage-data \
-snapshot.createURL=http://vmstorage:8482/snapshot/create \
-licenseFile=/etc/vm-license
```
Or define the access keys using environment variables:
```sh
export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
vmbackupmanager \
-dst=s3://vmstorage-data/backups \
-storageDataPath=/vmstorage-data \
-snapshot.createURL=http://vmstorage:8482/snapshot/create \
-licenseFile=/etc/vm-license
```
> To use non-AWS S3 buckets, you must [supply the `-customS3Endpoint` argument](#s3-compatible).
Automated backups on Google Cloud Storage take the following form:
```sh
vmbackupmanager \
-dst=gs://vmstorage-data/backups \
-credsFilePath=/etc/credentials \
-storageDataPath=/vmstorage-data \
-snapshot.createURL=http://vmstorage:8482/snapshot/create \
-licenseFile=/etc/vm-license
```
As with vmbackup and vmrestore, you can also define the path to the JSON credential file with `GOOGLE_APPLICATION_CREDENTIALS`:
```sh
export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
vmbackupmanager \
-dst=gs://vmstorage-data/backups \
-storageDataPath=/vmstorage-data \
-snapshot.createURL=http://vmstorage:8482/snapshot/create \
-licenseFile=/etc/vm-license
```
vmbackupmanager can also use Azure Blob Storage by defining environment variables:
```sh
export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
vmbackupmanager \
-dst=azblob://vmstorage-data/backups \
-storageDataPath=/vmstorage-data \
-snapshot.createURL=http://vmstorage:8482/snapshot/create \
-licenseFile=/etc/vm-license
```
### vmalert
> - vmalert cloud storage command line flags are prefixed with `-s3.` for S3 buckets *and* Google Cloud Storage.
> - Cloud storage only works in the [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) edition.
Read alerting rules from an S3 bucket. The `-rule` flag accepts a prefix, so it matches all files starting with `alerts_` in the `rules` folder:
```sh
vmalert \
-rule=s3://my-alert-bucket/rules/alerts_ \
-s3.credsFilePath=/etc/vmalert/aws-credentials \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
Instead of a credential file, you can supply the access keys using environment variables:
```sh
export AWS_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY
export AWS_SECRET_ACCESS_KEY=YOUR_SECRET_AWS_ACCESS_KEY
vmalert \
-rule=s3://my-alert-bucket/rules/alerts_ \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
> To use non-AWS S3 buckets, you must [supply the `-s3.customEndpoint` argument](#s3-compatible).
To read rules from Google Cloud Storage:
```sh
vmalert \
-rule=gs://my-alert-bucket/rules/alerts_ \
-s3.credsFilePath=/etc/vmalert/gcp-service-account.json \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
If you prefer, you can supply the path to the JSON credential file with the `GOOGLE_APPLICATION_CREDENTIALS` environment variable:
```sh
export GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials
vmalert \
-rule=gs://my-alert-bucket/rules/alerts_ \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
## Credentials files format
The file format depends on the storage provider.
### S3 credentials
The file uses the standard AWS shared credentials format used by the [AWS CLI](https://docs.aws.amazon.com/cli/v1/userguide/cli-configure-files.html) and [AWS SDKs](https://docs.aws.amazon.com/sdkref/latest/guide/file-format.html):
```ini
[default]
aws_access_key_id=YOUR_AWS_ACCESS_KEY
aws_secret_access_key=YOUR_AWS_SECRET_ACCESS_KEY
```
You can define multiple profiles in a single file:
```ini
[default]
aws_access_key_id=DEFAULT_ACCESS_KEY
aws_secret_access_key=DEFAULT_SECRET_KEY
[monitoring]
aws_access_key_id=MONITORING_ACCESS_KEY
aws_secret_access_key=MONITORING_SECRET_KEY
[alerts]
aws_access_key_id=ALERTS_ACCESS_KEY
aws_secret_access_key=ALERTS_SECRET_KEY
```
Use the `-configProfile` flag (or `-s3.configProfile` in vmalert) to select a non-default profile:
```sh
-configProfile=alerts
```
You can separate credentials from other configuration settings. Put credentials in one file:
```ini
[default]
aws_access_key_id=DEFAULT_ACCESS_KEY
aws_secret_access_key=DEFAULT_SECRET_KEY
```
And non-sensitive settings in another:
```ini
[default]
region=us-east-1
```
Then pass both files:
```sh
-configFilePath=/etc/aws-config \
-credsFilePath=/etc/credentials
```
### GCS credentials file format
The file is the JSON key downloaded from Google Cloud Console. Its content looks like this:
```json
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "key-id",
"private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
"client_email": "service-account-email",
"client_id": "client-id",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
}
```
This is the standard service account key format defined by [Google Cloud IAM](https://developers.google.com/workspace/guides/create-credentials).
### Azure Blob Storage
Azure does not support credentials via file. Use environment variables instead.
```sh
export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
```
## Self-hosted and S3-compatible endpoints
For S3-compatible storage such as MinIO or Ceph, set a custom endpoint with the `-customS3Endpoint` flag for vmbackup, vmrestore, and vmbackupmanager. For example:
```sh
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=s3://victoriametrics-backup/backup01 \
-customS3Endpoint=http://minio.example.local:9000
```
On vmalert, use the `-s3.customEndpoint` flag instead:
```sh
vmalert \
-rule=s3://my-alert-bucket/rules/alerts_ \
-s3.customEndpoint=http://minio.example.local:9000 \
-s3.credsFilePath=/etc/vmalert/aws-credentials \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
### Addressing S3-compatible buckets {#s3-compatible}
When connecting to non-AWS S3-compatible buckets, there is an additional flag you might need to configure:
- `-s3ForcePathStyle`: on vmbackupmanager, vmbackup, and vmrestore.
- `-s3.forcePathStyle`: on vmalert.
The flag changes the expected URL pattern for a bucket.
| Flag value | Address-style | Example | Use with |
|------------|---------------|---------|----------|
| `true` (default) | Path-style | `https://endpoint/bucket/key` | MinIO, Ceph, most S3-compatible storages |
| `false` | Virtual host-style | `https://bucket.endpoint/key` | [Aliyun OSS](https://www.aliyun.com/product/oss) and other endpoints that require it |
> The flag only takes effect when you use a custom endpoint (`-customS3Endpoint` or `-s3.customEndpoint` on vmalert). When connecting to real AWS S3, the SDK handles addressing automatically.
## Per-component flag reference
The table below shows how the same concept maps to different flag names across components.
| Concept | vmalert | vmbackup, vmrestore, and vmbackupmanager |
|---|---|---|---|---|
| Credentials file | `-s3.credsFilePath` | `-credsFilePath` |
| Config file | `-s3.configFilePath` | `-configFilePath` |
| Profile selection | `-s3.configProfile` | `-configProfile` |
| Custom endpoint | `-s3.customEndpoint` | `-customS3Endpoint` |
| Force path style | `-s3.forcePathStyle` | `-s3ForcePathStyle` |
| TLS insecure | N/A | `-s3TLSInsecureSkipVerify` |
| Storage class | N/A | `-s3StorageClass` |

View File

@@ -1,12 +0,0 @@
---
weight: 5
title: Connecting VictoriaMetrics components to cloud storage
menu:
docs:
parent: guides
weight: 5
tags:
- metrics
- guide
---
{{% content "README.md" %}}

View File

@@ -240,23 +240,23 @@ vmagent will write data into VictoriaMetrics single-node and cluster (with tenan
# compose.yaml
services:
vmsingle:
image: victoriametrics/victoria-metrics:v1.148.0
image: victoriametrics/victoria-metrics:v1.147.0
vmstorage:
image: victoriametrics/vmstorage:v1.148.0-cluster
image: victoriametrics/vmstorage:v1.147.0-cluster
vminsert:
image: victoriametrics/vminsert:v1.148.0-cluster
image: victoriametrics/vminsert:v1.147.0-cluster
command:
- -storageNode=vmstorage:8400
vmselect:
image: victoriametrics/vmselect:v1.148.0-cluster
image: victoriametrics/vmselect:v1.147.0-cluster
command:
- -storageNode=vmstorage:8401
vmagent:
image: victoriametrics/vmagent:v1.148.0
image: victoriametrics/vmagent:v1.147.0
volumes:
- ./scrape.yaml:/etc/vmagent/config.yaml
command:
@@ -308,7 +308,7 @@ Now add the vmauth service to `compose.yaml`:
# compose.yaml
services:
vmauth:
image: docker.io/victoriametrics/vmauth:v1.148.0
image: docker.io/victoriametrics/vmauth:v1.147.0
ports:
- 8427:8427
volumes:

View File

@@ -155,15 +155,15 @@ These services will store and query the metrics scraped by vmagent.
# compose.yaml
services:
vmstorage:
image: victoriametrics/vmstorage:v1.148.0-cluster
image: victoriametrics/vmstorage:v1.147.0-cluster
vminsert:
image: victoriametrics/vminsert:v1.148.0-cluster
image: victoriametrics/vminsert:v1.147.0-cluster
command:
- -storageNode=vmstorage:8400
vmselect:
image: victoriametrics/vmselect:v1.148.0-cluster
image: victoriametrics/vmselect:v1.147.0-cluster
command:
- -storageNode=vmstorage:8401
ports:
@@ -196,7 +196,7 @@ Add the vmauth service to `compose.yaml`:
# compose.yaml
services:
vmauth:
image: victoriametrics/vmauth:v1.148.0-enterprise
image: victoriametrics/vmauth:v1.147.0-enterprise
ports:
- 8427:8427
volumes:
@@ -251,7 +251,7 @@ Add the vmagent service to `compose.yaml` with OAuth2 configuration:
# compose.yaml
services:
vmagent:
image: victoriametrics/vmagent:v1.148.0
image: victoriametrics/vmagent:v1.147.0
volumes:
- ./scrape.yaml:/etc/vmagent/config.yaml
command:

View File

@@ -107,7 +107,7 @@ The final piece is the Docker Compose file. This ties all the services together
# compose.yml
services:
victoriametrics:
image: victoriametrics/victoria-metrics:v1.148.0
image: victoriametrics/victoria-metrics:v1.147.0
command:
- "--storageDataPath=/victoria-metrics-data"
- "--selfScrapeInterval=10s"
@@ -128,7 +128,7 @@ services:
- ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
vmalert:
image: victoriametrics/vmalert:v1.148.0
image: victoriametrics/vmalert:v1.147.0
depends_on:
- victoriametrics
- alertmanager

View File

@@ -114,8 +114,6 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
* [QCon London 2026: Wrangling Telemetry at Scale, a Guide to Self-Hosted Observability](https://www.infoq.com/news/2026/03/self-hosted-observability/)
* [How We Made Telemetry Queries 10x Faster: Chunk-Split Caching for Metrics, Logs, and Traces](https://mirastacklabs.ai/blog/chunk-split-caching/)
* [Claude Code: creating Kubernetes debugging AI Agent for VictoriaMetrics](https://rtfm.co.ua/en/claude-code-creating-kubernetes-debugging-ai-agent-for-victoriametrics/)
* [OpenTelemetry: OTel Collectors in Kubernetes and VictoriaMetrics Stack integration](https://itnext.io/opentelemetry-otel-collectors-in-kubernetes-and-victoriametrics-stack-integration-d907ed0a15a0)
* [VictoriaMetrics vs Prometheus: my default, and when I still pick Prometheus](https://jorijn.com/en/blog/victoriametrics-vs-prometheus/)
## Third-party articles and slides about VictoriaLogs
@@ -156,6 +154,13 @@ See [our blog](https://victoriametrics.com/blog) for the latest articles written
* [Why irate from Prometheus doesn't capture spikes](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
* [VictoriaMetrics: PromQL compliance](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e)
* [How do open source solutions for logs work: Elasticsearch, Loki and VictoriaLogs](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f)
* [How vmagent Collects and Ships Metrics Fast with Aggregation, Deduplication, and More](https://victoriametrics.com/blog/vmagent-how-it-works/)
* [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/)
* [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/)
* [How vmstorage Processes Data: Retention, Merging, Deduplication...](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/)
* [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/)
* [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/)
* [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/)
### Tutorials, guides and how-to articles
@@ -173,6 +178,12 @@ See [our guides](https://docs.victoriametrics.com/guides/) for the up-to-date gu
* [Prometheus storage: tech terms for humans](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
* [Cardinality explorer](https://victoriametrics.com/blog/cardinality-explorer/)
* [Rules backfilling via vmalert](https://victoriametrics.com/blog/rules-replay/)
* [vmagent: Key Features Explained in Under 15 Minutes](https://victoriametrics.com/blog/vmagent-key-features-explained/)
* [Prometheus Metrics Explained: Counters, Gauges, Histograms & Summaries](https://victoriametrics.com/blog/prometheus-monitoring-metrics-counters-gauges-histogram-summaries/)
* [Prometheus Monitoring: Instant Queries and Range Queries Explained](https://victoriametrics.com/blog/prometheus-monitoring-instant-range-query/)
* [Prometheus Monitoring: Functions, Subqueries, Operators, and Modifiers](https://victoriametrics.com/blog/prometheus-monitoring-function-operator-modifier/)
* [Prometheus Alerting 101: Rules, Recording Rules, and Alertmanager](https://victoriametrics.com/blog/alerting-recording-rules-alertmanager/)
* [Alerting Best Practices](https://victoriametrics.com/blog/alerting-best-practices/)
### Other articles

View File

@@ -59,6 +59,11 @@ It increases cluster availability, and simplifies cluster maintenance as well as
![Cluster Scheme](Cluster-VictoriaMetrics-components.webp)
> Further reading, deep dives into how each service works internally:
> - `vmstorage`: [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/), [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/), [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/).
> - `vminsert`: [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/).
> - `vmselect`: [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/).
## vmui
VictoriaMetrics cluster version provides UI for query troubleshooting and exploration. The UI is available at
@@ -834,7 +839,7 @@ This ensures that incoming metrics are evenly distributed across all `vmstorage`
The downside is that a single slow vmstorage node can throttle the entire cluster.
When `-disableRerouting=false` is enabled on `vminsert`,
the cluster will automatically re-route writes away from the slowest vmstorage node to preserve maximum ingestion throughput.
the cluster will automatically [re-route writes](https://victoriametrics.com/blog/vminsert-how-it-works/#31-rerouting) away from the slowest vmstorage node to preserve maximum ingestion throughput.
Re-routing occurs only when all of the following conditions hold:
- the storage send buffer is full.
@@ -877,7 +882,7 @@ See also [resource usage limits docs](#resource-usage-limits).
## Rebalancing
Every `vminsert` node evenly spreads (shards) incoming data among `vmstorage` nodes specified in the `-storageNode` command-line flag.
Every `vminsert` node [evenly spreads (shards) incoming data](https://victoriametrics.com/blog/vminsert-how-it-works/#3-sharding-and-buffering) among `vmstorage` nodes specified in the `-storageNode` command-line flag.
This guarantees even distribution of the ingested data among `vmstorage` nodes. When new `vmstorage` nodes are added to the `-storageNode`
command-line flag at `vminsert`, then only newly ingested data is distributed evenly among old and new `vmstorage` nodes, while
historical data remains on the old `vmstorage` nodes. This speeds up data ingestion and querying for the majority of production workloads,
@@ -1025,7 +1030,7 @@ By default, VictoriaMetrics offloads replication to the underlying storage point
which guarantees data durability. VictoriaMetrics supports application-level replication if replicated durable persistent disks cannot be used for some reason.
The replication can be enabled by passing `-replicationFactor=N` command-line flag to `vminsert`. This instructs `vminsert` to store `N` copies for every ingested sample
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable. See [how `vminsert` replicates each sample to `N` `vmstorage` nodes](https://victoriametrics.com/blog/vminsert-how-it-works/#4-replication-and-sending-data-to-vmstorage) for details.
Passing `-replicationFactor=N` command-line flag to `vmselect` instructs it to not mark responses as `partial` if less than `-replicationFactor` vmstorage nodes are unavailable during the query.
See [cluster availability docs](#cluster-availability) for details.
@@ -1060,7 +1065,7 @@ deduplication can't be guaranteed when samples and sample duplicates for the sam
- when `vmstorage` node has no enough capacity for processing incoming data stream. Then `vminsert` re-routes new samples to other `vmstorage` nodes.
It is recommended to set **the same** `-dedup.minScrapeInterval` command-line flag value to both `vmselect` and `vmstorage` nodes
to ensure query results consistency, even if storage layer didn't complete deduplication yet.
to ensure query results consistency, even if [storage layer didn't complete deduplication](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#deduplication) yet.
## Metrics Metadata

View File

@@ -27,5 +27,5 @@ to [the latest available releases](https://docs.victoriametrics.com/victoriametr
## Currently supported LTS release lines
- v1.136.x - the latest one is [v1.136.14 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.14)
- v1.122.x - the latest one is [v1.122.27 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.27)
- v1.136.x - the latest one is [v1.136.4 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.4)
- v1.122.x - the latest one is [v1.122.19 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.19)

View File

@@ -14,178 +14,6 @@ aliases:
- /quick-start/index.html
- /quick-start/
---
There are two ways to get started with VictoriaMetrics:
* [Try it locally](https://docs.victoriametrics.com/victoriametrics/quick-start/#try-it-locally) - if you just want to see how VictoriaMetrics works,
go with the single binary: download it, start it with one command, and see live metrics
in the built-in UI in a couple of minutes. No Docker, no configuration files
and no extra components are required;
* [Install it](https://docs.victoriametrics.com/victoriametrics/quick-start/#how-to-install) - if you want to set up VictoriaMetrics for real use,
pick a distribution (single-node, cluster or cloud) and an installation method
(Docker, Helm, binary releases, etc.).
If you'd rather not install anything at all, try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) -
a list of publicly available playgrounds for VictoriaMetrics software.
Whichever way you choose, you may also find interesting the other sections of this page,
like how to [write](https://docs.victoriametrics.com/victoriametrics/quick-start/#write-data) and [query](https://docs.victoriametrics.com/victoriametrics/quick-start/#query-data) data,
[alerting](https://docs.victoriametrics.com/victoriametrics/quick-start/#alerting),
[data migration](https://docs.victoriametrics.com/victoriametrics/quick-start/#data-migration) from other TSDBs,
and [productionization](https://docs.victoriametrics.com/victoriametrics/quick-start/#productionization)
best practices for running VictoriaMetrics in production.
## Try it locally
The fastest way to try VictoriaMetrics on your own machine is its binary - the only thing needed to run it.
### Step 1: Download the binary
Create a directory for this test drive, so all the files created along the way stay in one place:
```sh
mkdir vm-quick-start && cd vm-quick-start
```
Download the `victoria-metrics-<os>-<arch>-<version>.tar.gz` archive for your OS and architecture
from the [releases page](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest)
and unpack it. It contains a single `victoria-metrics-prod` binary.
For example, on Linux with `amd64` architecture:
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.148.0.tar.gz
```
The binary is self-contained and requires no installation - it is ready to run as is.
### Step 2: Start VictoriaMetrics
Starting VictoriaMetrics is as simple as executing the binary, with no arguments at all.
But since it is nicer to have some data to explore right after the start, let's also enable self-scraping
via the `-selfScrapeInterval` command-line flag, so VictoriaMetrics collects metrics about itself:
```sh
./victoria-metrics-prod -selfScrapeInterval=10s
```
VictoriaMetrics prints a couple of dozen log lines on start, describing the storage, caches
and memory limits it sets up. Look for these two lines confirming that it is up and running:
```sh
2026-07-10T16:55:06.615Z info app/victoria-metrics/main.go:102 started VictoriaMetrics in 0.019 seconds
...
2026-07-10T16:55:06.615Z info lib/httpserver/httpserver.go:148 started server at http://0.0.0.0:8428/
```
That's it - VictoriaMetrics is running, listening on port `8428` and scraping its own metrics
(CPU and memory usage, the number of stored metrics, request rates, etc.) every 10 seconds.
If you list the `vm-quick-start` directory, you can see a new `victoria-metrics-data` directory
created next to the binary - this is where the collected data is stored.
Now that metrics are being collected, it's time to look at them.
### Step 3: Explore the metrics
Open [http://localhost:8428/vmui](http://localhost:8428/vmui) in your browser to access [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui) -
the built-in UI for querying and graphing metrics. You should see its query page:
![vmui query page](quick-start-vmui.webp)
Self-scraped metrics become queryable within ~30 seconds after the start.
Try the following:
* Open the [metrics explorer](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#metrics-explorer)
at [http://localhost:8428/vmui/#/metrics](http://localhost:8428/vmui/#/metrics) to browse all collected metrics;
* Enter a query in the input field at [http://localhost:8428/vmui](http://localhost:8428/vmui) and press `Enter`. For example:
* `process_resident_memory_bytes` - memory used by VictoriaMetrics;
* `rate(process_cpu_seconds_total)` - its CPU usage;
* `vm_rows{type=~"storage/.+"}` - the number of stored [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples).
Queries are written in [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) -
a PromQL-compatible query language, so any PromQL query works here as well.
So far the only available metrics are the ones VictoriaMetrics reports about itself - let's collect something more interesting.
### Step 4 (optional): Collect metrics from your system
Self-scraped metrics only describe VictoriaMetrics itself. To monitor your machine (CPU, memory, disk, network),
run [node_exporter](https://github.com/prometheus/node_exporter) - the standard Prometheus exporter for host metrics -
and let VictoriaMetrics scrape it. Single-node VictoriaMetrics has a built-in
[Prometheus-compatible scraper](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter),
so no other components are needed.
1. Download the `node_exporter-<version>.<os>-<arch>.tar.gz` archive for your OS and architecture
from the [releases page](https://github.com/prometheus/node_exporter/releases/latest) and unpack it.
For example, on Linux with `amd64` architecture:
```sh
wget https://github.com/prometheus/node_exporter/releases/download/v1.12.0/node_exporter-1.12.0.linux-amd64.tar.gz
tar xzf node_exporter-1.12.0.linux-amd64.tar.gz
```
Unlike VictoriaMetrics, it unpacks into its own directory. Start it from there:
```sh
./node_exporter-1.12.0.linux-amd64/node_exporter
```
It exposes host metrics at [http://localhost:9100/metrics](http://localhost:9100/metrics).
1. Create a `scrape.yaml` file with the following contents:
```yaml
scrape_configs:
- job_name: node-exporter
static_configs:
- targets:
- localhost:9100
```
1. Restart VictoriaMetrics with the `-promscrape.config` command-line flag pointing to this file:
```sh
./victoria-metrics-prod -selfScrapeInterval=10s -promscrape.config=scrape.yaml
```
Check [http://localhost:8428/targets](http://localhost:8428/targets) - the `node-exporter` target should have `state: up`.
The target shows up as `state: down` until the first scrape happens, which can take some seconds -
just reload the page a bit later.
Now query host metrics in [vmui](http://localhost:8428/vmui). For example:
* `node_memory_MemAvailable_bytes` - available memory on your machine;
* `100 - avg(rate(node_cpu_seconds_total{mode="idle"})) * 100` - overall CPU usage in percent.
See [scrape config examples](https://docs.victoriametrics.com/victoriametrics/scrape_config_examples/) for more advanced scrape configurations.
Scraping pulls metrics from targets, but it is not the only way to get data in - metrics can also be pushed directly.
### Step 5 (optional): Push your own metrics
VictoriaMetrics also accepts metrics pushed via [many popular protocols](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#push-model).
For example, insert a measurement using the InfluxDB line protocol with a plain `curl` command:
```sh
curl -d 'room_temperature,room=kitchen value=21.5' http://localhost:8428/write
```
Then query `room_temperature_value` in [vmui](http://localhost:8428/vmui) to see it.
Take into account that freshly ingested data can take up to 30 seconds to show up in query results,
so don't worry if it doesn't appear immediately - just retry the query a bit later.
Once you're done experimenting, tidying everything up takes a single command.
### Cleanup
Stop VictoriaMetrics and node_exporter with `Ctrl+C` in their respective terminals. All the collected data lives in the `victoria-metrics-data` directory -
delete it if you want to start from scratch. To remove all traces of this test drive,
delete the whole `vm-quick-start` directory created at [step 1](https://docs.victoriametrics.com/victoriametrics/quick-start/#step-1-download-the-binary).
This test drive only scratches the surface of what VictoriaMetrics can do. Ready for a real setup?
Continue with the installation options below, or learn the [key concepts](https://docs.victoriametrics.com/victoriametrics/keyconcepts/)
of writing and querying data first.
## How to install
VictoriaMetrics is available in the following distributions:
@@ -214,6 +42,9 @@ Just download VictoriaMetrics and follow [these instructions](https://docs.victo
See [available integrations](https://docs.victoriametrics.com/victoriametrics/integrations/) with other systems like
[Prometheus](https://docs.victoriametrics.com/victoriametrics/integrations/prometheus/) or [Grafana](https://docs.victoriametrics.com/victoriametrics/integrations/grafana/).
> Want to see VictoriaMetrics in action, but without installing anything?
> Try [Playgrounds](https://docs.victoriametrics.com/playgrounds/) - a list of publicly available playgrounds for VictoriaMetrics software.
VictoriaMetrics is developed at a fast pace, so it is recommended to periodically check the [CHANGELOG](https://docs.victoriametrics.com/victoriametrics/changelog/)
and perform [regular upgrades](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-upgrade-victoriametrics).
@@ -229,9 +60,9 @@ Download the newest available [VictoriaMetrics release](https://docs.victoriamet
from [DockerHub](https://hub.docker.com/r/victoriametrics/victoria-metrics) or [Quay](https://quay.io/repository/victoriametrics/victoria-metrics?tab=tags):
```sh
docker pull victoriametrics/victoria-metrics:v1.148.0
docker pull victoriametrics/victoria-metrics:v1.147.0
docker run -it --rm -v `pwd`/victoria-metrics-data:/victoria-metrics-data -p 8428:8428 \
victoriametrics/victoria-metrics:v1.148.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
victoriametrics/victoria-metrics:v1.147.0 --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
```
_For Enterprise images, see [this link](https://docs.victoriametrics.com/victoriametrics/enterprise/#docker-images)._
@@ -634,15 +465,6 @@ It is recommended to read [Replication and data safety](https://docs.victoriamet
For backup configuration, please refer to [vmbackup documentation](https://docs.victoriametrics.com/victoriametrics/vmbackup/).
### Graceful shutdown
To gracefully shut down a VictoriaMetrics process, send SIGTERM or SIGINT signal and wait until the process exits.
See [how to send signals to processes](https://stackoverflow.com/questions/33239959/send-signal-to-process-from-command-line).
Graceful shutdown is required for data safety. A successful graceful shutdown guarantees that pending in-memory data and
ongoing writes are flushed on disk before the process exits. During graceful shutdown, VictoriaMetrics stops accepting
new HTTP connections and waits for in-flight requests to finish until `-http.maxGracefulShutdownDuration` expires.
### Configuring limits
To avoid excessive resource usage or performance degradation, limits must be in place:

View File

@@ -1405,6 +1405,9 @@ for fast block lookups, which belong to the given `TSID` and cover the given tim
* various background maintenance tasks such as [de-duplication](#deduplication), [downsampling](#downsampling)
and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge
See how `vmstorage` [selects parts for background merging](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#merge-process),
including merge limits and monitoring metrics.
Newly added `parts` either successfully appear in the storage or fail to appear.
The newly added `part` is atomically registered in the `parts.json` file under the corresponding partition
after it is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html) to the storage.
@@ -1532,6 +1535,8 @@ are **eventually deleted** during [background merge](https://medium.com/@valyala
The time range covered by data part is **not limited by retention period unit**. One data part can cover hours or days of
data. Hence, a data part can be deleted only **when fully outside the configured retention**.
See more about partitions and parts in the [Storage section](#storage).
See how the [retention and free-disk watchers manage storage](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#retention-free-disk-space-guard-and-downsampling)
for implementation details and monitoring metrics.
The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
For example, if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.

View File

@@ -26,53 +26,38 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
## [v1.148.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.148.0)
Released at 2026-07-20
* SECURITY: upgrade Go builder from Go1.26.4 to Go1.26.5. See [the list of issues addressed in Go1.26.5](https://github.com/golang/go/issues?q=milestone%3AGo1.26.5%20label%3ACherryPickApproved).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): support scraping metrics over Unix domain sockets. The socket path can be configured via the `__unix_socket__` target label. See [#11156](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11156). Thanks to @vinyas-bharadwaj for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Improve background discovery performance for [http_sd](https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs) discovery. See [#8838](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8838).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): allow overriding `max_scrape_size` on a per-target basis via the `__max_scrape_size__` label during target relabeling. See [#11188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11188).
* FEATURE: [vmstorage](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-maxBackfillAge` command-line flag for limiting ingestion of samples with historical timestamps, for example, when older data has been moved between storage tiers (nvme/hdd, hot/cold). See [#11199](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11199). Thanks to @AshwinRamaniPsg for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): automatically preload relabeling rules configured via `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` in the [metrics relabel debug UI](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging). See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Now drops metadata blocks when communicating with vmstorage nodes over the legacy RPC protocol. To avoid this limitation, upgrade `vmstorage` to a version that supports the new RPC protocol (>= [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11370)). See [#11146](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11146).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply limit to metrics metadata response. See [#11139](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11139).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a possible data race when processing OpenTelemetry metadata. See [#11238](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11238). Thanks to @nevgeny for contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): atomically write persistent queue metainfo to prevent possible file corruption on ungraceful shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix increased CPU and memory usage when `-remoteWrite.urlRelabelConfig` or `-remoteWrite.streamAggr.config` flags are used. The bug was introduced in [#10854](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10854) and existed since [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0). See [#11250](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11250).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve newline formatting in alert and rule annotations on the Alerting page. See [#11171](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11171).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): return `408 Request Timeout` instead of `400 Bad Request` when the request body isn't received within `-maxQueueDuration`. This prevents `vmagent` from incorrectly downgrading the remote write protocol and dropping data when `vmauth` is used as a proxy for a remote write endpoint. See [#11272](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11272).
## [v1.147.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.147.0)
Released at 2026-07-06
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): contains a bug that causes increased CPU and memory usage when `-remoteWrite.urlRelabelConfig` or `-remoteWrite.streamAggr.config` flags are used. The bug was introduced in [#10854](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10854). Upgrade to v1.148.0 or rollback to v1.146.0. See [#11250](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11250).
* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduces CPU usage by 10% at [sharding among remote storages](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages). See [#11113](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11113). Thanks to @bennf for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `64KiB` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): reduce CPU usage for storing scrape target labels. See [#10919](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10919).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): expose `vm_data_size_bytes{type="storage/metaindex"}` and `vm_data_size_bytes{type="indexdb/metaindex"}` metrics for tracking memory occupied by metaindex data. See [#11204](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11204). Thanks to @SamarthBagga for contribution.
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [#4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328), [#10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926), and the [documentation](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy).
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `InvalidAuthTokenRequestErrors` alerting rule to [vmauth alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmauth.yml). The new rule notifies when vmauth receives requests with invalid or missing auth tokens, which may indicate a client misconfiguration, expired token use, or brute-force attack. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328), [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926), and the [documentation](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#multi-tenancy).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow log requests with missing or invalid auth tokens to [access log](https://docs.victoriametrics.com/victoriametrics/vmauth/#access-log). This is useful for identifying `remote_addr` IPs performing brute-force attacks. See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): fall through to `unauthorized_user` when a [JWT token](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-token-auth-proxy) has no `vm_access` claim and no `default_vm_access_claim` is configured. Previously, vmauth returned `401 Unauthorized` immediately in this case, which prevented `unauthorized_user` from handling such requests. See [#5740](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5740).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): improve the selection algorithm of [buckets_limit](https://docs.victoriametrics.com/victoriametrics/metricsql/#buckets_limit) to remove consecutive empty buckets at the beginning and end to obtain more accurate min and max values. See [#10417](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10417).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): expose `vmalert_group_rule_results_limit` metric to indicate the number of alerts or recording results that a single rule within the group can produce. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
* FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): add `AlertingRuleResultsApproachingLimit` and `RecordingRuleResultsApproachingLimit` alerting rules to [vmalert alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmalert.yml). These alerts notify when a rule's last evaluation samples exceed 90% of the configured group results limit. See [#11179](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11179). Thanks to @vinyas-bharadwaj for the contribution.
* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly check values range for the limits configured with flags `-maxLabelsPerTimeseries`, `-maxLabelNameLen` and `-maxLabelValueLen`. It must be in range `1..65535`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
@@ -95,6 +80,7 @@ Released at 2026-06-22
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database. See [#11104](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11104).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly expose metric `vm_retention_filters_partitions_scheduled_rows`. See [#11138](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11138)
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
@@ -104,8 +90,8 @@ Released at 2026-06-22
* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): disallow restoring parts outside the configured `-storageDataPath` directory. See [710c920d](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/710c920d6083327042a309e449fae4383617d817).
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): correctly apply long tenant filters. Previously, such filters could be truncated, causing tenants to be matched incorrectly. See [#11096](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11096). Thanks for @fxrlv for the contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix corrupted metrics metadata when a response contains multiple rows. See [#11115](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11115). Thanks for @fxrlv for the contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): don't cache empty responses for tenant IDs discovery during [multitenant queries](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenant-reads). This problem was visible during integration tests when multitenant queries were executed before the first ingestion happened. See [#10982](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10982)
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): do not fail backup list if directory is absent while using `fs://` destination to align with other protocols. See [6c3c548](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/6c3c548ddb0385b749e731f52276f130e2a4e4a8)
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): don't cache empty responses for tenant IDs discovery during [multitenant queries](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#multitenant-reads). This problem was visible during integration tests when multitenant queries were executed before the first ingestion happened. See [#10982](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10982)
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
@@ -130,9 +116,9 @@ Released at 2026-06-08
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly calculate number of loaded users to be printed in startup log. Previously, it was only accounting for static users and skipped JWT configuration entries. See [#11050](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11050/).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): `integrate()` no longer extrapolates the last sample's value past the end of the time series. Previously, querying `integrate(metric[1h])` at a timestamp where the series had already ended would keep accruing area as if the last value continued indefinitely, producing values much larger than the true integral. See [#9474](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474). Thanks to @wtfashwin for contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid returning HTTP 503 for queries with partial results when a storage group is unavailable and `-search.denyPartialResponse` is disabled. See [#11009](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11009). Thanks to @fxrlv for the contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `utf-8` label names for [/federate](https://docs.victoriametrics.com/victoriametrics/#federation) API requests. See [#10968](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10968).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the `Disable deduplication` toggle under its own local storage key. Before this fix, the toggle state was lost after reload and could overwrite the `Compact view` table setting. See [#11004](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11004). Thanks to @immanuwell for the contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)
@@ -140,28 +126,28 @@ Released at 2026-05-22
* FEATURE: all VictoriaMetrics components: improve logging for the `-memory.allowedBytes` flag to warn about excessively low value (less than 1MB). See issue [#10935](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10935).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `basicAuth.usernameFile` command-line flags for reading basic auth username from a file, similar to the existing `basicAuth.passwordFile`. The file is re-read every second. See [#9436](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9436). Thanks to @kimjune01 for the contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-opentelemetry.labelNameUnderscoreSanitization` command-line flag to control whether to enable prepending of `key` to labels starting with `_` when `-opentelemetry.usePrometheusNaming` is enabled. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#9663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9663). Thanks to @andriibeee for the contribution.
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): improve the [Top Queries](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#top-queries) table UI. Duration columns now display human-readable values (e.g. `1.23s`) instead of raw seconds, memory column shows human-readable sizes (e.g. `1.23 MB`), instant queries are labeled as `instant` instead of empty string, and column headers now show tooltips with descriptions. See [#10790](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10790).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): drain in-memory remote write queue on shutdown within the 5-second grace period before falling back to persisting blocks to disk. See [#9996](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9996)
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Improve [slowness-based rerouting](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) to prevent rerouting storms under high cluster load. Previously, rerouting could cascade across storage nodes when the whole cluster was saturated, making the situation worse. Now rerouting only activates when the cluster p90 saturation is below 60%, and the slowest node is more than 20% slower than p90. See [#10876](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10876).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `{{.MetricsAccountID}}` and `{{.MetricsProjectID}}` [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating) for use in `headers` and `url_prefix` config fields. Previously, only the combined `{{.MetricsTenant}}` (`accountID:projectID`) JWT placeholder was supported, making it impossible to configure [multitenancy via headers](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers). See [#10927](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10927). Thanks to @Vinayak9769 for the contribution.
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): display `null` values on `Raw Query` chart. `null` values can be actual `NaN` or `null` values exposed by the exporter, or [stale markers](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers). Before, vmui Raw Query was silently dropping non-numeric values. Displaying such values on the chart could improve the debugging experience. See [#10986](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10986).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): stop emitting stale values for `quantiles(...)` outputs when a time series has no samples during the current aggregation interval. See [#10918](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10918). Thanks to @alexei38 for the contribution.
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): extend delay on aggregation windows flush by the biggest lag among pushed samples. Before, the delay was calculated as 95th percentile across samples, which could underrepresent outliers and reject them from aggregation as "too old". See [#10402](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10402).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a bug in [cardinality limiters](https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter) where series with different labels, like `{a="bc"}` and `{ab="c"}`, could be incorrectly treated as identical and dropped. See [#10937](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10937).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): hide values passed to `-remoteWrite.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive credentials.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): hide values passed to `-remoteWrite.headers`,`remoteRead.headers`, `datasource.headers` and `notifier.headers` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly establish [mtls](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#mtls-protection) connection between vmstorage and vminsert. Regression was introduced in v1.130.0 release for the enterprise version of vmstorage. See [#10972](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10972).
* BUGFIX: [vmrestore](https://docs.victoriametrics.com/victoriametrics/vmrestore/): fix a bug where specifying `-storageDataPath` with a trailing slash could cause `vmrestore` to panic. See [#10823](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10823). Thanks to @utafrali for the contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): prevent unintentional rerouting of samples to other sharding targets when one of the `-remoteWrite.url` targets with `-remoteWrite.disableOnDiskQueue` becomes blocked. Previously this could break the sharding guarantee by sending samples to wrong targets instead of dropping or retrying them. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return error on startup if `-remoteWrite.disableOnDiskQueue` is not configured uniformly across all `-remoteWrite.url` targets when `-remoteWrite.shardByURL` is enabled. Either all targets must have it enabled or all must have it disabled. See [#10507](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10507).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): hide values passed to `vmalert.proxyURL` in startup logs, `/metrics`, and `/flags`, since they can contain sensitive HTTP headers such as `Authorization` and API keys.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve exact series values in graph tooltips instead of rounding them by significant digits. See [#10952](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10952).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
* BUGFIX: all VictoriaMetrics components: fix int64 overflow when parsing [timestamp parameters](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats) with relative durations. See [#10880](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10880).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): `-denyQueriesOutsideRetention` now also rejects queries whose end time is beyond `-futureRetention`. See [#10879](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10879).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add missing `__timestamp__` and `__value__` columns to CSV exported from the table view on the Query tab. See [#10975](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10975).
## [v1.143.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.143.0)
@@ -171,10 +157,10 @@ Released at 2026-05-08
* FEATURE: all VictoriaMetrics components: suppress TCP health check errors when `-tls` flag is set. See [#10538](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10538).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `__meta_hetzner_robot_datacenter` label for `robot` role in [hetzner_sd_configs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs). See [#10909](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10909). Thanks to @juliusrickert for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-rule.stripFilePath` to support stripping rule file paths in logs and all API responses, including /metrics. See [#5625](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5625).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `formatTime` template function for formatting a Unix timestamp using the provided layout. For example, `{{ now | formatTime "2006-01-02T15:04:05Z07:00" }}` returns the current time in RFC3339 format. See issue [#10624](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10624). Thanks to @andriibeee for the contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add support for [Prometheus native histogram](https://prometheus.io/docs/specs/native_histograms/) during ingestion. See [#10743](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10743).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): introduce the `vm_fs_info` metric. It exposes the filesystem type (e.g., ext4, xfs, nfs) used for `-*Path` related flags. See [#10482](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10482).
* FEATURE: [dashboards/vmagent](https://grafana.com/grafana/dashboards/12683): add `Kafka (Enterprise)` row with panels for monitoring traffic (bytes), messages in/out, producer and consumer errors. See [#10728](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10728).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support sending data to the configured `-remoteWrite.url` via [VictoriaMetrics remote write protocol](https://docs.victoriametrics.com/victoriametrics/vmagent/#victoriametrics-remote-write-protocol). See [#10929](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10929).
@@ -347,27 +333,6 @@ It enables back `Discovered targets` debug UI by default.
* BUGFIX: `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply `extra_filters[]` filter when querying `vm_account_id` or `vm_project_id` labels via [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) request for `/api/v1/label/…/values` API. Before, `extra_filters` was ignored. See [#10503](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10503).
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): revert the use of rollup result cache for [instant queries](https://docs.victoriametrics.com/keyConcepts.html#instant-query) that contain [`rate`](https://docs.victoriametrics.com/MetricsQL.html#rate) function with a lookbehind window larger than `-search.minWindowForInstantRollupOptimization`. The cache usage was removed since [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.132.0). See [#10098](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10098#issuecomment-3895011084) for more details.
## [v1.136.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.14)
Released at 2026-07-17
**v1.136.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.136.x line will be supported for at least 12 months since [v1.136.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11360) release**
* SECURITY: upgrade Go builder from Go1.26.4 to Go1.26.5. See [the list of issues addressed in Go1.26.5](https://github.com/golang/go/issues?q=milestone%3AGo1.26.5%20label%3ACherryPickApproved).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): fix a possible data race when processing OpenTelemetry metadata. See [#11238](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11238). Thanks to @nevgeny for contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): atomically write persistent queue metainfo to prevent possible file corruption on ungraceful shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly apply limit to metrics metadata response. See [#11139](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11139).
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): Now drops metadata blocks when communicating with vmstorage nodes over the legacy RPC protocol. To avoid this limitation, upgrade `vmstorage` to a version that supports the new RPC protocol (>= [v1.137.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG.md#v11370)). See [#11146](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11146).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): keep only one header navigation dropdown (`Explore`, `Tools`) open at a time. Previously, hovering across two dropdowns could briefly leave both open due to the close delay. See [#11224](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11224). Thanks to @antedotee for contribution.
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve newline formatting in alert and rule annotations on the Alerting page. See [#11171](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11171).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): return `408 Request Timeout` instead of `400 Bad Request` when the request body isn't received within `-maxQueueDuration`. This prevents `vmagent` from incorrectly downgrading the remote write protocol and dropping data when `vmauth` is used as a proxy for a remote write endpoint. See [#11272](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11272).
## [v1.136.13](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.13)
Released at 2026-07-03
@@ -766,18 +731,6 @@ See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/ch
See changes [here](https://docs.victoriametrics.com/victoriametrics/changelog/changelog_2025/#v11230)
## [v1.122.27](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.27)
Released at 2026-07-17
**v1.122.x is a line of [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/). It contains important up-to-date bugfixes for [VictoriaMetrics enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/).
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): flush pending persistent queue data to chunk file before updating the metadata. This prevents the metadata writer offset from getting ahead of the chunk file size and avoids losing the persistent queue after an unclean shutdown. See [#11192](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11192).
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/victoriametrics/vmbackup/) and [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): retry S3 requests failing with `HTTP 429` status code or `TooManyRequests` error code. Previously such requests were not retried, so a short burst of rate limiting would fail the whole backup. See [#11218](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11218). Thanks to @gautamrizwani for contribution.
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): hide `Total metric names` stats on [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) page when user selects a specific metric or label to focus. See [#11154](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11154) for details. Thanks to @lghuy05 for the contribution.
## [v1.122.26](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.122.26)
Released at 2026-07-03

View File

@@ -121,7 +121,7 @@ It is allowed to run Enterprise components in [cases listed here](https://docs.v
Binary releases of Enterprise components are available at [the releases page for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest),
[the releases page for VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaLogs/releases/latest)
and [the releases page for VictoriaTraces](https://github.com/VictoriaMetrics/VictoriaTraces/releases/latest).
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz`.
Enterprise binaries and packages have `enterprise` suffix in their names. For example, `victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz`.
In order to run binary release of Enterprise component, please download the `*-enterprise.tar.gz` archive for your OS and architecture
from the corresponding releases page and unpack it. Then run the unpacked binary.
@@ -139,8 +139,8 @@ For example, the following command runs VictoriaMetrics Enterprise binary with t
obtained at [this page](https://victoriametrics.com/products/enterprise/trial/):
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz
tar -xzf victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz
./victoria-metrics-prod -license=BASE64_ENCODED_LICENSE_KEY
```
@@ -155,7 +155,7 @@ Alternatively, VictoriaMetrics Enterprise license can be stored in the file and
It is allowed to run Enterprise components in [cases listed here](https://docs.victoriametrics.com/victoriametrics/enterprise/#valid-cases-for-victoriametrics-enterprise).
Docker images for Enterprise components are available at [VictoriaMetrics Docker Hub](https://hub.docker.com/u/victoriametrics) and [VictoriaMetrics Quay](https://quay.io/organization/victoriametrics).
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.148.0-enterprise`.
Enterprise docker images have `enterprise` suffix in their names. For example, `victoriametrics/victoria-metrics:v1.147.0-enterprise`.
In order to run Docker image of VictoriaMetrics Enterprise component, it is required to provide the license key via the command-line
flag as described in the [binary-releases](https://docs.victoriametrics.com/victoriametrics/enterprise/#binary-releases) section.
@@ -165,13 +165,13 @@ Enterprise license key can be obtained at [this page](https://victoriametrics.co
For example, the following command runs VictoriaMetrics Enterprise Docker image with the specified license key:
```sh
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.148.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
docker run --name=victoria-metrics victoriametrics/victoria-metrics:v1.147.0-enterprise -license=BASE64_ENCODED_LICENSE_KEY
```
Alternatively, the license code can be stored in the file and then referred via `-licenseFile` command-line flag:
```sh
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.148.0-enterprise -licenseFile=/path/to/vm-license
docker run --name=victoria-metrics -v /vm-license:/vm-license victoriametrics/victoria-metrics:v1.147.0-enterprise -licenseFile=/path/to/vm-license
```
Example docker-compose configuration:
@@ -181,7 +181,7 @@ version: "3.5"
services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.148.0
image: victoriametrics/victoria-metrics:v1.147.0
ports:
- 8428:8428
volumes:
@@ -213,7 +213,7 @@ is used to provide the license key in plain-text:
```yaml
server:
image:
tag: v1.148.0-enterprise
tag: v1.147.0-enterprise
license:
key: {BASE64_ENCODED_LICENSE_KEY}
@@ -224,7 +224,7 @@ In order to provide the license key via existing secret, the following values fi
```yaml
server:
image:
tag: v1.148.0-enterprise
tag: v1.147.0-enterprise
license:
secret:
@@ -274,7 +274,7 @@ spec:
license:
key: {BASE64_ENCODED_LICENSE_KEY}
image:
tag: v1.148.0-enterprise
tag: v1.147.0-enterprise
```
In order to provide the license key via an existing secret, the following custom resource is used:
@@ -291,7 +291,7 @@ spec:
name: vm-license
key: license
image:
tag: v1.148.0-enterprise
tag: v1.147.0-enterprise
```
Example secret with license key:
@@ -342,7 +342,7 @@ Builds are available for amd64 and arm64 architectures.
Example archive:
`victoria-metrics-linux-amd64-v1.148.0-enterprise.tar.gz`
`victoria-metrics-linux-amd64-v1.147.0-enterprise.tar.gz`
Includes:
@@ -351,7 +351,7 @@ Includes:
Example Docker image:
`victoriametrics/victoria-metrics:v1.148.0-enterprise-fips` uses the FIPS-compatible binary and based on `scratch` image.
`victoriametrics/victoria-metrics:v1.147.0-enterprise-fips` uses the FIPS-compatible binary and based on `scratch` image.
## What Happens to Licensed Components When a License Expires

Binary file not shown.

Before

Width:  |  Height:  |  Size: 16 KiB

View File

@@ -513,7 +513,6 @@ URLs for scrape targets are composed of the following parts:
[scrape_config](https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs)
or by updating/setting the corresponding `__param_*` labels during
relabeling.
- Unix domain socket path (e.g. `/var/run/node_exporter.sock`) can be configured during target relabeling via a special label - `__unix_socket__`. If this label is set, `vmagent` tunnels the scrape request through the specified Unix domain socket instead of connecting to a TCP socket. The `__address__` label is still used to populate the `instance` label and construct the HTTP request URL, but the actual network connection goes through the Unix socket.
The resulting scrape URL looks like the following:

View File

@@ -35,8 +35,8 @@ scrape_configs:
After you created the `scrape.yaml` file, download and unpack [single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) to the same directory:
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.148.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.147.0.tar.gz
```
Then start VictoriaMetrics and instruct it to scrape targets defined in `scrape.yaml` and save scraped metrics
@@ -150,8 +150,8 @@ Then start [single-node VictoriaMetrics](https://docs.victoriametrics.com/victor
```yaml
# Download and unpack single-node VictoriaMetrics
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/victoria-metrics-linux-amd64-v1.148.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.148.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/victoria-metrics-linux-amd64-v1.147.0.tar.gz
tar xzf victoria-metrics-linux-amd64-v1.147.0.tar.gz
# Run single-node VictoriaMetrics with the given scrape.yaml
./victoria-metrics-prod -promscrape.config=scrape.yaml

View File

@@ -645,9 +645,9 @@ See also [why you shouldn't put an aggregator behind a load balancer](https://do
# Troubleshooting
- [Unexpected spikes for `total` or `increase` outputs](#staleness).
- [Unexpected spikes for `total` or `increase` outputs](#data-delay-and-staleness).
- [Excessively large values for `total*`, `increase*`, and `rate*` outputs](#counter-resets).
- [Lower than expected values for `total_prometheus` and `increase_prometheus` outputs](#staleness).
- [Lower than expected values for `total_prometheus` and `increase_prometheus` outputs](#data-delay-and-staleness).
- [High memory usage and CPU usage](#high-resource-usage).
- [Unexpected results in vmagent cluster mode](#cluster-mode).
- [Inaccurate aggregation results for histograms](#aggregation-windows)

View File

@@ -188,9 +188,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/
Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
-loggerWarnsPerSecondLimit int
Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-maxBackfillAge value
The maximum allowed age for the ingested samples with historical timestamps. Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 0)
-maxConcurrentInserts int
The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 2*cgroup.AvailableCPUs())
-maxIngestionRate int

View File

@@ -141,6 +141,7 @@ to other remote storage systems that support Prometheus `remote_write` protocol
If a single remote storage instance is temporarily unavailable, the collected data remains available on the other remote storage instances.
`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
Then it sends the buffered data to the remote storage in order to prevent data gaps.
See how `vmagent` [selects shards and places replicas](https://victoriametrics.com/blog/vmagent-how-it-works/#step-4-sharding--replication) for implementation details.
[VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) already supports replication,
so there is no need to specify multiple `-remoteWrite.url` flags when writing data to the same cluster.
@@ -150,7 +151,8 @@ See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victor
`vmagent` can add, remove, or update labels on the collected data before sending it to the remote storage.
It can filter scrape targets or remove unwanted samples via Prometheus-like relabeling.
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for details.
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for configuration examples.
For ingestion pipeline internals, see how `vmagent` [applies global relabeling and cardinality limits](https://victoriametrics.com/blog/vmagent-how-it-works/#step-2-global-relabeling-cardinality-reduction).
### Sharding among remote storages
@@ -268,7 +270,9 @@ for the collected samples. Examples:
```sh
./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
```
See how `vmagent` [orders global deduplication and stream aggregation](https://victoriametrics.com/blog/vmagent-how-it-works/#step-3-global-deduplication--stream-aggregation) in the ingestion pipeline.
### Monitoring Data eXchange
The Monitoring Data eXchange (MDX){{% available_from "v1.147.0" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
@@ -357,6 +361,8 @@ in addition to the pull-based Prometheus-compatible targets' scraping:
* Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
* Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data).
See how `vmagent` [handles concurrency, decompression, and stream parsing](https://victoriametrics.com/blog/vmagent-how-it-works/#step-1-receiving-data-via-api-or-scrape) during ingestion.
## How to collect metrics in Prometheus format
Specify the path to the `prometheus.yml` file via the `-promscrape.config` command-line flag. `vmagent` takes into account the following
@@ -998,6 +1004,7 @@ This behavior can be changed with the `-remoteWrite.inmemoryQueues` {{% availabl
When set to a non-zero value, vmagent starts the given number of additional workers,
which send only recently ingested data from the in-memory queue, while the workers configured via `-remoteWrite.queues` drain the file-based backlog concurrently.
This reduces the delivery lag for fresh samples after remote storage outages or slowdowns. The flag can be set individually per each `-remoteWrite.url`.
See how the [in-memory and file-based queues manage blocks](https://victoriametrics.com/blog/vmagent-how-it-works/#in-memory-queue) for implementation details.
Note that these workers are started in addition to the workers configured via `-remoteWrite.queues`, so the total number of concurrent connections to
the remote storage becomes the sum of both flags. Take this into account if the remote storage limits the number of concurrent requests.

View File

@@ -546,7 +546,7 @@ tags at [Docker Hub](https://hub.docker.com/r/victoriametrics/vmalert/tags) and
## Reading rules from object storage
The [Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
[Enterprise version](https://docs.victoriametrics.com/victoriametrics/enterprise/) of `vmalert` may read alerting and recording rules
from object storage:
* `./bin/vmalert -rule=s3://bucket/dir/alert.rules` would read rules from the given path at S3 bucket
@@ -563,48 +563,6 @@ The following [command-line flags](#flags) can be used for fine-tuning access to
* `-s3.customEndpoint` - custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set.
* `-s3.forcePathStyle` - prefixing endpoint with bucket name when set false, true by default.
### S3 (AWS and S3-compatible)
The following example reads rules from an S3 bucket:
```sh
./bin/vmalert \
-rule=s3://my-alert-bucket/rules/alerts_ \
-s3.credsFilePath=/etc/vmalert/aws-credentials \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
For S3-compatible backends such as [MinIO](https://www.min.io/) or [Ceph](https://ceph.io/), add `-s3.customEndpoint`:
```sh
./bin/vmalert \
-rule=s3://victoriametrics-alert-rules/rules_ \
-s3.credsFilePath=/etc/vmalert/aws-credentials \
-s3.customEndpoint=http://minio.example.local:9000 \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for details on creating IAM users, credentials file format, environment variables, and IAM roles.
### Google Cloud Storage (GCS)
The following example reads rules from a GCS bucket:
```sh
./bin/vmalert \
-rule=gs://my-alert-bucket/rules/alerts_ \
-s3.credsFilePath=/etc/vmalert/gcp-service-account.json \
-datasource.url=http://vmselect:8481/select/0/prometheus \
-notifier.url=http://alertmanager:9093 \
-licenseFile=/etc/vm-license
```
See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for details on creating service accounts and downloading JSON keys.
## Topology examples
The following sections are showing how `vmalert` may be used and configured
@@ -1024,6 +982,9 @@ Try the following tips to avoid common issues:
In that case, the default step will be used (`-datasource.queryStep`) and may cause unexpected results compared to
executing this query in vmui/Grafana, where step is adjusted differently.
See [practical examples for reducing alert noise](https://victoriametrics.com/blog/alerting-best-practices/#reducing-noise),
including aggregating alerts and configuring inhibition.
### Rule state
vmalert keeps the last `-rule.updateEntriesLimit` updates (or `update_entries_limit` [per-rule config](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules))
@@ -1079,6 +1040,9 @@ Sometimes, it's hard to understand why a specific alert fired or not. Keep in mi
If evaluation returns error (i.e. datasource is unavailable), alert state doesn't change.
If at least one evaluation returns no data, then alert's `for` state resets.
See [how to tune the `for` parameter](https://victoriametrics.com/blog/alerting-best-practices/#the-for-param),
including its tradeoff with the query lookbehind window.
> Note: The alert state is tracked separately for each time series returned during evaluation.
> For example, if the 1st evaluation returns series A and B, and the 2nd evaluation returns only B the alert will remain active **only for B**.

View File

@@ -204,45 +204,61 @@ See [this article](https://medium.com/@valyala/speeding-up-backups-for-big-time-
### Providing credentials as a file
See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for instructions on obtaining credentials and providing them via credential files, environment variables, cloud provider metadata service, or Kubernetes secrets and IAM roles.
Obtaining credentials from a file.
The following examples show the most common authentication patterns for each storage provider.
Add flag `-credsFilePath=/etc/credentials` with the following content:
#### S3 (AWS and S3-compatible)
* for S3 (AWS, MinIO or other S3 compatible storages):
```sh
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=s3://victoriametrics-backup/backup01 \
-credsFilePath=/etc/credentials
```
```sh
[default]
aws_access_key_id=theaccesskey
aws_secret_access_key=thesecretaccesskeyvalue
```
#### Google Cloud Storage (GCS)
* for GCP cloud storage:
```sh
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=gs://victoriametrics-backup/backup01 \
-credsFilePath=/etc/credentials
```
```json
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "key-id",
"private_key": "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n",
"client_email": "service-account-email",
"client_id": "client-id",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/service-account-email"
}
```
#### Azure Blob Storage
### Providing credentials via env variables
```sh
export AZURE_STORAGE_ACCOUNT_NAME=mystorageaccount
export AZURE_STORAGE_ACCOUNT_KEY=myaccountkey
Obtaining credentials from env variables.
vmbackup \
-storageDataPath=/data \
-snapshot.createURL=http://localhost:8428/snapshot/create \
-dst=azblob://victoriametrics-backup/backup01
```
* For AWS S3 compatible storages set env variable `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
Also you can set env variable `AWS_SHARED_CREDENTIALS_FILE` with path to credentials file.
* For GCE cloud storage set env variable `GOOGLE_APPLICATION_CREDENTIALS` with path to credentials file.
* For Azure storage use one of these env variables:
* `AZURE_STORAGE_ACCOUNT_CONNECTION_STRING`: use a connection string (must be either SAS Token or Account/Key)
* `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY`: use a specific account name and key (either primary or secondary)
* `AZURE_USE_DEFAULT_CREDENTIAL` and `AZURE_STORAGE_ACCOUNT_NAME`: use the `DefaultAzureCredential` to allow the Azure library
to search for multiple options (for example, managed identity related variables). Note that if multiple credentials are available,
it is required to specify the `AZURE_CLIENT_ID` to select specific credentials.
The `AZURE_STORAGE_DOMAIN` can be used for optionally overriding the default domain for the Azure storage service.
Please, note that `vmbackup` will use credentials provided by cloud providers metadata service [when applicable](https://docs.victoriametrics.com/victoriametrics/vmbackup/#using-cloud-providers-metadata-service).
### Using cloud providers metadata service
`vmbackup` and `vmbackupmanager` will automatically use cloud providers metadata service in order to obtain credentials if they are running in cloud environment and credentials are not explicitly provided via flags or env variables.
### Providing credentials in Kubernetes
The simplest way to provide credentials in Kubernetes is to use [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) and inject them into the pod as environment variables. For example, the following secret can be used for AWS S3 credentials:
The simplest way to provide credentials in Kubernetes is to use [Secrets](https://kubernetes.io/docs/concepts/configuration/secret/)
and inject them into the pod as environment variables. For example, the following secret can be used for AWS S3 credentials:
```yaml
apiVersion: v1
@@ -250,13 +266,14 @@ kind: Secret
metadata:
name: vmbackup-credentials
data:
access_key: <base64-encoded-key>
secret_key: <base64-encoded-secret>
access_key: key
secret_key: secret
```
And then it can be injected into the pod as environment variables:
```yaml
...
env:
- name: AWS_ACCESS_KEY_ID
valueFrom:
@@ -268,6 +285,7 @@ env:
secretKeyRef:
key: secret_key
name: vmbackup-credentials
...
```
A more secure way is to use IAM roles to provide tokens for pods instead of managing credentials manually.

View File

@@ -88,21 +88,34 @@ There are two flags which could help with performance tuning:
### Example of Usage
The following command backs up data to a GCS bucket:
GCS and cluster version. You need to have a credentials file in json format with following structure:
credentials.json
```json
{
"type": "service_account",
"project_id": "<project>",
"private_key_id": "",
"private_key": "-----BEGIN PRIVATE KEY-----\-----END PRIVATE KEY-----\n",
"client_email": "test@<project>.iam.gserviceaccount.com",
"client_id": "",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40<project>.iam.gserviceaccount.com"
}
```
Backup manager launched with the following configuration:
```sh
export NODE_IP=192.168.0.10
export VMSTORAGE_ENDPOINT=http://127.0.0.1:8428
./vmbackupmanager \
-dst=gs://vmstorage-data/$NODE_IP \
-credsFilePath=credentials.json \
-storageDataPath=/vmstorage-data \
-snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create \
-licenseFile=/path/to/vm-license
./vmbackupmanager -dst=gs://vmstorage-data/$NODE_IP -credsFilePath=credentials.json -storageDataPath=/vmstorage-data -snapshot.createURL=$VMSTORAGE_ENDPOINT/snapshot/create -licenseFile=/path/to/vm-license
```
See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for instructions on obtaining credentials and configuring access to S3, GCS, and Azure Blob Storage.
Expected logs in vmbackupmanager:
```sh
@@ -134,7 +147,8 @@ objects. Typical object storage systems implement server-side copy by creating n
This is very fast and efficient. Unfortunately there are systems such as [S3 Glacier](https://aws.amazon.com/s3/storage-classes/glacier/),
which perform full object copy during server-side copying. This may be slow and expensive.
See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for more examples of authentication with different storage types.
Please, see [vmbackup docs](https://docs.victoriametrics.com/victoriametrics/vmbackup/#advanced-usage) for more examples of authentication with different
storage types.
### Backup Retention Policy

View File

@@ -34,9 +34,9 @@ vmctl command-line tool is available as:
Download and unpack vmctl:
```sh
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.148.0/vmutils-darwin-arm64-v1.148.0.tar.gz
wget https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.147.0/vmutils-darwin-arm64-v1.147.0.tar.gz
tar xzf vmutils-darwin-arm64-v1.148.0.tar.gz
tar xzf vmutils-darwin-arm64-v1.147.0.tar.gz
```
Once binary is unpacked, see the full list of supported modes by running the following command:

View File

@@ -47,13 +47,13 @@ i.e. the end result would be similar to [rsync --delete](https://askubuntu.com/q
## Troubleshooting
* See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for instructions on configuring credentials.
* See [how to setup credentials via environment variables](https://docs.victoriametrics.com/victoriametrics/vmbackup/#providing-credentials-via-env-variables).
* If `vmrestore` consumes all the network bandwidth, then set `-maxBytesPerSecond` to the desired value.
* If `vmrestore` has been interrupted due to temporary error, then just restart it with the same args. It will resume the restore process.
## Advanced usage
See [Connecting VM components to cloud storage](https://docs.victoriametrics.com/guides/connecting-vm-components-to-cloud-storage/) for examples of authentication
Please, see [vmbackup docs](https://docs.victoriametrics.com/victoriametrics/vmbackup/#advanced-usage) for examples of authentication
with different storage types.
Run `vmrestore -help` in order to see all the available options:

View File

@@ -41,8 +41,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Flag value can be read from the given http/https url when using -deleteAuthKey=http://host/path or -deleteAuthKey=https://host/path
-denyQueryTracing
Whether to disable the ability to trace queries. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#query-tracing
-enableMetadata
Whether to enable metadata processing for metrics scraped from targets, received via VictoriaMetrics remote write, Prometheus remote write v1 or OpenTelemetry protocol. See also remoteWrite.maxMetadataPerBlock (default true)
-enableMultitenancyViaHeaders
Enables multitenancy via HTTP headers. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy-via-headers
-enableTCP6
@@ -105,8 +103,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Whether to use proxy protocol for connections accepted at the given -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt . With enabled proxy protocol http server cannot serve regular /metrics endpoint. Use -pushmetrics.url for metrics pushing
Supports array of values separated by comma or specified via multiple flags.
Empty values are set to false.
-insert.maxQueueDuration duration
The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
-internStringCacheExpireDuration duration
The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
-internStringDisableCache
@@ -131,8 +127,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
-loggerWarnsPerSecondLimit int
Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-maxConcurrentInserts int
The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 24)
-memory.allowedBytes size
Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from the OS page cache resulting in higher disk IO usage. The process may behave unexpectedly if this flag is set too small (e.g., 1 byte).
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0)
@@ -154,119 +148,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Flag value can be read from the given http/https url when using -pprofAuthKey=http://host/path or -pprofAuthKey=https://host/path
-prevCacheRemovalPercent float
Items in the previous caches are removed when the percent of requests it serves becomes lower than this value. Higher values reduce memory usage at the cost of higher CPU usage. See also -cacheExpireDuration (default 0.1)
-promscrape.azureSDCheckInterval duration
Interval for checking for changes in Azure. This works only if azure_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#azure_sd_configs for details (default 1m0s)
-promscrape.cluster.memberLabel string
If non-empty, then the label with this name and the -promscrape.cluster.memberNum value is added to all the scraped metrics. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info
-promscrape.cluster.memberNum string
The number of vmagent instance in the cluster of scrapers. It must be a unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster. Can be specified as pod name of Kubernetes StatefulSet - pod-name-Num, where Num is a numeric part of pod name. See also -promscrape.cluster.memberLabel . See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info (default "0")
-promscrape.cluster.memberURLTemplate string
An optional template for URL to access vmagent instance with the given -promscrape.cluster.memberNum value. Every %d occurrence in the template is substituted with -promscrape.cluster.memberNum at urls to vmagent instances responsible for scraping the given target at /service-discovery page. For example -promscrape.cluster.memberURLTemplate='http://vmagent-%d:8429/targets'. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more details
-promscrape.cluster.membersCount int
The number of members in a cluster of scrapers. Each member must have a unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default, cluster scraping is disabled, i.e. a single scraper scrapes all the targets. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info (default 1)
-promscrape.cluster.name string
Optional name of the cluster. If multiple vmagent clusters scrape the same targets, then each cluster must have unique name in order to properly de-duplicate samples received from these clusters. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info
-promscrape.cluster.replicationFactor int
The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 1, then the deduplication must be enabled at remote storage side. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info (default 1)
-promscrape.cluster.shardByLabels array
Optional list of target labels, which will be used for sharding targets among cluster members if -promscrape.cluster.membersCount is greater than 1. If none of the specified labels are found in a target, then all the target labels will be used for sharding. See https://docs.victoriametrics.com/victoriametrics/vmagent/#scraping-big-number-of-targets for more info
Supports an array of values separated by comma or specified via multiple flags.
Each array item can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
-promscrape.config string
Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. The path can point to local file and to http url. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
-promscrape.config.dryRun
Checks -promscrape.config file for errors and unsupported fields and then exits. Returns non-zero exit code on parsing errors and emits these errors to stderr. See also -promscrape.config.strictParse command-line flag. Pass -loggerLevel=ERROR if you don't need to see info messages in the output.
-promscrape.config.strictParse
Whether to deny unsupported fields in -promscrape.config . Set to false in order to silently skip unsupported fields (default true)
-promscrape.configCheckInterval duration
Interval for checking for changes in -promscrape.config file. By default, the checking is disabled. See how to reload -promscrape.config file at https://docs.victoriametrics.com/victoriametrics/vmagent/#configuration-update
-promscrape.consul.waitTime duration
Wait time used by Consul service discovery. Default value is used if not set
-promscrape.consulSDCheckInterval duration
Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#consul_sd_configs for details (default 30s)
-promscrape.consulagentSDCheckInterval duration
Interval for checking for changes in Consul Agent. This works only if consulagent_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#consulagent_sd_configs for details (default 30s)
-promscrape.digitaloceanSDCheckInterval duration
Interval for checking for changes in digital ocean. This works only if digitalocean_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#digitalocean_sd_configs for details (default 1m0s)
-promscrape.disableCompression
Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine-grained control
-promscrape.disableKeepAlive
Whether to disable HTTP keep-alive connections when scraping all the targets. This may be useful when targets has no support for HTTP keep-alive connection. It is possible to set 'disable_keepalive: true' individually per each 'scrape_config' section in '-promscrape.config' for fine-grained control. Note that disabling HTTP keep-alive may increase load on both vmagent and scrape targets
-promscrape.discovery.concurrency int
The maximum number of concurrent requests to Prometheus autodiscovery API (Consul, Kubernetes, etc.) (default 100)
-promscrape.discovery.concurrentWaitTime duration
The maximum duration for waiting to perform API requests if more than -promscrape.discovery.concurrency requests are simultaneously performed (default 1m0s)
-promscrape.dnsSDCheckInterval duration
Interval for checking for changes in dns. This works only if dns_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#dns_sd_configs for details (default 30s)
-promscrape.dockerSDCheckInterval duration
Interval for checking for changes in docker. This works only if docker_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#docker_sd_configs for details (default 30s)
-promscrape.dockerswarmSDCheckInterval duration
Interval for checking for changes in dockerswarm. This works only if dockerswarm_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#dockerswarm_sd_configs for details (default 30s)
-promscrape.dropOriginalLabels
Whether to drop original labels for scrape targets at /targets and /api/v1/targets pages. This may be needed for reducing memory usage when original labels for big number of scrape targets occupy big amounts of memory. Note that this reduces debuggability for improper per-target relabeling configs
-promscrape.ec2SDCheckInterval duration
Interval for checking for changes in ec2. This works only if ec2_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#ec2_sd_configs for details (default 1m0s)
-promscrape.eurekaSDCheckInterval duration
Interval for checking for changes in eureka. This works only if eureka_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#eureka_sd_configs for details (default 30s)
-promscrape.fileSDCheckInterval duration
Interval for checking for changes in 'file_sd_config'. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#file_sd_configs for details (default 1m0s)
-promscrape.gceSDCheckInterval duration
Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#gce_sd_configs for details (default 1m0s)
-promscrape.hetznerSDCheckInterval duration
Interval for checking for changes in Hetzner API. This works only if hetzner_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#hetzner_sd_configs for details (default 1m0s)
-promscrape.httpSDCheckInterval duration
Interval for checking for changes in http endpoint service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#http_sd_configs for details (default 1m0s)
-promscrape.kubernetes.apiServerTimeout duration
How frequently to reload the full state from Kubernetes API server (default 30m0s)
-promscrape.kubernetes.attachNamespaceMetadataAll
Whether to set attach_metadata.namespace=true for all the kubernetes_sd_configs at -promscrape.config . It is possible to set attach_metadata.namespace=false individually per each kubernetes_sd_configs . See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs
-promscrape.kubernetes.attachNodeMetadataAll
Whether to set attach_metadata.node=true for all the kubernetes_sd_configs at -promscrape.config . It is possible to set attach_metadata.node=false individually per each kubernetes_sd_configs . See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs
-promscrape.kubernetes.useHTTP2Client
Whether to use HTTP/2 client for connection to Kubernetes API server. This may reduce amount of concurrent connections to API server when watching for a big number of Kubernetes objects.
-promscrape.kubernetesSDCheckInterval duration
Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kubernetes_sd_configs for details (default 30s)
-promscrape.kumaSDCheckInterval duration
Interval for checking for changes in kuma service discovery. This works only if kuma_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#kuma_sd_configs for details (default 30s)
-promscrape.marathonSDCheckInterval duration
Interval for checking for changes in Marathon REST API. This works only if marathon_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#marathon_sd_configs for details (default 30s)
-promscrape.maxDroppedTargets int
The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 10000)
-promscrape.maxResponseHeadersSize size
The maximum size of http response headers from Prometheus scrape targets
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 4096)
-promscrape.maxScrapeSize size
The maximum size of uncompressed scrape response in bytes to process from Prometheus targets. Bigger uncompressed responses are rejected. See also max_scrape_size option at https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 16777216)
-promscrape.minResponseSizeForStreamParse size
The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/victoriametrics/vmagent/#stream-parsing-mode
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000)
-promscrape.noStaleMarkers
Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
-promscrape.nomad.waitTime duration
Wait time used by Nomad service discovery. Default value is used if not set
-promscrape.nomadSDCheckInterval duration
Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#nomad_sd_configs for details (default 30s)
-promscrape.openstackSDCheckInterval duration
Interval for checking for changes in openstack API server. This works only if openstack_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#openstack_sd_configs for details (default 30s)
-promscrape.ovhcloudSDCheckInterval duration
Interval for checking for changes in OVH Cloud API. This works only if ovhcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#ovhcloud_sd_configs for details (default 30s)
-promscrape.puppetdbSDCheckInterval duration
Interval for checking for changes in PuppetDB API. This works only if puppetdb_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#puppetdb_sd_configs for details (default 30s)
-promscrape.seriesLimitPerTarget int
Optional limit on the number of unique time series a single scrape target can expose. See https://docs.victoriametrics.com/victoriametrics/vmagent/#cardinality-limiter for more info
-promscrape.streamParse
Whether to enable stream parsing for metrics obtained from scrape targets. This may be useful for reducing memory usage when millions of metrics are exposed per each scrape target. It is possible to set 'stream_parse: true' individually per each 'scrape_config' section in '-promscrape.config' for fine-grained control
-promscrape.suppressDuplicateScrapeTargetErrors
Whether to suppress 'duplicate scrape target' errors; see https://docs.victoriametrics.com/victoriametrics/vmagent/#troubleshooting for details
-promscrape.suppressScrapeErrors
Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay
-promscrape.suppressScrapeErrorsDelay duration
The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors
-promscrape.vultrSDCheckInterval duration
Interval for checking for changes in Vultr. This works only if vultr_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#vultr_sd_configs for details (default 30s)
-promscrape.yandexcloudSDCheckInterval duration
Interval for checking for changes in Yandex Cloud API. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs for details (default 30s)
-pushmetrics.disableCompression
Whether to disable request body compression when pushing metrics to every -pushmetrics.url
-pushmetrics.extraLabel array

View File

@@ -130,9 +130,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/cluster-victori
Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC")
-loggerWarnsPerSecondLimit int
Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-maxBackfillAge value
The maximum allowed age for the ingested samples with historical timestamps. Samples with timestamps older than now-maxBackfillAge are rejected during data ingestion. By default, or when set to 0, -maxBackfillAge equals to -retentionPeriod, e.g. it is unlimited within the configured retention. This can be useful for limiting ingestion of historical samples, for example, when older data has been moved to another storage tier. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention
The following optional suffixes are supported: s (second), h (hour), d (day), w (week), M (month), y (year). If suffix isn't set, then the duration is counted in months (default 0)
-maxConcurrentInserts int
The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 2*cgroup.AvailableCPUs())
-memory.allowedBytes size

2
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/VictoriaMetrics/VictoriaMetrics
go 1.26.5
go 1.26.4
require (
cloud.google.com/go/storage v1.62.3

View File

@@ -648,7 +648,7 @@ func (mi *metainfo) WriteToFile(path string) error {
if err != nil {
return fmt.Errorf("cannot marshal persistent queue metainfo %#v: %w", mi, err)
}
fs.MustWriteAtomic(path, data, true)
fs.MustWriteSync(path, data)
return nil
}

View File

@@ -9,48 +9,47 @@ import (
)
// WriteMetricRelabelDebug writes /metric-relabel-debug page to w with the corresponding args.
func WriteMetricRelabelDebug(w io.Writer, targetID, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, format string, err error) {
writeRelabelDebug(w, false, targetID, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, format, err)
func WriteMetricRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, format string, err error) {
writeRelabelDebug(w, false, targetID, metric, relabelConfigs, format, err)
}
// WriteTargetRelabelDebug writes /target-relabel-debug page to w with the corresponding args.
func WriteTargetRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, format string, err error) {
writeRelabelDebug(w, true, targetID, metric, relabelConfigs, 0, 0, format, err)
writeRelabelDebug(w, true, targetID, metric, relabelConfigs, format, err)
}
func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, format string, err error) {
func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, relabelConfigs, format string, err error) {
if metric == "" {
metric = "{}"
}
targetURL := ""
if err != nil {
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
return
}
metric, err = normalizeInputLabels(metric)
if err != nil {
err = fmt.Errorf("cannot parse metric: %w", err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
return
}
labels, err := promutil.NewLabelsFromString(metric)
if err != nil {
err = fmt.Errorf("cannot parse metric: %w", err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
return
}
pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
if err != nil {
err = fmt.Errorf("cannot parse relabel configs: %w", err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
return
}
dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, nil)
WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, nil)
}
func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutil.Labels, isTargetRelabel bool) ([]DebugStep, string) {
@@ -141,10 +140,6 @@ func getChangedLabelNames(in, out *promutil.Labels) map[string]struct{} {
func normalizeInputLabels(metric string) (string, error) {
metric = strings.TrimSpace(metric)
if strings.ContainsRune(metric, '\n') {
return metric, fmt.Errorf("only one time series is allowed; got multiple lines")
}
openBrace := strings.Contains(metric, `{`)
closeBrace := strings.Contains(metric, `}`)

View File

@@ -6,66 +6,37 @@
{% stripspace %}
{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) %}
{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) %}
{% if format == "json" %}
{%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err) %}
{%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, err) %}
{% else %}
{%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err) %}
{%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, err) %}
{% endif %}
{% endfunc %}
{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) %}
{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
<!DOCTYPE html>
<html lang="en">
<head>
{%= htmlcomponents.CommonHeader() %}
<title>Metric relabel debug</title>
<script>
function setRelabelDebugFormMethod(form) {
form.method = (form.elements["relabel_configs"].value.length + form.elements["metric"].value.length > 1000) ? "POST" : "GET";
}
function reloadRelabelConfigs(select) {
if (!confirm('Reload will discard all modifications to the current configuration. Continue?')) {
select.value = select.prevValue;
return;
function submitRelabelDebugForm(e) {
var form = e.target;
var method = "GET";
if (form.elements["relabel_configs"].value.length + form.elements["metric"].value.length > 1000) {
method = "POST";
}
document.getElementById('reload_url_relabel_configs').value = '1';
setRelabelDebugFormMethod(select.form);
select.form.submit();
form.method = method;
}
function initRelabelConfigsHighlight() {
var ta = document.getElementById('relabel-configs-input');
var bd = document.getElementById('relabel-configs-backdrop');
if (!ta || !bd) return;
function escapeHtml(s) {
return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
}
function highlight(text) {
return text.split('\n').map(function(line) {
var e = escapeHtml(line);
return /^\s*#/.test(line)
? '<span style="color:#999">'+e+'</span>'
: '<span style="color:#212529">'+e+'</span>';
}).join('\n');
}
function update() { bd.innerHTML = highlight(ta.value)+'\n'; }
ta.addEventListener('input', update);
ta.addEventListener('scroll', function() { bd.scrollTop = ta.scrollTop; });
update();
}
document.addEventListener('DOMContentLoaded', initRelabelConfigsHighlight);
</script>
</head>
<body>
{%= htmlcomponents.Navbar() %}
<div class="container-fluid">
<a href="https://docs.victoriametrics.com/victoriametrics/relabeling/" target="_blank">Relabeling Cookbook</a>{% space %}|{% space %}
<a href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages" target="_blank">Relabeling Stages</a>
<a href="https://docs.victoriametrics.com/victoriametrics/relabeling/" target="_blank">Relabeling docs</a>{% space %}
{% if targetID != "" %}
{% space %}|{% space %}
{% if targetURL != "" %}
<a href="metric-relabel-debug?id={%s targetID %}">Metric relabel debug</a>
{% else %}
@@ -79,14 +50,14 @@ document.addEventListener('DOMContentLoaded', initRelabelConfigsHighlight);
{% endif %}
<div class="m-3">
<form method="POST" onsubmit="setRelabelDebugFormMethod(this)">
{%= relabelDebugFormInputs(metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, targetID) %}
<form method="POST" onsubmit="submitRelabelDebugForm(event)">
{%= relabelDebugFormInputs(metric, relabelConfigs) %}
{% if targetID != "" %}
<input type="hidden" name="id" value="{%s targetID %}" />
{% endif %}
<input type="submit" value="Submit" class="btn btn-primary m-1" />
{% if targetID != "" %}
<button type="button" onclick="location.href='?id={%s targetID %}'" class="btn btn-secondary m-1">Reset All</button>
<button type="button" onclick="location.href='?id={%s targetID %}'" class="btn btn-secondary m-1">Reset</button>
{% endif %}
</form>
</div>
@@ -101,44 +72,15 @@ document.addEventListener('DOMContentLoaded', initRelabelConfigsHighlight);
</html>
{% endfunc %}
{% func relabelDebugFormInputs(metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, targetID string) %}
{% func relabelDebugFormInputs(metric, relabelConfigs string) %}
<div>
<!-- show remote write relabel reload only for scrape metric relabel debug and pure relabel debug. discovery debug should not display this section -->
{% if !isTargetRelabel %}
<div>
<div class="m-1">
<div class="d-flex align-items-center gap-2 mt-1">
Configs:
{% if urlRelabelIndexLength > 0 %}
<input type="hidden" name="reload_url_relabel_configs" id="reload_url_relabel_configs" value="" />
<select name="url_relabel_configs_index" class="form-select form-select-sm w-auto" onfocus="this.prevValue=this.value" onchange="reloadRelabelConfigs(this)">
{% for i := range urlRelabelIndexLength %}
{% if urlRelabelIndexCurrent == i %}
<option value="{%d i %}" selected="selected">remote-write-url-{%d i %}</option>
{% else %}
<option value="{%d i %}">remote-write-url-{%d i %}</option>
{% endif %}
{% endfor %}
</select>
{% endif %}
</div>
</div>
</div>
{% else %}
Configs:
{% endif %}
<!-- the following text area css was generated with the help of AI to display yaml comments (starting with #) in gray. it could be rewritten in the future -->
<div class="m-1" style="position:relative;height:15em;">
<div id="relabel-configs-backdrop" style="position:absolute;top:0;left:0;right:0;bottom:0;pointer-events:none;overflow:hidden;font-family:monospace;white-space:pre-wrap;padding:0.375rem 0.75rem;border:1px solid transparent;"></div>
<textarea id="relabel-configs-input" name="relabel_configs" class="form-control" style="position:absolute;top:0;left:0;height:100%;font-family:monospace;color:transparent;caret-color:#212529;background:transparent;resize:none;overflow-y:scroll;">
{%s relabelConfigs %}
</textarea>
</div>
Relabel configs:<br/>
<textarea name="relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">{%s relabelConfigs %}</textarea>
</div>
<div class="mt-2">
A Time Series:<br/>
<textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1" placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}">{%s metric %}</textarea>
<div>
Labels:<br/>
<textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">{%s metric %}</textarea>
</div>
{% endfunc %}
@@ -211,7 +153,7 @@ document.addEventListener('DOMContentLoaded', initRelabelConfigsHighlight);
{% endif %}
{% endfunc %}
{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) %}
{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
{
{% if err != nil %}
"status": "error",

View File

@@ -25,37 +25,37 @@ var (
)
//line lib/promrelabel/debug.qtpl:9
func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) {
func StreamRelabelDebugSteps(qw422016 *qt422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) {
//line lib/promrelabel/debug.qtpl:10
if format == "json" {
//line lib/promrelabel/debug.qtpl:11
StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:12
} else {
//line lib/promrelabel/debug.qtpl:13
StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:14
}
//line lib/promrelabel/debug.qtpl:15
}
//line lib/promrelabel/debug.qtpl:15
func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) {
func WriteRelabelDebugSteps(qq422016 qtio422016.Writer, targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) {
//line lib/promrelabel/debug.qtpl:15
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:15
StreamRelabelDebugSteps(qw422016, targetURL, targetID, format, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
StreamRelabelDebugSteps(qw422016, targetURL, targetID, format, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:15
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:15
}
//line lib/promrelabel/debug.qtpl:15
func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) string {
func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) string {
//line lib/promrelabel/debug.qtpl:15
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:15
WriteRelabelDebugSteps(qb422016, targetURL, targetID, format, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
WriteRelabelDebugSteps(qb422016, targetURL, targetID, format, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:15
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:15
@@ -66,495 +66,431 @@ func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metr
}
//line lib/promrelabel/debug.qtpl:17
func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) {
func StreamRelabelDebugStepsHTML(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
//line lib/promrelabel/debug.qtpl:17
qw422016.N().S(`<!DOCTYPE html><html lang="en"><head>`)
//line lib/promrelabel/debug.qtpl:21
htmlcomponents.StreamCommonHeader(qw422016)
//line lib/promrelabel/debug.qtpl:21
qw422016.N().S(`<title>Metric relabel debug</title><script>function setRelabelDebugFormMethod(form) {form.method = (form.elements["relabel_configs"].value.length + form.elements["metric"].value.length > 1000) ? "POST" : "GET";}function reloadRelabelConfigs(select) {if (!confirm('Reload will discard all modifications to the current configuration. Continue?')) {select.value = select.prevValue;return;}document.getElementById('reload_url_relabel_configs').value = '1';setRelabelDebugFormMethod(select.form);select.form.submit();}function initRelabelConfigsHighlight() {var ta = document.getElementById('relabel-configs-input');var bd = document.getElementById('relabel-configs-backdrop');if (!ta || !bd) return;function escapeHtml(s) {return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');}function highlight(text) {return text.split('\n').map(function(line) {var e = escapeHtml(line);return /^\s*#/.test(line)? '<span style="color:#999">'+e+'</span>': '<span style="color:#212529">'+e+'</span>';}).join('\n');}function update() { bd.innerHTML = highlight(ta.value)+'\n'; }ta.addEventListener('input', update);ta.addEventListener('scroll', function() { bd.scrollTop = ta.scrollTop; });update();}document.addEventListener('DOMContentLoaded', initRelabelConfigsHighlight);</script></head><body>`)
//line lib/promrelabel/debug.qtpl:62
qw422016.N().S(`<title>Metric relabel debug</title><script>function submitRelabelDebugForm(e) {var form = e.target;var method = "GET";if (form.elements["relabel_configs"].value.length + form.elements["metric"].value.length > 1000) {method = "POST";}form.method = method;}</script></head><body>`)
//line lib/promrelabel/debug.qtpl:35
htmlcomponents.StreamNavbar(qw422016)
//line lib/promrelabel/debug.qtpl:62
qw422016.N().S(`<div class="container-fluid"><a href="https://docs.victoriametrics.com/victoriametrics/relabeling/" target="_blank">Relabeling Cookbook</a>`)
//line lib/promrelabel/debug.qtpl:64
//line lib/promrelabel/debug.qtpl:35
qw422016.N().S(`<div class="container-fluid"><a href="https://docs.victoriametrics.com/victoriametrics/relabeling/" target="_blank">Relabeling docs</a>`)
//line lib/promrelabel/debug.qtpl:37
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:64
qw422016.N().S(`|`)
//line lib/promrelabel/debug.qtpl:64
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:64
qw422016.N().S(`<a href="https://docs.victoriametrics.com/victoriametrics/relabeling/#relabeling-stages" target="_blank">Relabeling Stages</a>`)
//line lib/promrelabel/debug.qtpl:67
//line lib/promrelabel/debug.qtpl:39
if targetID != "" {
//line lib/promrelabel/debug.qtpl:68
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:68
qw422016.N().S(`|`)
//line lib/promrelabel/debug.qtpl:68
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:69
//line lib/promrelabel/debug.qtpl:40
if targetURL != "" {
//line lib/promrelabel/debug.qtpl:69
//line lib/promrelabel/debug.qtpl:40
qw422016.N().S(`<a href="metric-relabel-debug?id=`)
//line lib/promrelabel/debug.qtpl:70
//line lib/promrelabel/debug.qtpl:41
qw422016.E().S(targetID)
//line lib/promrelabel/debug.qtpl:70
//line lib/promrelabel/debug.qtpl:41
qw422016.N().S(`">Metric relabel debug</a>`)
//line lib/promrelabel/debug.qtpl:71
//line lib/promrelabel/debug.qtpl:42
} else {
//line lib/promrelabel/debug.qtpl:71
//line lib/promrelabel/debug.qtpl:42
qw422016.N().S(`<a href="target-relabel-debug?id=`)
//line lib/promrelabel/debug.qtpl:72
//line lib/promrelabel/debug.qtpl:43
qw422016.E().S(targetID)
//line lib/promrelabel/debug.qtpl:72
//line lib/promrelabel/debug.qtpl:43
qw422016.N().S(`">Target relabel debug</a>`)
//line lib/promrelabel/debug.qtpl:73
//line lib/promrelabel/debug.qtpl:44
}
//line lib/promrelabel/debug.qtpl:74
//line lib/promrelabel/debug.qtpl:45
}
//line lib/promrelabel/debug.qtpl:74
//line lib/promrelabel/debug.qtpl:45
qw422016.N().S(`<br>`)
//line lib/promrelabel/debug.qtpl:77
//line lib/promrelabel/debug.qtpl:48
if err != nil {
//line lib/promrelabel/debug.qtpl:78
//line lib/promrelabel/debug.qtpl:49
htmlcomponents.StreamErrorNotification(qw422016, err)
//line lib/promrelabel/debug.qtpl:79
//line lib/promrelabel/debug.qtpl:50
}
//line lib/promrelabel/debug.qtpl:79
qw422016.N().S(`<div class="m-3"><form method="POST" onsubmit="setRelabelDebugFormMethod(this)">`)
//line lib/promrelabel/debug.qtpl:83
streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, targetID)
//line lib/promrelabel/debug.qtpl:84
//line lib/promrelabel/debug.qtpl:50
qw422016.N().S(`<div class="m-3"><form method="POST" onsubmit="submitRelabelDebugForm(event)">`)
//line lib/promrelabel/debug.qtpl:54
streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
//line lib/promrelabel/debug.qtpl:55
if targetID != "" {
//line lib/promrelabel/debug.qtpl:84
//line lib/promrelabel/debug.qtpl:55
qw422016.N().S(`<input type="hidden" name="id" value="`)
//line lib/promrelabel/debug.qtpl:85
//line lib/promrelabel/debug.qtpl:56
qw422016.E().S(targetID)
//line lib/promrelabel/debug.qtpl:85
//line lib/promrelabel/debug.qtpl:56
qw422016.N().S(`" />`)
//line lib/promrelabel/debug.qtpl:86
//line lib/promrelabel/debug.qtpl:57
}
//line lib/promrelabel/debug.qtpl:86
//line lib/promrelabel/debug.qtpl:57
qw422016.N().S(`<input type="submit" value="Submit" class="btn btn-primary m-1" />`)
//line lib/promrelabel/debug.qtpl:88
//line lib/promrelabel/debug.qtpl:59
if targetID != "" {
//line lib/promrelabel/debug.qtpl:88
//line lib/promrelabel/debug.qtpl:59
qw422016.N().S(`<button type="button" onclick="location.href='?id=`)
//line lib/promrelabel/debug.qtpl:89
//line lib/promrelabel/debug.qtpl:60
qw422016.E().S(targetID)
//line lib/promrelabel/debug.qtpl:89
qw422016.N().S(`'" class="btn btn-secondary m-1">Reset All</button>`)
//line lib/promrelabel/debug.qtpl:90
//line lib/promrelabel/debug.qtpl:60
qw422016.N().S(`'" class="btn btn-secondary m-1">Reset</button>`)
//line lib/promrelabel/debug.qtpl:61
}
//line lib/promrelabel/debug.qtpl:90
//line lib/promrelabel/debug.qtpl:61
qw422016.N().S(`</form></div><div class="row"><main class="col-12">`)
//line lib/promrelabel/debug.qtpl:96
//line lib/promrelabel/debug.qtpl:67
streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
//line lib/promrelabel/debug.qtpl:96
//line lib/promrelabel/debug.qtpl:67
qw422016.N().S(`</main></div></div></body></html>`)
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
}
//line lib/promrelabel/debug.qtpl:102
func WriteRelabelDebugStepsHTML(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) {
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
func WriteRelabelDebugStepsHTML(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
//line lib/promrelabel/debug.qtpl:73
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:102
StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
StreamRelabelDebugStepsHTML(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:73
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
}
//line lib/promrelabel/debug.qtpl:102
func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) string {
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
//line lib/promrelabel/debug.qtpl:73
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:102
WriteRelabelDebugStepsHTML(qb422016, targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
WriteRelabelDebugStepsHTML(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:73
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
qt422016.ReleaseByteBuffer(qb422016)
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
return qs422016
//line lib/promrelabel/debug.qtpl:102
//line lib/promrelabel/debug.qtpl:73
}
//line lib/promrelabel/debug.qtpl:104
func streamrelabelDebugFormInputs(qw422016 *qt422016.Writer, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, targetID string) {
//line lib/promrelabel/debug.qtpl:104
qw422016.N().S(`<div><!-- show remote write relabel reload only for scrape metric relabel debug and pure relabel debug. discovery debug should not display this section -->`)
//line lib/promrelabel/debug.qtpl:107
if !isTargetRelabel {
//line lib/promrelabel/debug.qtpl:107
qw422016.N().S(`<div><div class="m-1"><div class="d-flex align-items-center gap-2 mt-1">Configs:`)
//line lib/promrelabel/debug.qtpl:112
if urlRelabelIndexLength > 0 {
//line lib/promrelabel/debug.qtpl:112
qw422016.N().S(`<input type="hidden" name="reload_url_relabel_configs" id="reload_url_relabel_configs" value="" /><select name="url_relabel_configs_index" class="form-select form-select-sm w-auto" onfocus="this.prevValue=this.value" onchange="reloadRelabelConfigs(this)">`)
//line lib/promrelabel/debug.qtpl:115
for i := range urlRelabelIndexLength {
//line lib/promrelabel/debug.qtpl:116
if urlRelabelIndexCurrent == i {
//line lib/promrelabel/debug.qtpl:116
qw422016.N().S(`<option value="`)
//line lib/promrelabel/debug.qtpl:117
qw422016.N().D(i)
//line lib/promrelabel/debug.qtpl:117
qw422016.N().S(`" selected="selected">remote-write-url-`)
//line lib/promrelabel/debug.qtpl:117
qw422016.N().D(i)
//line lib/promrelabel/debug.qtpl:117
qw422016.N().S(`</option>`)
//line lib/promrelabel/debug.qtpl:118
} else {
//line lib/promrelabel/debug.qtpl:118
qw422016.N().S(`<option value="`)
//line lib/promrelabel/debug.qtpl:119
qw422016.N().D(i)
//line lib/promrelabel/debug.qtpl:119
qw422016.N().S(`">remote-write-url-`)
//line lib/promrelabel/debug.qtpl:119
qw422016.N().D(i)
//line lib/promrelabel/debug.qtpl:119
qw422016.N().S(`</option>`)
//line lib/promrelabel/debug.qtpl:120
}
//line lib/promrelabel/debug.qtpl:121
}
//line lib/promrelabel/debug.qtpl:121
qw422016.N().S(`</select>`)
//line lib/promrelabel/debug.qtpl:123
}
//line lib/promrelabel/debug.qtpl:123
qw422016.N().S(`</div></div></div>`)
//line lib/promrelabel/debug.qtpl:127
} else {
//line lib/promrelabel/debug.qtpl:127
qw422016.N().S(`Configs:`)
//line lib/promrelabel/debug.qtpl:129
}
//line lib/promrelabel/debug.qtpl:129
qw422016.N().S(`<!-- the following text area css was generated with the help of AI to display yaml comments (starting with #) in gray. it could be rewritten in the future --><div class="m-1" style="position:relative;height:15em;"><div id="relabel-configs-backdrop" style="position:absolute;top:0;left:0;right:0;bottom:0;pointer-events:none;overflow:hidden;font-family:monospace;white-space:pre-wrap;padding:0.375rem 0.75rem;border:1px solid transparent;"></div><textarea id="relabel-configs-input" name="relabel_configs" class="form-control" style="position:absolute;top:0;left:0;height:100%;font-family:monospace;color:transparent;caret-color:#212529;background:transparent;resize:none;overflow-y:scroll;">`)
//line lib/promrelabel/debug.qtpl:135
//line lib/promrelabel/debug.qtpl:75
func streamrelabelDebugFormInputs(qw422016 *qt422016.Writer, metric, relabelConfigs string) {
//line lib/promrelabel/debug.qtpl:75
qw422016.N().S(`<div>Relabel configs:<br/><textarea name="relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">`)
//line lib/promrelabel/debug.qtpl:78
qw422016.E().S(relabelConfigs)
//line lib/promrelabel/debug.qtpl:135
qw422016.N().S(`</textarea></div></div><div class="mt-2">A Time Series:<br/><textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1" placeholder="up{job=&quot;job_name&quot;,instance=&quot;host:port&quot;}">`)
//line lib/promrelabel/debug.qtpl:141
//line lib/promrelabel/debug.qtpl:78
qw422016.N().S(`</textarea></div><div>Labels:<br/><textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">`)
//line lib/promrelabel/debug.qtpl:83
qw422016.E().S(metric)
//line lib/promrelabel/debug.qtpl:141
//line lib/promrelabel/debug.qtpl:83
qw422016.N().S(`</textarea></div>`)
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
}
//line lib/promrelabel/debug.qtpl:143
func writerelabelDebugFormInputs(qq422016 qtio422016.Writer, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, targetID string) {
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
func writerelabelDebugFormInputs(qq422016 qtio422016.Writer, metric, relabelConfigs string) {
//line lib/promrelabel/debug.qtpl:85
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:143
streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, targetID)
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
streamrelabelDebugFormInputs(qw422016, metric, relabelConfigs)
//line lib/promrelabel/debug.qtpl:85
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
}
//line lib/promrelabel/debug.qtpl:143
func relabelDebugFormInputs(metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, targetID string) string {
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
func relabelDebugFormInputs(metric, relabelConfigs string) string {
//line lib/promrelabel/debug.qtpl:85
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:143
writerelabelDebugFormInputs(qb422016, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, targetID)
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
writerelabelDebugFormInputs(qb422016, metric, relabelConfigs)
//line lib/promrelabel/debug.qtpl:85
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
qt422016.ReleaseByteBuffer(qb422016)
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
return qs422016
//line lib/promrelabel/debug.qtpl:143
//line lib/promrelabel/debug.qtpl:85
}
//line lib/promrelabel/debug.qtpl:145
//line lib/promrelabel/debug.qtpl:87
func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []DebugStep, targetURL, targetID string) {
//line lib/promrelabel/debug.qtpl:146
//line lib/promrelabel/debug.qtpl:88
if len(dss) > 0 {
//line lib/promrelabel/debug.qtpl:146
//line lib/promrelabel/debug.qtpl:88
qw422016.N().S(`<div class="m-3"><b>Original labels:</b> <samp>`)
//line lib/promrelabel/debug.qtpl:148
//line lib/promrelabel/debug.qtpl:90
streammustFormatLabels(qw422016, dss[0].In)
//line lib/promrelabel/debug.qtpl:148
//line lib/promrelabel/debug.qtpl:90
qw422016.N().S(`</samp></div>`)
//line lib/promrelabel/debug.qtpl:150
//line lib/promrelabel/debug.qtpl:92
}
//line lib/promrelabel/debug.qtpl:150
//line lib/promrelabel/debug.qtpl:92
qw422016.N().S(`<table class="table table-striped table-hover table-bordered table-sm"><thead><tr><th scope="col" style="width: 5%">Step</th><th scope="col" style="width: 25%">Relabeling Rule</th><th scope="col" style="width: 35%">Input Labels</th><th scope="col" stile="width: 35%">Output labels</a></tr></thead><tbody>`)
//line lib/promrelabel/debug.qtpl:161
//line lib/promrelabel/debug.qtpl:103
for i, ds := range dss {
//line lib/promrelabel/debug.qtpl:163
//line lib/promrelabel/debug.qtpl:105
inLabels, inErr := promutil.NewLabelsFromString(ds.In)
outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
changedLabels := getChangedLabelNames(inLabels, outLabels)
//line lib/promrelabel/debug.qtpl:166
//line lib/promrelabel/debug.qtpl:108
qw422016.N().S(`<tr><td>`)
//line lib/promrelabel/debug.qtpl:168
//line lib/promrelabel/debug.qtpl:110
qw422016.N().D(i)
//line lib/promrelabel/debug.qtpl:168
//line lib/promrelabel/debug.qtpl:110
qw422016.N().S(`</td><td><b><pre class="m-2">`)
//line lib/promrelabel/debug.qtpl:169
//line lib/promrelabel/debug.qtpl:111
qw422016.E().S(ds.Rule)
//line lib/promrelabel/debug.qtpl:169
//line lib/promrelabel/debug.qtpl:111
qw422016.N().S(`</pre></b></td><td>`)
//line lib/promrelabel/debug.qtpl:171
//line lib/promrelabel/debug.qtpl:113
if inErr == nil {
//line lib/promrelabel/debug.qtpl:171
//line lib/promrelabel/debug.qtpl:113
qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="deleted and updated labels highlighted in red">`)
//line lib/promrelabel/debug.qtpl:173
//line lib/promrelabel/debug.qtpl:115
streamlabelsWithHighlight(qw422016, inLabels, changedLabels, "#D15757")
//line lib/promrelabel/debug.qtpl:173
//line lib/promrelabel/debug.qtpl:115
qw422016.N().S(`</div>`)
//line lib/promrelabel/debug.qtpl:175
//line lib/promrelabel/debug.qtpl:117
} else {
//line lib/promrelabel/debug.qtpl:175
//line lib/promrelabel/debug.qtpl:117
qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing input labels"><pre>`)
//line lib/promrelabel/debug.qtpl:177
//line lib/promrelabel/debug.qtpl:119
qw422016.E().S(inErr.Error())
//line lib/promrelabel/debug.qtpl:177
//line lib/promrelabel/debug.qtpl:119
qw422016.N().S(`</pre></div>`)
//line lib/promrelabel/debug.qtpl:179
//line lib/promrelabel/debug.qtpl:121
break
//line lib/promrelabel/debug.qtpl:180
//line lib/promrelabel/debug.qtpl:122
}
//line lib/promrelabel/debug.qtpl:180
//line lib/promrelabel/debug.qtpl:122
qw422016.N().S(`</td><td>`)
//line lib/promrelabel/debug.qtpl:183
//line lib/promrelabel/debug.qtpl:125
if outErr == nil {
//line lib/promrelabel/debug.qtpl:183
//line lib/promrelabel/debug.qtpl:125
qw422016.N().S(`<div class="m-2" style="font-size: 0.9em" title="added and updated labels highlighted in blue">`)
//line lib/promrelabel/debug.qtpl:185
//line lib/promrelabel/debug.qtpl:127
streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "#4495e0")
//line lib/promrelabel/debug.qtpl:185
//line lib/promrelabel/debug.qtpl:127
qw422016.N().S(`</div>`)
//line lib/promrelabel/debug.qtpl:187
//line lib/promrelabel/debug.qtpl:129
} else {
//line lib/promrelabel/debug.qtpl:187
//line lib/promrelabel/debug.qtpl:129
qw422016.N().S(`<div class="m-2" style="font-size: 0.9em; color: red" title="error parsing output labels"><pre>`)
//line lib/promrelabel/debug.qtpl:189
//line lib/promrelabel/debug.qtpl:131
qw422016.E().S(outErr.Error())
//line lib/promrelabel/debug.qtpl:189
//line lib/promrelabel/debug.qtpl:131
qw422016.N().S(`</pre></div>`)
//line lib/promrelabel/debug.qtpl:191
//line lib/promrelabel/debug.qtpl:133
break
//line lib/promrelabel/debug.qtpl:192
//line lib/promrelabel/debug.qtpl:134
}
//line lib/promrelabel/debug.qtpl:192
//line lib/promrelabel/debug.qtpl:134
qw422016.N().S(`</td></tr>`)
//line lib/promrelabel/debug.qtpl:195
//line lib/promrelabel/debug.qtpl:137
}
//line lib/promrelabel/debug.qtpl:195
//line lib/promrelabel/debug.qtpl:137
qw422016.N().S(`</tbody></table>`)
//line lib/promrelabel/debug.qtpl:198
//line lib/promrelabel/debug.qtpl:140
if len(dss) > 0 {
//line lib/promrelabel/debug.qtpl:198
//line lib/promrelabel/debug.qtpl:140
qw422016.N().S(`<div class="m-3"><b>Resulting labels:</b> <samp>`)
//line lib/promrelabel/debug.qtpl:200
//line lib/promrelabel/debug.qtpl:142
streammustFormatLabels(qw422016, dss[len(dss)-1].Out)
//line lib/promrelabel/debug.qtpl:200
//line lib/promrelabel/debug.qtpl:142
qw422016.N().S(`</samp>`)
//line lib/promrelabel/debug.qtpl:201
//line lib/promrelabel/debug.qtpl:143
if targetURL != "" {
//line lib/promrelabel/debug.qtpl:201
//line lib/promrelabel/debug.qtpl:143
qw422016.N().S(`<div><b>Target URL:</b>`)
//line lib/promrelabel/debug.qtpl:203
//line lib/promrelabel/debug.qtpl:145
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:203
//line lib/promrelabel/debug.qtpl:145
qw422016.N().S(`<a href="`)
//line lib/promrelabel/debug.qtpl:203
//line lib/promrelabel/debug.qtpl:145
qw422016.E().S(targetURL)
//line lib/promrelabel/debug.qtpl:203
//line lib/promrelabel/debug.qtpl:145
qw422016.N().S(`" target="_blank">`)
//line lib/promrelabel/debug.qtpl:203
//line lib/promrelabel/debug.qtpl:145
qw422016.E().S(targetURL)
//line lib/promrelabel/debug.qtpl:203
//line lib/promrelabel/debug.qtpl:145
qw422016.N().S(`</a>`)
//line lib/promrelabel/debug.qtpl:204
//line lib/promrelabel/debug.qtpl:146
if targetID != "" {
//line lib/promrelabel/debug.qtpl:205
//line lib/promrelabel/debug.qtpl:147
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:205
//line lib/promrelabel/debug.qtpl:147
qw422016.N().S(`(<a href="target_response?id=`)
//line lib/promrelabel/debug.qtpl:206
//line lib/promrelabel/debug.qtpl:148
qw422016.E().S(targetID)
//line lib/promrelabel/debug.qtpl:206
//line lib/promrelabel/debug.qtpl:148
qw422016.N().S(`" target="_blank" title="click to fetch target response on behalf of the scraper">response</a>)`)
//line lib/promrelabel/debug.qtpl:207
//line lib/promrelabel/debug.qtpl:149
}
//line lib/promrelabel/debug.qtpl:207
//line lib/promrelabel/debug.qtpl:149
qw422016.N().S(`</div>`)
//line lib/promrelabel/debug.qtpl:209
//line lib/promrelabel/debug.qtpl:151
}
//line lib/promrelabel/debug.qtpl:209
//line lib/promrelabel/debug.qtpl:151
qw422016.N().S(`</div>`)
//line lib/promrelabel/debug.qtpl:211
//line lib/promrelabel/debug.qtpl:153
}
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
}
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []DebugStep, targetURL, targetID string) {
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
streamrelabelDebugSteps(qw422016, dss, targetURL, targetID)
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
}
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
func relabelDebugSteps(dss []DebugStep, targetURL, targetID string) string {
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
writerelabelDebugSteps(qb422016, dss, targetURL, targetID)
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
qt422016.ReleaseByteBuffer(qb422016)
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
return qs422016
//line lib/promrelabel/debug.qtpl:212
//line lib/promrelabel/debug.qtpl:154
}
//line lib/promrelabel/debug.qtpl:214
func StreamRelabelDebugStepsJSON(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) {
//line lib/promrelabel/debug.qtpl:214
//line lib/promrelabel/debug.qtpl:156
func StreamRelabelDebugStepsJSON(qw422016 *qt422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
//line lib/promrelabel/debug.qtpl:156
qw422016.N().S(`{`)
//line lib/promrelabel/debug.qtpl:216
//line lib/promrelabel/debug.qtpl:158
if err != nil {
//line lib/promrelabel/debug.qtpl:216
//line lib/promrelabel/debug.qtpl:158
qw422016.N().S(`"status": "error","error":`)
//line lib/promrelabel/debug.qtpl:218
//line lib/promrelabel/debug.qtpl:160
qw422016.N().Q(fmt.Sprintf("Error: %s", err))
//line lib/promrelabel/debug.qtpl:219
//line lib/promrelabel/debug.qtpl:161
} else {
//line lib/promrelabel/debug.qtpl:220
//line lib/promrelabel/debug.qtpl:162
var hasError bool
//line lib/promrelabel/debug.qtpl:220
//line lib/promrelabel/debug.qtpl:162
qw422016.N().S(`"status": "success","steps": [`)
//line lib/promrelabel/debug.qtpl:223
//line lib/promrelabel/debug.qtpl:165
for i, ds := range dss {
//line lib/promrelabel/debug.qtpl:225
//line lib/promrelabel/debug.qtpl:167
inLabels, inErr := promutil.NewLabelsFromString(ds.In)
outLabels, outErr := promutil.NewLabelsFromString(ds.Out)
changedLabels := getChangedLabelNames(inLabels, outLabels)
//line lib/promrelabel/debug.qtpl:228
//line lib/promrelabel/debug.qtpl:170
qw422016.N().S(`{"inLabels":`)
//line lib/promrelabel/debug.qtpl:230
//line lib/promrelabel/debug.qtpl:172
qw422016.N().Q(labelsWithHighlight(inLabels, changedLabels, "#D15757"))
//line lib/promrelabel/debug.qtpl:230
//line lib/promrelabel/debug.qtpl:172
qw422016.N().S(`,"outLabels":`)
//line lib/promrelabel/debug.qtpl:231
//line lib/promrelabel/debug.qtpl:173
qw422016.N().Q(labelsWithHighlight(outLabels, changedLabels, "#4495e0"))
//line lib/promrelabel/debug.qtpl:231
//line lib/promrelabel/debug.qtpl:173
qw422016.N().S(`,"rule":`)
//line lib/promrelabel/debug.qtpl:232
//line lib/promrelabel/debug.qtpl:174
qw422016.N().Q(ds.Rule)
//line lib/promrelabel/debug.qtpl:232
//line lib/promrelabel/debug.qtpl:174
qw422016.N().S(`,"errors": {`)
//line lib/promrelabel/debug.qtpl:234
//line lib/promrelabel/debug.qtpl:176
if inErr != nil {
//line lib/promrelabel/debug.qtpl:234
//line lib/promrelabel/debug.qtpl:176
qw422016.N().S(`"inLabels":`)
//line lib/promrelabel/debug.qtpl:235
//line lib/promrelabel/debug.qtpl:177
qw422016.N().Q(`<span style="color: #D15757">` + inErr.Error() + `</span>`)
//line lib/promrelabel/debug.qtpl:235
//line lib/promrelabel/debug.qtpl:177
if outErr != nil {
//line lib/promrelabel/debug.qtpl:235
//line lib/promrelabel/debug.qtpl:177
qw422016.N().S(`,`)
//line lib/promrelabel/debug.qtpl:235
//line lib/promrelabel/debug.qtpl:177
}
//line lib/promrelabel/debug.qtpl:236
//line lib/promrelabel/debug.qtpl:178
hasError = true
//line lib/promrelabel/debug.qtpl:237
//line lib/promrelabel/debug.qtpl:179
} else {
//line lib/promrelabel/debug.qtpl:238
//line lib/promrelabel/debug.qtpl:180
}
//line lib/promrelabel/debug.qtpl:239
//line lib/promrelabel/debug.qtpl:181
if outErr != nil {
//line lib/promrelabel/debug.qtpl:239
//line lib/promrelabel/debug.qtpl:181
qw422016.N().S(`"outLabels":`)
//line lib/promrelabel/debug.qtpl:240
//line lib/promrelabel/debug.qtpl:182
qw422016.N().Q(`<span style="color: #D15757">` + outErr.Error() + `</span>`)
//line lib/promrelabel/debug.qtpl:241
//line lib/promrelabel/debug.qtpl:183
hasError = true
//line lib/promrelabel/debug.qtpl:242
//line lib/promrelabel/debug.qtpl:184
}
//line lib/promrelabel/debug.qtpl:242
//line lib/promrelabel/debug.qtpl:184
qw422016.N().S(`}}`)
//line lib/promrelabel/debug.qtpl:245
//line lib/promrelabel/debug.qtpl:187
if i != len(dss)-1 {
//line lib/promrelabel/debug.qtpl:245
//line lib/promrelabel/debug.qtpl:187
qw422016.N().S(`,`)
//line lib/promrelabel/debug.qtpl:245
//line lib/promrelabel/debug.qtpl:187
}
//line lib/promrelabel/debug.qtpl:246
//line lib/promrelabel/debug.qtpl:188
}
//line lib/promrelabel/debug.qtpl:246
//line lib/promrelabel/debug.qtpl:188
qw422016.N().S(`]`)
//line lib/promrelabel/debug.qtpl:248
//line lib/promrelabel/debug.qtpl:190
if len(dss) > 0 && !hasError {
//line lib/promrelabel/debug.qtpl:248
//line lib/promrelabel/debug.qtpl:190
qw422016.N().S(`,"originalLabels":`)
//line lib/promrelabel/debug.qtpl:250
//line lib/promrelabel/debug.qtpl:192
qw422016.N().Q(mustFormatLabels(dss[0].In))
//line lib/promrelabel/debug.qtpl:250
//line lib/promrelabel/debug.qtpl:192
qw422016.N().S(`,"resultingLabels":`)
//line lib/promrelabel/debug.qtpl:251
//line lib/promrelabel/debug.qtpl:193
qw422016.N().Q(mustFormatLabels(dss[len(dss)-1].Out))
//line lib/promrelabel/debug.qtpl:252
//line lib/promrelabel/debug.qtpl:194
}
//line lib/promrelabel/debug.qtpl:253
//line lib/promrelabel/debug.qtpl:195
}
//line lib/promrelabel/debug.qtpl:253
//line lib/promrelabel/debug.qtpl:195
qw422016.N().S(`}`)
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
}
//line lib/promrelabel/debug.qtpl:255
func WriteRelabelDebugStepsJSON(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) {
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
func WriteRelabelDebugStepsJSON(qq422016 qtio422016.Writer, targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) {
//line lib/promrelabel/debug.qtpl:197
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:255
StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
StreamRelabelDebugStepsJSON(qw422016, targetURL, targetID, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:197
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
}
//line lib/promrelabel/debug.qtpl:255
func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) string {
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) string {
//line lib/promrelabel/debug.qtpl:197
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:255
WriteRelabelDebugStepsJSON(qb422016, targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
WriteRelabelDebugStepsJSON(qb422016, targetURL, targetID, dss, metric, relabelConfigs, err)
//line lib/promrelabel/debug.qtpl:197
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
qt422016.ReleaseByteBuffer(qb422016)
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
return qs422016
//line lib/promrelabel/debug.qtpl:255
//line lib/promrelabel/debug.qtpl:197
}
//line lib/promrelabel/debug.qtpl:257
//line lib/promrelabel/debug.qtpl:199
func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
//line lib/promrelabel/debug.qtpl:259
//line lib/promrelabel/debug.qtpl:201
labelsList := labels.GetLabels()
metricName := ""
for i, label := range labelsList {
@@ -565,153 +501,153 @@ func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutil.Label
}
}
//line lib/promrelabel/debug.qtpl:269
//line lib/promrelabel/debug.qtpl:211
if metricName != "" {
//line lib/promrelabel/debug.qtpl:270
//line lib/promrelabel/debug.qtpl:212
if _, ok := highlight["__name__"]; ok {
//line lib/promrelabel/debug.qtpl:270
//line lib/promrelabel/debug.qtpl:212
qw422016.N().S(`<span style="font-weight:bold;color:`)
//line lib/promrelabel/debug.qtpl:271
//line lib/promrelabel/debug.qtpl:213
qw422016.E().S(color)
//line lib/promrelabel/debug.qtpl:271
//line lib/promrelabel/debug.qtpl:213
qw422016.N().S(`">`)
//line lib/promrelabel/debug.qtpl:271
//line lib/promrelabel/debug.qtpl:213
qw422016.E().S(metricName)
//line lib/promrelabel/debug.qtpl:271
//line lib/promrelabel/debug.qtpl:213
qw422016.N().S(`</span>`)
//line lib/promrelabel/debug.qtpl:272
//line lib/promrelabel/debug.qtpl:214
} else {
//line lib/promrelabel/debug.qtpl:273
//line lib/promrelabel/debug.qtpl:215
qw422016.E().S(metricName)
//line lib/promrelabel/debug.qtpl:274
//line lib/promrelabel/debug.qtpl:216
}
//line lib/promrelabel/debug.qtpl:275
//line lib/promrelabel/debug.qtpl:217
if len(labelsList) == 0 {
//line lib/promrelabel/debug.qtpl:275
//line lib/promrelabel/debug.qtpl:217
return
//line lib/promrelabel/debug.qtpl:275
//line lib/promrelabel/debug.qtpl:217
}
//line lib/promrelabel/debug.qtpl:276
//line lib/promrelabel/debug.qtpl:218
}
//line lib/promrelabel/debug.qtpl:276
//line lib/promrelabel/debug.qtpl:218
qw422016.N().S(`{`)
//line lib/promrelabel/debug.qtpl:278
//line lib/promrelabel/debug.qtpl:220
for i, label := range labelsList {
//line lib/promrelabel/debug.qtpl:279
//line lib/promrelabel/debug.qtpl:221
if _, ok := highlight[label.Name]; ok {
//line lib/promrelabel/debug.qtpl:279
//line lib/promrelabel/debug.qtpl:221
qw422016.N().S(`<span style="font-weight:bold;color:`)
//line lib/promrelabel/debug.qtpl:280
//line lib/promrelabel/debug.qtpl:222
qw422016.E().S(color)
//line lib/promrelabel/debug.qtpl:280
//line lib/promrelabel/debug.qtpl:222
qw422016.N().S(`">`)
//line lib/promrelabel/debug.qtpl:280
//line lib/promrelabel/debug.qtpl:222
qw422016.E().S(label.Name)
//line lib/promrelabel/debug.qtpl:280
//line lib/promrelabel/debug.qtpl:222
qw422016.N().S(`=`)
//line lib/promrelabel/debug.qtpl:280
//line lib/promrelabel/debug.qtpl:222
qw422016.E().Q(label.Value)
//line lib/promrelabel/debug.qtpl:280
//line lib/promrelabel/debug.qtpl:222
qw422016.N().S(`</span>`)
//line lib/promrelabel/debug.qtpl:281
//line lib/promrelabel/debug.qtpl:223
} else {
//line lib/promrelabel/debug.qtpl:282
//line lib/promrelabel/debug.qtpl:224
qw422016.E().S(label.Name)
//line lib/promrelabel/debug.qtpl:282
//line lib/promrelabel/debug.qtpl:224
qw422016.N().S(`=`)
//line lib/promrelabel/debug.qtpl:282
//line lib/promrelabel/debug.qtpl:224
qw422016.E().Q(label.Value)
//line lib/promrelabel/debug.qtpl:283
//line lib/promrelabel/debug.qtpl:225
}
//line lib/promrelabel/debug.qtpl:284
//line lib/promrelabel/debug.qtpl:226
if i < len(labelsList)-1 {
//line lib/promrelabel/debug.qtpl:284
//line lib/promrelabel/debug.qtpl:226
qw422016.N().S(`,`)
//line lib/promrelabel/debug.qtpl:284
//line lib/promrelabel/debug.qtpl:226
qw422016.N().S(` `)
//line lib/promrelabel/debug.qtpl:284
//line lib/promrelabel/debug.qtpl:226
}
//line lib/promrelabel/debug.qtpl:285
//line lib/promrelabel/debug.qtpl:227
}
//line lib/promrelabel/debug.qtpl:285
//line lib/promrelabel/debug.qtpl:227
qw422016.N().S(`}`)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
}
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutil.Labels, highlight map[string]struct{}, color string) {
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
streamlabelsWithHighlight(qw422016, labels, highlight, color)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
}
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
func labelsWithHighlight(labels *promutil.Labels, highlight map[string]struct{}, color string) string {
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
writelabelsWithHighlight(qb422016, labels, highlight, color)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
qt422016.ReleaseByteBuffer(qb422016)
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
return qs422016
//line lib/promrelabel/debug.qtpl:287
//line lib/promrelabel/debug.qtpl:229
}
//line lib/promrelabel/debug.qtpl:289
//line lib/promrelabel/debug.qtpl:231
func streammustFormatLabels(qw422016 *qt422016.Writer, s string) {
//line lib/promrelabel/debug.qtpl:290
//line lib/promrelabel/debug.qtpl:232
labels, err := promutil.NewLabelsFromString(s)
//line lib/promrelabel/debug.qtpl:291
//line lib/promrelabel/debug.qtpl:233
if err != nil {
//line lib/promrelabel/debug.qtpl:291
//line lib/promrelabel/debug.qtpl:233
qw422016.N().S(`<span style="color: red" title="error parsing labels:`)
//line lib/promrelabel/debug.qtpl:292
//line lib/promrelabel/debug.qtpl:234
qw422016.E().S(err.Error())
//line lib/promrelabel/debug.qtpl:292
//line lib/promrelabel/debug.qtpl:234
qw422016.N().S(`">`)
//line lib/promrelabel/debug.qtpl:292
//line lib/promrelabel/debug.qtpl:234
qw422016.E().S("error parsing labels: " + err.Error())
//line lib/promrelabel/debug.qtpl:292
//line lib/promrelabel/debug.qtpl:234
qw422016.N().S(`</span>`)
//line lib/promrelabel/debug.qtpl:293
//line lib/promrelabel/debug.qtpl:235
} else {
//line lib/promrelabel/debug.qtpl:294
//line lib/promrelabel/debug.qtpl:236
streamlabelsWithHighlight(qw422016, labels, nil, "")
//line lib/promrelabel/debug.qtpl:295
//line lib/promrelabel/debug.qtpl:237
}
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
}
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
func writemustFormatLabels(qq422016 qtio422016.Writer, s string) {
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
qw422016 := qt422016.AcquireWriter(qq422016)
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
streammustFormatLabels(qw422016, s)
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
qt422016.ReleaseWriter(qw422016)
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
}
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
func mustFormatLabels(s string) string {
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
qb422016 := qt422016.AcquireByteBuffer()
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
writemustFormatLabels(qb422016, s)
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
qs422016 := string(qb422016.B)
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
qt422016.ReleaseByteBuffer(qb422016)
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
return qs422016
//line lib/promrelabel/debug.qtpl:296
//line lib/promrelabel/debug.qtpl:238
}

View File

@@ -10,10 +10,10 @@ import (
// TestWriteRelabelDebugSupportFormats verifies the relabeling debug input, rules and output.
func TestWriteRelabelDebugSupportFormats(t *testing.T) {
f := func(input, rules, expect string) {
f := func(input, rule, expect string) {
// execute
outputWriter := bytes.NewBuffer(nil)
writeRelabelDebug(outputWriter, false, "", input, rules, 0, 0, "json", nil)
writeRelabelDebug(outputWriter, false, "", input, rule, "json", nil)
// the response is in JSON with HTML content, extract the `resultingLabels` in JSON and unescape it.
resultingLabels := fastjson.GetString(outputWriter.Bytes(), `resultingLabels`)
@@ -41,20 +41,4 @@ func TestWriteRelabelDebugSupportFormats(t *testing.T) {
f(`{_name__="metric_name"`, ruleTestParsing, ``)
f(`_name__="metric_name}"`, ruleTestParsing, ``)
f(`metrics_name}"`, ruleTestParsing, ``)
// test multiple rules including remote writes
// drop all labels and add one in URL relabeling
rule1 := `
- action: labeldrop
regex: "drop_me_metrics_relabel"
`
rule2 := `
- action: labeldrop
regex: "drop_me_remote_write_relabel"
`
rule3 := `
- target_label: add_me_url_relabel
replacement: added
`
f(`{__name__="metric_name", drop_me_metrics_relabel="1", drop_me_remote_write_relabel="2"}`, rule1+rule2+rule3, `metric_name{add_me_url_relabel="added"}`)
}

View File

@@ -5,7 +5,6 @@ import (
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
@@ -54,18 +53,10 @@ func newClient(ctx context.Context, sw *ScrapeWork) (*client, error) {
return nil
}
dialFunc := netutil.NewStatDialFunc("vm_promscrape")
if sw.UnixSocket != "" {
dialFunc = netutil.NewStatDialFuncWithDial("vm_promscrape", func(ctx context.Context, _, _ string) (net.Conn, error) {
return netutil.Dialer.DialContext(ctx, "unix", sw.UnixSocket)
})
}
proxyURL := sw.ProxyURL
var proxyURLFunc func(*http.Request) (*url.URL, error)
if proxyURL != nil {
if sw.UnixSocket != "" {
return nil, fmt.Errorf("proxyURL: %q cannot be used for scraping unix_socket target: %q", proxyURL, sw.UnixSocket)
}
// case for direct http proxy connection.
// must be used for http based scrape targets
// since standard golang http.transport has special case for it

View File

@@ -1325,9 +1325,6 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
}
streamParse = b
}
// Read __unix_socket__ option from __unix_socket__ label.
unixSocket := labels.Get("__unix_socket__")
// Remove labels with "__" prefix according to https://www.robustperception.io/life-of-a-label/
labels.RemoveLabelsWithDoubleUnderscorePrefix()
// Add missing "instance" label according to https://www.robustperception.io/life-of-a-label
@@ -1372,7 +1369,6 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel
LabelLimit: labelLimit,
NoStaleMarkers: swc.noStaleMarkers,
AuthToken: at,
UnixSocket: unixSocket,
jobNameOriginal: swc.jobName,
}

View File

@@ -3,104 +3,34 @@ package promscrape
import (
"fmt"
"net/http"
"strconv"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
)
// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page.
// remotewrite-related relabel configs could be empty as vmsingle doesn't provide remote write feature.
func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request, rwGlobalRelabelConfigs string, rwURLRelabelConfigss []string) {
// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page
func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
targetID := r.FormValue("id")
metric := r.FormValue("metric")
relabelConfigs := r.FormValue("relabel_configs")
// if set, it means user selected another URL from the dropdown and everything will be reloaded.
reloadRWURLRelabelConfigs := r.FormValue("reload_url_relabel_configs")
rwURLRelabelConfigsIdxStr := r.FormValue("url_relabel_configs_index")
format := r.FormValue("format")
var err error
// if all per-URL config is empty, it means no per-URL rule is configured.
// set it to 0 so the user do not see the options in debug page.
rwURLRelabelConfigsLength := 0
for _, urlRelabelConfig := range rwURLRelabelConfigss {
if urlRelabelConfig != "" {
rwURLRelabelConfigsLength = len(rwURLRelabelConfigss)
break
if metric == "" && relabelConfigs == "" && targetID != "" {
pcs, labels, ok := getMetricRelabelContextByTargetID(targetID)
if !ok {
err = fmt.Errorf("cannot find target for id=%s", targetID)
targetID = ""
} else {
metric = labels.String()
relabelConfigs = pcs.String()
}
}
rwURLRelabelConfigsIdx, idxErr := strconv.Atoi(rwURLRelabelConfigsIdxStr)
if idxErr != nil {
rwURLRelabelConfigsIdx = -1
}
// load the initial data with specific remote write URL index (default 0) in 2 cases:
// - relabel config is empty. load scrape relabel (if targetID exist) + remote write related relabel (always).
// - `reload` is set. load scrape relabel (if targetID exist) + reload remote write related relabel (by the URL index).
init := metric == "" && relabelConfigs == "" && reloadRWURLRelabelConfigs == ""
reload := reloadRWURLRelabelConfigs != ""
if init || reload {
// scrape related relabel labels & rules
var (
pcs = &promrelabel.ParsedConfigs{} // could be empty
labels *promutil.Labels
ok bool
)
if targetID != "" {
pcs, labels, ok = getMetricRelabelContextByTargetID(targetID)
if !ok {
err = fmt.Errorf("cannot find target for id=%s", targetID)
targetID = ""
} else {
metric = "up"
metric += labels.String()
}
}
// general relabel rules (remote write)
// set the per-URL remote write relabel according to index, any error will fall back the index to 0.
rwURLRelabelConfigs := ""
if len(rwURLRelabelConfigss) > 0 {
// ignore the error if the input is invalid or exceed the length, and fallback to 0.
if rwURLRelabelConfigsIdx < 0 || rwURLRelabelConfigsIdx >= len(rwURLRelabelConfigss) {
rwURLRelabelConfigsIdx = 0
}
rwURLRelabelConfigs = rwURLRelabelConfigss[rwURLRelabelConfigsIdx]
}
relabelConfigs = composeRelabelConfigs(pcs.String(), rwGlobalRelabelConfigs, rwURLRelabelConfigs, rwURLRelabelConfigsIdx)
}
if format == "json" {
httpserver.EnableCORS(w, r)
w.Header().Set("Content-Type", "application/json")
}
promrelabel.WriteMetricRelabelDebug(w, targetID, metric, relabelConfigs, rwURLRelabelConfigsLength, rwURLRelabelConfigsIdx, format, err)
}
func composeRelabelConfigs(relabelConfigs, rwGlobalRelabelConfigs, rwURLRelabelConfigs string, rwURLIdx int) string {
if relabelConfigs != "" {
relabelConfigs = "# -promscrape.config .scrape_configs[].metric_relabel_configs\n" + strings.TrimSpace(relabelConfigs) + "\n"
}
if rwGlobalRelabelConfigs != "" {
relabelConfigs += "\n# -remoteWrite.relabelConfig"
relabelConfigs += "\n" + strings.TrimSpace(rwGlobalRelabelConfigs) + "\n"
}
if rwURLRelabelConfigs != "" {
relabelConfigs += fmt.Sprintf("\n# -remoteWrite.urlRelabelConfig=remote-write-url-%d", rwURLIdx)
relabelConfigs += "\n" + strings.TrimSpace(rwURLRelabelConfigs) + "\n"
}
return relabelConfigs
promrelabel.WriteMetricRelabelDebug(w, targetID, metric, relabelConfigs, format, err)
}
// WriteTargetRelabelDebug generates response for /target-relabel-debug page
@@ -118,7 +48,7 @@ func WriteTargetRelabelDebug(w http.ResponseWriter, r *http.Request) {
targetID = ""
} else {
metric = labels.labelsString()
relabelConfigs = "# -promscrape.config .scrape_configs[].relabel_configs\n" + pcs.String()
relabelConfigs = pcs.String()
}
}
if format == "json" {

View File

@@ -157,9 +157,6 @@ type ScrapeWork struct {
// The Tenant Info
AuthToken *auth.Token
// Optional path to Unix domain socket for scraping metrics over Unix domain socket.
UnixSocket string
// The original 'job_name'
jobNameOriginal string
}
@@ -177,12 +174,12 @@ func (sw *ScrapeWork) key() string {
// Do not take into account OriginalLabels, since they can be changed with relabeling.
// Do not take into account RelabelConfigs, since it is already applied to Labels.
// Take into account JobNameOriginal in order to capture the case when the original job_name is changed via relabeling.
key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, UnixSocket=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, "+
key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, "+
"HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, ExternalLabels=%s, MaxScrapeSize=%d, "+
"ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%q, "+
"SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+
"ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, LabelLimit=%d, NoStaleMarkers=%v",
sw.jobNameOriginal, sw.ScrapeURL, sw.UnixSocket, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels,
sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels,
sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(), sw.ExternalLabels.String(), sw.MaxScrapeSize,
sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(),
sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse,

View File

@@ -28,9 +28,9 @@ func (rs *Rows) Reset() {
rs.tu.reset()
}
// Unmarshal unmarshals json line protocol from s.
// Unmarshal unmarshals influx line protocol rows from s.
//
// See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#json-line-format
// See https://docs.influxdata.com/influxdb/v1.7/write_protocols/line_protocol_tutorial/
//
// s shouldn't be modified when rs is in use.
func (rs *Rows) Unmarshal(s string) {

View File

@@ -485,65 +485,78 @@ func TestIndexDBOpenClose(t *testing.T) {
}
func TestIndexDB(t *testing.T) {
defer testRemoveAll(t)
const metricGroups = 10
timestamp := time.Now().UnixMilli()
f := func(t *testing.T, concurrency int, disablePerDayIndex bool) {
const metricGroups = 10
now := time.Now().UTC()
timestamp := now.UnixMilli()
date := uint64(timestamp / msecPerDay)
searchTR := TimeRange{
MinTimestamp: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
}
if disablePerDayIndex {
searchTR = globalIndexTimeRange
}
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
t.Run("serial", func(t *testing.T) {
const path = "TestIndexDB-serial"
s := MustOpenStorage(path, OpenOptions{})
ptw := s.tb.MustGetPartition(timestamp)
db := ptw.pt.idb
mns, tsids, err := testIndexDBGetOrCreateTSIDByName(db, metricGroups, timestamp)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if err := testIndexDBCheckTSIDByName(db, mns, tsids, timestamp, false); err != nil {
t.Fatalf("unexpected error: %s", err)
}
// Re-open the storage and verify it works as expected.
s.tb.PutPartition(ptw)
s.MustClose()
s = MustOpenStorage(path, OpenOptions{})
ptw = s.tb.MustGetPartition(timestamp)
db = ptw.pt.idb
if err := testIndexDBCheckTSIDByName(db, mns, tsids, timestamp, false); err != nil {
t.Fatalf("unexpected error: %s", err)
}
s.tb.PutPartition(ptw)
s.MustClose()
fs.MustRemoveDir(path)
})
t.Run("concurrent", func(t *testing.T) {
const path = "TestIndexDB-concurrent"
s := MustOpenStorage(path, OpenOptions{})
ptw := s.tb.MustGetPartition(timestamp)
defer s.tb.PutPartition(ptw)
db := ptw.pt.idb
var wg sync.WaitGroup
errs := make([]error, concurrency)
isConcurrent := concurrency > 1
for i := range concurrency {
wg.Go(func() {
mns, tsids, err := testIndexDBGetOrCreateTSIDByName(db, metricGroups, date)
ch := make(chan error, 3)
for range cap(ch) {
go func() {
mns, tsid, err := testIndexDBGetOrCreateTSIDByName(db, metricGroups, timestamp)
if err != nil {
errs[i] = fmt.Errorf("testIndexDBGetOrCreateTSIDByName failed unexpectedly: %w", err)
ch <- err
return
}
if err := testIndexDBCheckTSIDByName(db, mns, tsids, date, searchTR, isConcurrent); err != nil {
errs[i] = fmt.Errorf("testIndexDBCheckTSIDByName failed unexpectedly: %w", err)
if err := testIndexDBCheckTSIDByName(db, mns, tsid, timestamp, true); err != nil {
ch <- err
return
}
})
ch <- nil
}()
}
wg.Wait()
for i, err := range errs {
if err != nil {
t.Errorf("[worker %d] %s", i, err)
deadlineCh := time.After(30 * time.Second)
for range cap(ch) {
select {
case err := <-ch:
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
case <-deadlineCh:
t.Fatalf("timeout")
}
}
}
for _, concurrency := range []int{1, 4} {
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
f(t, concurrency, disablePerDayIndex)
// Repeat the same test on non-empty reopened storage.
f(t, concurrency, disablePerDayIndex)
})
}
}
s.tb.PutPartition(ptw)
s.MustClose()
fs.MustRemoveDir(path)
})
}
func testIndexDBGetOrCreateTSIDByName(db *indexDB, metricGroups int, date uint64) ([]MetricName, []TSID, error) {
func testIndexDBGetOrCreateTSIDByName(db *indexDB, metricGroups int, timestamp int64) ([]MetricName, []TSID, error) {
r := rand.New(rand.NewSource(1))
// Create tsids.
var mns []MetricName
@@ -551,6 +564,8 @@ func testIndexDBGetOrCreateTSIDByName(db *indexDB, metricGroups int, date uint64
is := db.getIndexSearch(noDeadline)
date := uint64(timestamp) / msecPerDay
var metricNameBuf []byte
for i := range 401 {
var mn MetricName
@@ -587,7 +602,7 @@ func testIndexDBGetOrCreateTSIDByName(db *indexDB, metricGroups int, date uint64
return mns, tsids, nil
}
func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, date uint64, tr TimeRange, isConcurrent bool) error {
func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, timestamp int64, isConcurrent bool) error {
timeseriesCounters := make(map[uint64]bool)
var tsidLocal TSID
var metricNameCopy []byte
@@ -603,7 +618,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, dat
metricName := mn.Marshal(nil)
is := db.getIndexSearch(noDeadline)
if !is.getTSIDByMetricName(&tsidLocal, metricName, date) {
if !is.getTSIDByMetricName(&tsidLocal, metricName, uint64(timestamp)/msecPerDay) {
return fmt.Errorf("cannot obtain tsid #%d for mn %s", i, mn)
}
db.putIndexSearch(is)
@@ -637,7 +652,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, dat
}
// Test SearchLabelValues
lvs, err := db.SearchLabelValues(nil, "__name__", nil, tr, 1e5, 1e9, noDeadline)
lvs, err := db.SearchLabelValues(nil, "__name__", nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("error in SearchLabelValues(labelName=%q): %w", "__name__", err)
}
@@ -646,7 +661,7 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, dat
}
for i := range mn.Tags {
tag := &mn.Tags[i]
lvs, err := db.SearchLabelValues(nil, string(tag.Key), nil, tr, 1e5, 1e9, noDeadline)
lvs, err := db.SearchLabelValues(nil, string(tag.Key), nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("error in SearchLabelValues(labelName=%q): %w", tag.Key, err)
}
@@ -657,10 +672,10 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, dat
}
}
// Test SearchLabelNames (empty filter)
lns, err := db.SearchLabelNames(nil, nil, tr, 1e5, 1e9, noDeadline)
// Test SearchLabelNames (empty filters, global time range)
lns, err := db.SearchLabelNames(nil, nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("error in SearchLabelNames(empty filter): %w", err)
return fmt.Errorf("error in SearchLabelNames(empty filter, global time range): %w", err)
}
if _, ok := lns["__name__"]; !ok {
return fmt.Errorf("cannot find __name__ in %q", lns)
@@ -685,6 +700,10 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, dat
}
// Try tag filters.
tr := TimeRange{
MinTimestamp: timestamp - msecPerDay,
MaxTimestamp: timestamp + msecPerDay,
}
for i := range mns {
mn := &mns[i]
tsid := &tsids[i]

View File

@@ -17,7 +17,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/syncwg"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
)
@@ -131,9 +130,6 @@ type partition struct {
// wg.Add() must be called under partsLock after checking whether stopCh isn't closed.
// This should prevent from calling wg.Add() after stopCh is closed and wg.Wait() is called.
wg sync.WaitGroup
// Use syncwg instead of sync, since Add/Wait may be called from concurrent goroutines.
flushPendingItemsWG syncwg.WaitGroup
}
// partWrapper is a wrapper for the part.
@@ -885,9 +881,6 @@ func (pt *partition) MustClose() {
func (pt *partition) DebugFlush() {
pt.idb.tb.DebugFlush()
pt.flushPendingRows(true)
// Wait for background flushers to finish.
pt.flushPendingItemsWG.Wait()
}
func (pt *partition) startInmemoryPartsMergers() {
@@ -1018,9 +1011,7 @@ func (pt *partition) pendingRowsFlusher() {
}
func (pt *partition) flushPendingRows(isFinal bool) {
pt.flushPendingItemsWG.Add(1)
pt.rawRows.flush(pt.flushRowssToInmemoryParts, isFinal)
pt.flushPendingItemsWG.Done()
}
func (pt *partition) flushInmemoryRowsToFiles() {

View File

@@ -8,7 +8,6 @@ import (
"reflect"
"regexp"
"sort"
"sync"
"testing"
"testing/quick"
"time"
@@ -192,67 +191,6 @@ func TestSearch_VariousTimeRanges(t *testing.T) {
testStorageOpOnVariousTimeRanges(t, f)
}
// TestStorageAddFlushSearchDataConcurrently verifies that concurrent goroutines
// can read their own writes.
//
// This test focuses on reading the data. For reading the index see
// TestStorageAddFlushSearchMetricNamesConcurrently in storage_test.go.
func TestStorageAddFlushSearchDataConcurrently(t *testing.T) {
defer testRemoveAll(t)
s := MustOpenStorage(t.Name(), OpenOptions{})
defer s.MustClose()
const numMetrics = 100
f := func(workerID int, tr TimeRange) error {
mrs := make([]MetricRow, numMetrics)
step := (tr.MaxTimestamp - tr.MinTimestamp) / int64(numMetrics)
for i := range numMetrics {
name := fmt.Sprintf("metric_%04d_%04d", workerID, i)
mn := MetricName{MetricGroup: []byte(name)}
mrs[i].MetricNameRaw = mn.marshalRaw(nil)
mrs[i].Timestamp = tr.MinTimestamp + int64(i)*step
mrs[i].Value = float64(i)
}
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
tfs := NewTagFilters()
re := fmt.Sprintf(`metric_%04d.*`, workerID)
if err := tfs.Add(nil, []byte(re), false, true); err != nil {
return fmt.Errorf("tfs.Add(%q) failed unexpectedly: %w", re, err)
}
return testAssertSearchResult(s, tr, tfs, mrs)
}
const concurrency = 20
var wg sync.WaitGroup
errs := make([]error, concurrency)
for workerID := range concurrency {
wg.Go(func() {
for m := time.Month(1); m <= 12; m++ {
tr := TimeRange{
MinTimestamp: time.Date(2025, m, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(2025, m+1, 0, 0, 0, 0, 0, time.UTC).UnixMilli() - 1,
}
err := f(workerID, tr)
if err != nil {
errs[workerID] = fmt.Errorf("worker %d failed on tr=%v: %w", workerID, &tr, err)
break
}
}
})
}
wg.Wait()
for _, err := range errs {
if err != nil {
t.Errorf("%s", err)
}
}
}
func testSearchInternal(s *Storage, tr TimeRange, mrs []MetricRow) error {
for i := range 10 {
// Prepare TagFilters for search.

View File

@@ -551,7 +551,7 @@ func TestStorageAddRows_nextDayIndexPrefill(t *testing.T) {
nextDaySlowInserts := m.SlowPerDayIndexInserts
slowInserts := nextDaySlowInserts - currDaySlowInserts
if slowInserts >= numSeries {
t.Fatalf("unexpected amount of slow inserts: got %d, want < %d", slowInserts, numSeries)
t.Errorf("unexpected amount of slow inserts: got %d, want < %d", slowInserts, numSeries)
}
})
@@ -842,7 +842,7 @@ func TestStorageLastPartitionMetrics(t *testing.T) {
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
if got := s.newTimeseriesCreated.Load(); got != want {
t.Fatalf("unexpected number of new timeseries: got %d, want %d", got, want)
t.Errorf("unexpected number of new timeseries: got %d, want %d", got, want)
}
// wait for merged parts to be attached to the table
time.Sleep(time.Minute)
@@ -1218,7 +1218,7 @@ func TestStorage_denyQueriesOutsideRetention(t *testing.T) {
slices.Sort(gotData)
if diff := cmp.Diff(wantData, gotData); diff != "" {
t.Fatalf("unexpected tag value suffixes (-want, +got):\n%s", diff)
t.Errorf("unexpected tag value suffixes (-want, +got):\n%s", diff)
}
}
@@ -1487,100 +1487,3 @@ func TestStorageAddRows_MaxBackfillAge(t *testing.T) {
s.MustClose()
})
}
func TestStorageAddRows_currHourMetricIDs(t *testing.T) {
defer testRemoveAll(t)
f := func(t *testing.T, disablePerDayIndex bool) {
synctest.Test(t, func(t *testing.T) {
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
now := time.Now().UTC()
currHourTR := TimeRange{
MinTimestamp: time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 59, 59, 999_999_999, time.UTC).UnixMilli(),
}
currHour := uint64(currHourTR.MinTimestamp / 1000 / 3600)
prevHourTR := TimeRange{
MinTimestamp: currHourTR.MinTimestamp - msecPerHour,
MaxTimestamp: currHourTR.MaxTimestamp - msecPerHour,
}
rng := rand.New(rand.NewSource(1))
// Test current hour metricIDs population when data ingestion takes the
// slow path. The database is empty, therefore the index and the
// tsidCache contain no metricIDs, therefore the data ingestion will
// take slow path.
mrs := testGenerateMetricRowsWithPrefix(rng, 1000, "slow_path", currHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 1000; got != want {
t.Fatalf("[slow path] unexpected current hour metric ID count: got %d, want %d", got, want)
}
// Test current hour metricIDs population when data ingestion takes the
// fast path (when the metricIDs are found in the tsidCache)
// First insert samples to populate the tsidCache. The samples belong to
// the previous hour, therefore the metricIDs won't be added to
// currHourMetricIDs.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "fast_path", prevHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 1000; got != want {
t.Fatalf("[fast path] unexpected current hour metric ID count after ingesting samples for previous hour: got %d, want %d", got, want)
}
// Now ingest the same metrics. This time the metricIDs will be found in
// tsidCache so the ingestion will take the fast path.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "fast_path", currHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 2000; got != want {
t.Fatalf("[fast path] unexpected current hour metric ID count: got %d, want %d", got, want)
}
// Test current hour metricIDs population when data ingestion takes the
// slower path (when the metricIDs are not found in the tsidCache but
// found in the index)
// First insert samples to populate the index. The samples belong to
// the previous hour, therefore the metricIDs won't be added to
// currHourMetricIDs.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "slower_path", prevHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 2000; got != want {
t.Fatalf("[slower path] unexpected current hour metric ID count after ingesting samples for previous hour: got %d, want %d", got, want)
}
// Inserted samples were also added to the tsidCache. Drop it to
// enforce the fallback to index search.
s.resetAndSaveTSIDCache()
// Now ingest the same metrics. This time the metricIDs will be searched
// and found in index so the ingestion will take the slower path.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "slower_path", currHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 3000; got != want {
t.Fatalf("[slower path] unexpected current hour metric ID count: got %d, want %d", got, want)
}
})
}
t.Run("disablePerDayIndex=false", func(t *testing.T) {
f(t, false)
})
t.Run("disablePerDayIndex=true", func(t *testing.T) {
f(t, true)
})
}

View File

@@ -528,87 +528,113 @@ func TestStorageDeletePendingSeries(t *testing.T) {
}
func TestStorageDeleteSeries(t *testing.T) {
defer testRemoveAll(t)
path := "TestStorageDeleteSeries"
s := MustOpenStorage(path, OpenOptions{})
for _, concurrency := range []int{1, 4} {
for _, disablePerDayIndex := range []bool{false, true} {
name := fmt.Sprintf("concurrency=%d/disablePerDayIndex=%t", concurrency, disablePerDayIndex)
t.Run(name, func(t *testing.T) {
testStorageDeleteSeries(t, concurrency, disablePerDayIndex)
})
}
// Verify no label names exist
lns, err := s.SearchLabelNames(nil, nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
t.Fatalf("error in SearchLabelNames() at the start: %s", err)
}
}
func testStorageDeleteSeries(t *testing.T, concurrency int, disablePerDayIndex bool) {
tr := TimeRange{
MinTimestamp: time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(2026, 1, 15, 23, 59, 59, 999_999_999, time.UTC).UnixMilli(),
if len(lns) != 0 {
t.Fatalf("found non-empty tag keys at the start: %q", lns)
}
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
errs := make([]error, concurrency)
var wg sync.WaitGroup
for workerNum := range concurrency {
wg.Go(func() {
var err error
for range 10 {
err = testStorageDeleteSeriesForWorker(workerNum, s, tr)
if err != nil {
break
}
t.Run("serial", func(t *testing.T) {
for i := range 3 {
if err := testStorageDeleteSeries(s, 0); err != nil {
t.Fatalf("unexpected error on iteration %d: %s", i, err)
}
errs[workerNum] = err
})
}
wg.Wait()
for i, err := range errs {
if err != nil {
t.Fatalf("[worker %d] %s", i, err)
// Re-open the storage in order to check how deleted metricIDs
// are persisted.
s.MustClose()
s = MustOpenStorage(path, OpenOptions{})
}
})
t.Run("concurrent", func(t *testing.T) {
ch := make(chan error, 3)
for i := range cap(ch) {
go func(workerNum int) {
var err error
for range 2 {
err = testStorageDeleteSeries(s, workerNum)
if err != nil {
break
}
}
ch <- err
}(i)
}
tt := time.NewTimer(30 * time.Second)
for i := range cap(ch) {
select {
case err := <-ch:
if err != nil {
t.Fatalf("unexpected error on iteration %d: %s", i, err)
}
case <-tt.C:
t.Fatalf("timeout on iteration %d", i)
}
}
})
// Verify no more tag keys exist
lns, err = s.SearchLabelNames(nil, nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
t.Fatalf("error in SearchLabelNames after the test: %s", err)
}
if len(lns) != 0 {
t.Fatalf("found non-empty tag keys after the test: %q", lns)
}
s.MustClose()
fs.MustRemoveDir(path)
}
func testStorageDeleteSeriesForWorker(workerNum int, s *Storage, tr TimeRange) error {
func testStorageDeleteSeries(s *Storage, workerNum int) error {
rng := rand.New(rand.NewSource(1))
const rowsPerMetric = 100
const metricsCount = 30
workerTag := fmt.Appendf(nil, "workerTag_%d", workerNum)
lnsAll := make(map[string]bool)
lnsAll["__name__"] = true
for i := range metricsCount {
mn := MetricName{
MetricGroup: fmt.Appendf(nil, "metric_%d_%d", i, workerNum),
Tags: []Tag{
{[]byte("job"), fmt.Appendf(nil, "job_%d_%d", i, workerNum)},
{[]byte("instance"), fmt.Appendf(nil, "instance_%d_%d", i, workerNum)},
{workerTag, []byte("foobar")},
},
var mrs []MetricRow
var mn MetricName
job := fmt.Sprintf("job_%d_%d", i, workerNum)
instance := fmt.Sprintf("instance_%d_%d", i, workerNum)
mn.Tags = []Tag{
{[]byte("job"), []byte(job)},
{[]byte("instance"), []byte(instance)},
{workerTag, []byte("foobar")},
}
for i := range mn.Tags {
lnsAll[string(mn.Tags[i].Key)] = true
}
mn.MetricGroup = fmt.Appendf(nil, "metric_%d_%d", i, workerNum)
metricNameRaw := mn.marshalRaw(nil)
var mrs []MetricRow
for range rowsPerMetric {
mrs = append(mrs, MetricRow{
MetricNameRaw: mn.marshalRaw(nil),
Timestamp: tr.MinTimestamp + rng.Int63n(tr.MaxTimestamp-tr.MinTimestamp),
Value: rng.NormFloat64() * 1e6,
})
timestamp := rng.Int63n(1e10)
value := rng.NormFloat64() * 1e6
mr := MetricRow{
MetricNameRaw: metricNameRaw,
Timestamp: timestamp,
Value: value,
}
mrs = append(mrs, mr)
}
s.AddRows(mrs, defaultPrecisionBits)
}
s.DebugFlush()
// Verify tag values exist
tvs, err := s.SearchLabelValues(nil, string(workerTag), nil, tr, 1e5, 1e9, noDeadline)
tvs, err := s.SearchLabelValues(nil, string(workerTag), nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("error in SearchLabelValues before metrics removal: %w", err)
}
@@ -617,7 +643,7 @@ func testStorageDeleteSeriesForWorker(workerNum int, s *Storage, tr TimeRange) e
}
// Verify tag keys exist
lns, err := s.SearchLabelNames(nil, nil, tr, 1e5, 1e9, noDeadline)
lns, err := s.SearchLabelNames(nil, nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("error in SearchLabelNames before metrics removal: %w", err)
}
@@ -625,18 +651,20 @@ func testStorageDeleteSeriesForWorker(workerNum int, s *Storage, tr TimeRange) e
return fmt.Errorf("unexpected label names before metrics removal: %w", err)
}
countMetricBlocks := func(tfs *TagFilters) (int, error) {
var sr Search
sr.Init(nil, s, []*TagFilters{tfs}, tr, 1e5, noDeadline)
defer sr.MustClose()
var sr Search
tr := TimeRange{
MinTimestamp: 0,
MaxTimestamp: 2e10,
}
metricBlocksCount := func(tfs *TagFilters) int {
// Verify the number of blocks
n := 0
sr.Init(nil, s, []*TagFilters{tfs}, tr, 1e5, noDeadline)
for sr.NextMetricBlock() {
n++
}
if err := sr.Error(); err != nil {
return 0, err
}
return n, nil
sr.MustClose()
return n
}
for i := range metricsCount {
tfs := NewTagFilters()
@@ -647,26 +675,17 @@ func testStorageDeleteSeriesForWorker(workerNum int, s *Storage, tr TimeRange) e
if err := tfs.Add([]byte("job"), []byte(job), false, false); err != nil {
return fmt.Errorf("cannot add job tag filter: %w", err)
}
n, err := countMetricBlocks(tfs)
if err != nil {
return fmt.Errorf("failed to count metric blocks for tfs=%s: %w", tfs, err)
}
if n == 0 {
if n := metricBlocksCount(tfs); n == 0 {
return fmt.Errorf("expecting non-zero number of metric blocks for tfs=%s", tfs)
}
deletedCount, err := s.DeleteSeries(nil, []*TagFilters{tfs}, 1e9)
if err != nil {
return fmt.Errorf("cannot delete metrics: %w", err)
}
s.DebugFlush()
if deletedCount == 0 {
return fmt.Errorf("expecting non-zero number of deleted metrics on iteration %d", i)
}
n, err = countMetricBlocks(tfs)
if err != nil {
return fmt.Errorf("failed to count metric blocks for tfs=%s: %w", tfs, err)
}
if n != 0 {
if n := metricBlocksCount(tfs); n != 0 {
return fmt.Errorf("expecting zero metric blocks after DeleteSeries call for tfs=%s; got %d blocks", tfs, n)
}
@@ -675,7 +694,6 @@ func testStorageDeleteSeriesForWorker(workerNum int, s *Storage, tr TimeRange) e
if err != nil {
return fmt.Errorf("cannot delete empty tfss: %w", err)
}
s.DebugFlush()
if deletedCount != 0 {
return fmt.Errorf("expecting zero deleted metrics for empty tfss; got %d", deletedCount)
}
@@ -686,14 +704,10 @@ func testStorageDeleteSeriesForWorker(workerNum int, s *Storage, tr TimeRange) e
if err := tfs.Add(nil, fmt.Appendf(nil, "metric_.+_%d", workerNum), false, true); err != nil {
return fmt.Errorf("cannot add regexp tag filter for worker metrics: %w", err)
}
n, err := countMetricBlocks(tfs)
if err != nil {
return fmt.Errorf("failed to count metric blocks for tfs=%s: %w", tfs, err)
}
if n != 0 {
if n := metricBlocksCount(tfs); n != 0 {
return fmt.Errorf("expecting zero metric blocks after deleting all the metrics; got %d blocks", n)
}
tvs, err = s.SearchLabelValues(nil, string(workerTag), nil, tr, 1e5, 1e9, noDeadline)
tvs, err = s.SearchLabelValues(nil, string(workerTag), nil, TimeRange{}, 1e5, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("error in SearchLabelValues after all the metrics are removed: %w", err)
}
@@ -3463,6 +3477,103 @@ func TestStorageAddRows_SamplesWithZeroDate(t *testing.T) {
})
}
func TestStorageAddRows_currHourMetricIDs(t *testing.T) {
defer testRemoveAll(t)
f := func(t *testing.T, disablePerDayIndex bool) {
t.Helper()
s := MustOpenStorage(t.Name(), OpenOptions{
DisablePerDayIndex: disablePerDayIndex,
})
defer s.MustClose()
now := time.Now().UTC()
currHourTR := TimeRange{
MinTimestamp: time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 59, 59, 999_999_999, time.UTC).UnixMilli(),
}
currHour := uint64(currHourTR.MinTimestamp / 1000 / 3600)
prevHourTR := TimeRange{
MinTimestamp: currHourTR.MinTimestamp - 3600*1000,
MaxTimestamp: currHourTR.MaxTimestamp - 3600*1000,
}
rng := rand.New(rand.NewSource(1))
// Test current hour metricIDs population when data ingestion takes the
// slow path. The database is empty, therefore the index and the
// tsidCache contain no metricIDs, therefore the data ingestion will
// take slow path.
mrs := testGenerateMetricRowsWithPrefix(rng, 1000, "slow_path", currHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 1000; got != want {
t.Errorf("[slow path] unexpected current hour metric ID count: got %d, want %d", got, want)
}
// Test current hour metricIDs population when data ingestion takes the
// fast path (when the metricIDs are found in the tsidCache)
// First insert samples to populate the tsidCache. The samples belong to
// the previous hour, therefore the metricIDs won't be added to
// currHourMetricIDs.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "fast_path", prevHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 1000; got != want {
t.Errorf("[fast path] unexpected current hour metric ID count after ingesting samples for previous hour: got %d, want %d", got, want)
}
// Now ingest the same metrics. This time the metricIDs will be found in
// tsidCache so the ingestion will take the fast path.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "fast_path", currHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 2000; got != want {
t.Errorf("[fast path] unexpected current hour metric ID count: got %d, want %d", got, want)
}
// Test current hour metricIDs population when data ingestion takes the
// slower path (when the metricIDs are not found in the tsidCache but
// found in the index)
// First insert samples to populate the index. The samples belong to
// the previous hour, therefore the metricIDs won't be added to
// currHourMetricIDs.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "slower_path", prevHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 2000; got != want {
t.Errorf("[slower path] unexpected current hour metric ID count after ingesting samples for previous hour: got %d, want %d", got, want)
}
// Inserted samples were also added to the tsidCache. Drop it to
// enforce the fallback to index search.
s.resetAndSaveTSIDCache()
// Now ingest the same metrics. This time the metricIDs will be searched
// and found in index so the ingestion will take the slower path.
mrs = testGenerateMetricRowsWithPrefix(rng, 1000, "slower_path", currHourTR)
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
s.updateCurrHourMetricIDs(currHour)
if got, want := s.currHourMetricIDs.Load().m.Len(), 3000; got != want {
t.Errorf("[slower path] unexpected current hour metric ID count: got %d, want %d", got, want)
}
}
t.Run("disablePerDayIndex=false", func(t *testing.T) {
f(t, false)
})
t.Run("disablePerDayIndex=true", func(t *testing.T) {
f(t, true)
})
}
// testSearchMetricIDs returns metricIDs for the given tfss and tr.
//
// The returned metricIDs are sorted. The function panics in in case of error.
@@ -4310,82 +4421,3 @@ func TestStorage_futureTimestamps(t *testing.T) {
})
})
}
// TestStorageAddFlushSearchMetricNamesConcurrently verifies that concurrent
// goroutines can read their own writes.
//
// This test focuses on reading the index. For reading the data see
// TestStorageAddFlushSearchDataConcurrently in search_test.go.
func TestStorageAddFlushSearchMetricNamesConcurrently(t *testing.T) {
defer testRemoveAll(t)
s := MustOpenStorage(t.Name(), OpenOptions{})
defer s.MustClose()
const numMetrics = 100
f := func(workerID int, tr TimeRange) error {
mrs := make([]MetricRow, numMetrics)
want := make([]string, numMetrics)
step := (tr.MaxTimestamp - tr.MinTimestamp) / int64(numMetrics)
for i := range numMetrics {
timestamp := tr.MinTimestamp + int64(i)*step
name := fmt.Sprintf("metric_%04d_%d", workerID, timestamp)
want[i] = name
mn := MetricName{MetricGroup: []byte(name)}
mrs[i].MetricNameRaw = mn.marshalRaw(nil)
mrs[i].Timestamp = timestamp
}
s.AddRows(mrs, defaultPrecisionBits)
s.DebugFlush()
tfs := NewTagFilters()
re := fmt.Sprintf(`metric_%04d.*`, workerID)
if err := tfs.Add(nil, []byte(re), false, true); err != nil {
return fmt.Errorf("tfs.Add(%q) failed unexpectedly: %w", re, err)
}
got, err := s.SearchMetricNames(nil, []*TagFilters{tfs}, tr, 1e9, noDeadline)
if err != nil {
return fmt.Errorf("SearchMetricNames(%v) failed unexpectedly: %w", tfs, err)
}
for i, name := range got {
var mn MetricName
if err := mn.UnmarshalString(name); err != nil {
return fmt.Errorf("Could not unmarshal metric name %q: %w", name, err)
}
got[i] = string(mn.MetricGroup)
}
slices.Sort(got)
if diff := cmp.Diff(want, got); diff != "" {
return fmt.Errorf("unexpected metric names (-want, +got):\n%s", diff)
}
return nil
}
const concurrency = 20
var wg sync.WaitGroup
errs := make([]error, concurrency)
for workerID := range concurrency {
wg.Go(func() {
for m := time.Month(1); m <= 12; m++ {
tr := TimeRange{
MinTimestamp: time.Date(2025, m, 1, 0, 0, 0, 0, time.UTC).UnixMilli(),
MaxTimestamp: time.Date(2025, m+1, 0, 0, 0, 0, 0, time.UTC).UnixMilli() - 1,
}
err := f(workerID, tr)
if err != nil {
errs[workerID] = fmt.Errorf("worker %d failed on tr=%v: %w", workerID, &tr, err)
break
}
}
})
}
wg.Wait()
for _, err := range errs {
if err != nil {
t.Errorf("%s", err)
}
}
}