Compare commits

..

20 Commits

Author SHA1 Message Date
Max Kotliar
cca659365a review comments
- Remove info near "Reload" button. It does not make much sense and
buggy in FF.
- Instead info, added a confirmation pop-up that warns about lost of all
changes
- Added `# -promemtheus.config` to metrics and targets debug pages.
- Commeent `# -remoteWrite.urlRelabelConfig=xxx` now reflects selected
RW.
2026-06-30 15:28:34 +03:00
Max Kotliar
84f9a27971 Merge remote-tracking branch 'opensource/master' into feature/enhance-relabel-debug 2026-06-30 14:16:43 +03:00
Jiekun
669477e22c revert unnecessary changes 2026-06-22 17:41:27 +08:00
Jiekun
2d1ca10dda html: use only 1 textarea for relabel rule input in debug, and shows reload button only for scrape target metrics relabel debug. display yaml comments starting with # in gray 2026-06-22 17:17:36 +08:00
Zhu Jiekun
2843f442da Update docs/victoriametrics/changelog/CHANGELOG.md
Co-authored-by: Max Kotliar <mkotlyar@victoriametrics.com>
Signed-off-by: Zhu Jiekun <jiekun@victoriametrics.com>
2026-06-16 16:01:02 +08:00
Jiekun
9c7196d065 html: hide the reload button if there's no per-URL relabeling rule configured 2026-06-16 15:33:08 +08:00
Jiekun
f8e9bfd62b relabel: add tests for multi relabels 2026-06-16 11:57:31 +08:00
Jiekun
9d960bd6c5 relabel: display currenct selection of relabelconfig correctly 2026-06-16 11:45:15 +08:00
Zhu Jiekun
3b3019fb67 Update lib/promscrape/relabel_debug.go
Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
Signed-off-by: Zhu Jiekun <jiekun@victoriametrics.com>
2026-06-12 11:29:55 +08:00
Zhu Jiekun
af8720bc86 Merge branch 'master' into feature/enhance-relabel-debug 2026-06-12 11:23:56 +08:00
Jiekun
ecdea28021 revert unnecessary change on WriteURLRelabelConfigData 2026-06-12 11:18:39 +08:00
Jiekun
e0ee9be080 feature: remote write relabel debug 2026-06-12 11:11:40 +08:00
Jiekun
41bf228bb2 feature: remote write relabel debug 2026-06-11 16:48:51 +08:00
Jiekun
b8d60bb716 doc: solve conflict 2026-06-11 11:16:06 +08:00
Jiekun
6db36e244c feature: [relabel debug] remove unnecessary comments 2026-03-08 02:33:25 +08:00
Jiekun
abfd742a0f feature: [relabel debug] remove unnecessary comments 2026-03-08 02:32:47 +08:00
Jiekun
937e3654f3 feature: [relabel debug] simplify the functions 2026-03-08 02:32:11 +08:00
Jiekun
bcbe6d98cc feature: [relabel debug] fix incorrect init 2026-03-08 02:22:51 +08:00
Jiekun
c00ecdde57 feature: [relabel debug] add changelog 2026-03-08 02:16:11 +08:00
Jiekun
ef5174fef3 feature: [relabel debug] add remote write relabel config to debug page 2026-03-08 02:13:52 +08:00
94 changed files with 1165 additions and 5505 deletions

View File

@@ -447,7 +447,7 @@ vet:
go vet ./app/...
go vet ./apptest/...
check-all: fmt vet golangci-lint
check-all: fmt vet golangci-lint govulncheck
clean-checkers: remove-golangci-lint remove-govulncheck
@@ -471,9 +471,8 @@ test-full-386:
apptest:
$(MAKE) victoria-metrics-race vmagent-race vmalert-race vmauth-race vmctl-race vmbackup-race vmrestore-race
go test ./apptest/... -skip="^Test(Cluster|Mixed|Legacy).*"
go test ./apptest/... -skip="^Test(Cluster|Legacy).*"
# App tests for legacy indexDB
apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
@@ -490,20 +489,6 @@ apptest-legacy: victoria-metrics-race vmbackup-race vmrestore-race
VMSTORAGE_V1_132_0_PATH=$${DIR}/vmstorage-prod \
go test ./apptest/tests -run="^TestLegacySingle.*"
# App tests for mixed setups where vmsingle and vmcluster coexist.
apptest-mixed: victoria-metrics-race
OS=$$(uname | tr '[:upper:]' '[:lower:]'); \
ARCH=$$(uname -m | tr '[:upper:]' '[:lower:]' | sed 's/x86_64/amd64/'); \
VERSION=v1.145.0; \
VMCLUSTER=victoria-metrics-$${OS}-$${ARCH}-$${VERSION}-cluster.tar.gz; \
URL=https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/$${VERSION}; \
DIR=/tmp/$${VERSION}; \
test -d $${DIR} || (mkdir $${DIR} && \
curl --output-dir /tmp -LO $${URL}/$${VMCLUSTER} && tar xzf /tmp/$${VMCLUSTER} -C $${DIR} \
); \
VMSELECT_PATH=$${DIR}/vmselect-prod \
go test ./apptest/tests -run="^TestMixed.*"
benchmark:
go test -run=NO_TESTS -bench=. ./lib/...
go test -run=NO_TESTS -bench=. ./app/...

View File

@@ -89,7 +89,7 @@ func main() {
}
logger.Infof("starting VictoriaMetrics at %q...", listenAddrs)
startTime := time.Now()
vmstorage.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
vmstorage.Init(*vmselectMaxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
vmselect.Init(*vmselectMaxConcurrentRequests, *vmselectMaxQueueDuration)
vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
vminsert.Init()

View File

@@ -51,7 +51,7 @@ func InsertHandlerForHTTP(at *auth.Token, req *http.Request) error {
}
q := req.URL.Query()
precision := q.Get("precision")
// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
db := q.Get("db")
encoding := req.Header.Get("Content-Encoding")
isStreamMode := req.Header.Get("Stream-Mode") == "1"

View File

@@ -462,7 +462,9 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
return true
case "/prometheus/metric-relabel-debug", "/metric-relabel-debug":
promscrapeMetricRelabelDebugRequests.Inc()
promscrape.WriteMetricRelabelDebug(w, r)
rwGlobalRelabelConfigs := remotewrite.GetRemoteWriteRelabelConfigString()
rwURLRelabelConfigss := remotewrite.GetURLRelabelConfigString()
promscrape.WriteMetricRelabelDebug(w, r, rwGlobalRelabelConfigs, rwURLRelabelConfigss)
return true
case "/prometheus/target-relabel-debug", "/target-relabel-debug":
promscrapeTargetRelabelDebugRequests.Inc()

View File

@@ -35,9 +35,6 @@ func InsertHandler(at *auth.Token, req *http.Request) error {
}
func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.MetricMetadata, extraLabels []prompb.Label) error {
if len(extraLabels) == 0 && !prommetadata.IsEnabled() && at == nil {
return insertRowsFast(at, timeseries)
}
ctx := common.GetPushCtx()
defer common.PutPushCtx(ctx)
@@ -105,17 +102,3 @@ func insertRows(at *auth.Token, timeseries []prompb.TimeSeries, mms []prompb.Met
rowsPerInsert.Update(float64(rowsTotal))
return nil
}
func insertRowsFast(at *auth.Token, timeseries []prompb.TimeSeries) error {
rowsTotal := 0
for i := range timeseries {
rowsTotal += len(timeseries[i].Samples)
}
wr := &prompb.WriteRequest{Timeseries: timeseries}
if !remotewrite.TryPush(at, wr) {
return remotewrite.ErrQueueFullHTTPRetry
}
rowsInserted.Add(rowsTotal)
rowsPerInsert.Update(float64(rowsTotal))
return nil
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/VictoriaMetrics/metrics"
"gopkg.in/yaml.v2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -82,6 +83,16 @@ func WriteRelabelConfigData(w io.Writer) {
_, _ = w.Write(*p)
}
// GetRemoteWriteRelabelConfigString returns -remoteWrite.relabelConfig contents in string
func GetRemoteWriteRelabelConfigString() string {
var bb bytesutil.ByteBuffer
WriteRelabelConfigData(&bb)
if bb.Len() == 0 {
return ""
}
return string(bb.B)
}
// WriteURLRelabelConfigData writes -remoteWrite.urlRelabelConfig contents to w
func WriteURLRelabelConfigData(w io.Writer) {
p := remoteWriteURLRelabelConfigData.Load()
@@ -108,6 +119,24 @@ func WriteURLRelabelConfigData(w io.Writer) {
_, _ = w.Write(d)
}
// GetURLRelabelConfigString returns -remoteWrite.urlRelabelConfig contents in []string
func GetURLRelabelConfigString() []string {
p := remoteWriteURLRelabelConfigData.Load()
if p == nil {
return nil
}
var ss []string
for i := range *remoteWriteURLs {
cfgData := (*p)[i]
var cfgDataBytes []byte
if cfgData != nil {
cfgDataBytes, _ = yaml.Marshal(cfgData)
}
ss = append(ss, string(cfgDataBytes))
}
return ss
}
func reloadRelabelConfigs() {
rcs := allRelabelConfigs.Load()
if !rcs.isSet() {

View File

@@ -12,18 +12,19 @@ import (
"sync/atomic"
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/cespare/xxhash/v2"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bloomfilter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/consistenthash"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/mdx"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/memory"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/persistentqueue"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
@@ -105,9 +106,6 @@ var (
"cannot be pushed into the configured -remoteWrite.url systems in a timely manner. See https://docs.victoriametrics.com/victoriametrics/vmagent/#disabling-on-disk-persistence")
disableMetadataPerURL = flagutil.NewArrayBool("remoteWrite.disableMetadata", "Whether to disable sending metadata to the corresponding -remoteWrite.url. "+
"By default, metadata sending is controlled by the global -enableMetadata flag")
enableMdx = flagutil.NewArrayBool("remoteWrite.mdx.enable", "Whether to only retain metrics from VictoriaMetrics services before sending them to the corresponding -remoteWrite.url. "+
"Please see https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange")
)
var (
@@ -164,8 +162,8 @@ func InitSecretFlags() {
}
var (
shardByURLLabelsFilter []string
shardByURLIgnoreLabelsFilter []string
shardByURLLabelsMap map[string]struct{}
shardByURLIgnoreLabelsMap map[string]struct{}
)
// Init initializes remotewrite.
@@ -212,8 +210,8 @@ func Init() {
logger.Fatalf("-remoteWrite.shardByURL.labels and -remoteWrite.shardByURL.ignoreLabels cannot be set simultaneously; " +
"see https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages")
}
shardByURLLabelsFilter = slices.Clone(*shardByURLLabels)
shardByURLIgnoreLabelsFilter = slices.Clone(*shardByURLIgnoreLabels)
shardByURLLabelsMap = newMapFromStrings(*shardByURLLabels)
shardByURLIgnoreLabelsMap = newMapFromStrings(*shardByURLIgnoreLabels)
initLabelsGlobal()
@@ -309,10 +307,6 @@ func initRemoteWriteCtxs(urls []string) {
}
fs.RegisterPathFsMetrics(*tmpDataPath)
if slices.Contains(*enableMdx, true) && *shardByURL {
logger.Fatalf("-remoteWrite.mdx.enable and -remoteWrite.shardByURL cannot be set to true simultaneously.")
}
if *shardByURL {
consistentHashNodes := make([]string, 0, len(urls))
for i, url := range urls {
@@ -571,14 +565,6 @@ func tryPushMetadataToRemoteStorages(at *auth.Token, rwctxs []*remoteWriteCtx, m
mm.ProjectID = at.ProjectID
}
}
tmp := mms[:0]
for _, mm := range mms {
if timeserieslimits.IsMetricMetadataExceeding(&mm) {
continue
}
tmp = append(tmp, mm)
}
mms = tmp
// Do not shard metadata even if -remoteWrite.shardByURL is set, just replicate it among rwctxs.
// Since metadata is usually small and there is no guarantee that metadata can be sent to
// the same remote storage with the corresponding metrics.
@@ -712,18 +698,18 @@ func shardAmountRemoteWriteCtx(tssBlock []prompb.TimeSeries, shards [][]prompb.T
for _, ts := range tssBlock {
hashLabels := ts.Labels
if len(shardByURLLabelsFilter) > 0 {
if len(shardByURLLabelsMap) > 0 {
hashLabels = tmpLabels.Labels[:0]
for _, label := range ts.Labels {
if slices.Contains(shardByURLLabelsFilter, label.Name) {
if _, ok := shardByURLLabelsMap[label.Name]; ok {
hashLabels = append(hashLabels, label)
}
}
tmpLabels.Labels = hashLabels
} else if len(shardByURLIgnoreLabelsFilter) > 0 {
} else if len(shardByURLIgnoreLabelsMap) > 0 {
hashLabels = tmpLabels.Labels[:0]
for _, label := range ts.Labels {
if !slices.Contains(shardByURLIgnoreLabelsFilter, label.Name) {
if _, ok := shardByURLIgnoreLabelsMap[label.Name]; !ok {
hashLabels = append(hashLabels, label)
}
}
@@ -824,26 +810,34 @@ var (
// it omits the '=' separator between label name and value for backward compatibility.
// Changing it would re-shard all series across remoteWrite targets.
func getLabelsHashForShard(labels []prompb.Label) uint64 {
var d xxhash.Digest
d.Reset()
bb := labelsHashBufPool.Get()
b := bb.B[:0]
for _, label := range labels {
_, _ = d.WriteString(label.Name)
_, _ = d.WriteString(label.Value)
b = append(b, label.Name...)
b = append(b, label.Value...)
}
return d.Sum64()
h := xxhash.Sum64(b)
bb.B = b
labelsHashBufPool.Put(bb)
return h
}
func getLabelsHash(labels []prompb.Label) uint64 {
var d xxhash.Digest
d.Reset()
bb := labelsHashBufPool.Get()
b := bb.B[:0]
for _, label := range labels {
_, _ = d.WriteString(label.Name)
_, _ = d.WriteString("=")
_, _ = d.WriteString(label.Value)
b = append(b, label.Name...)
b = append(b, '=')
b = append(b, label.Value...)
}
return d.Sum64()
h := xxhash.Sum64(b)
bb.B = b
labelsHashBufPool.Put(bb)
return h
}
var labelsHashBufPool bytesutil.ByteBufferPool
func logSkippedSeries(labels []prompb.Label, flagName string, flagValue int) {
select {
case <-logSkippedSeriesTicker.C:
@@ -868,7 +862,6 @@ type remoteWriteCtx struct {
sas atomic.Pointer[streamaggr.Aggregators]
deduplicator *streamaggr.Deduplicator
mdxFilter *mdx.Filter
streamAggrKeepInput bool
streamAggrDropInput bool
@@ -883,7 +876,6 @@ type remoteWriteCtx struct {
rowsPushedAfterRelabel *metrics.Counter
rowsDroppedByRelabel *metrics.Counter
mdxRowsPreserved *metrics.Counter
pushFailures *metrics.Counter
metadataDroppedOnPushFailure *metrics.Counter
@@ -980,6 +972,7 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
for i := range pss {
pss[i] = newPendingSeries(fq, &c.useVMProto, sf, rd)
}
rwctx := &remoteWriteCtx{
idx: argIdx,
fq: fq,
@@ -996,16 +989,6 @@ func newRemoteWriteCtx(argIdx int, remoteWriteURL *url.URL, sanitizedURL string)
}
rwctx.initStreamAggrConfig()
if enableMdx.GetOptionalArg(argIdx) {
mdxFilter := mdx.NewFilter()
rwctx.mdxFilter = mdxFilter
rwctx.mdxRowsPreserved = metrics.GetOrCreateCounter(fmt.Sprintf(`vmagent_remotewrite_mdx_rows_preserved_total{path=%q,url=%q}`, queuePath, sanitizedURL))
_ = metrics.NewGauge(fmt.Sprintf(`vmagent_remotewrite_mdx_tracked_instances{path=%q,url=%q}`, queuePath, sanitizedURL), func() float64 {
return float64(mdxFilter.VMInstancesCount())
})
}
return rwctx
}
@@ -1019,11 +1002,6 @@ func (rwctx *remoteWriteCtx) MustStop() {
rwctx.deduplicator.MustStop()
rwctx.deduplicator = nil
}
if rwctx.mdxFilter != nil {
rwctx.mdxFilter.MustStop()
rwctx.mdxFilter = nil
rwctx.mdxRowsPreserved = nil
}
for _, ps := range rwctx.pss {
ps.MustStop()
@@ -1039,7 +1017,6 @@ func (rwctx *remoteWriteCtx) MustStop() {
rwctx.rowsPushedAfterRelabel = nil
rwctx.rowsDroppedByRelabel = nil
}
// TryPushTimeSeries sends tss series to the configured remote write endpoint
@@ -1047,41 +1024,16 @@ func (rwctx *remoteWriteCtx) MustStop() {
// TryPushTimeSeries doesn't modify tss, so tss can be passed concurrently to TryPush across distinct rwctx instances.
func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDropSamplesOnFailure bool) bool {
var rctx *relabelCtx
var mctx *mdx.Ctx
var v *[]prompb.TimeSeries
defer func() {
if v != nil {
*v = prompb.ResetTimeSeries(tss)
tssPool.Put(v)
}
if rctx != nil {
putRelabelCtx(rctx)
}
if mctx != nil {
mdx.PutContext(mctx)
if rctx == nil {
return
}
*v = prompb.ResetTimeSeries(tss)
tssPool.Put(v)
putRelabelCtx(rctx)
}()
copyTimeSeriesIfNeeded := func() {
if v == nil {
v := tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
}
}
if rwctx.mdxFilter != nil {
mctx = mdx.GetContext()
// Make a copy of tss before applying relabeling in order to prevent
// from affecting time series for other remoteWrite.mdx configs.
copyTimeSeriesIfNeeded()
tss = rwctx.mdxFilter.Filter(mctx, tss)
if len(tss) == 0 {
return true
}
rowsCount := getRowsCount(tss)
rwctx.mdxRowsPreserved.Add(rowsCount)
}
// Apply relabeling
rcs := allRelabelConfigs.Load()
pcs := rcs.perURL[rwctx.idx]
@@ -1091,7 +1043,8 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
// from affecting time series for other remoteWrite.url configs.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/467
// and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/599
copyTimeSeriesIfNeeded()
v = tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
rowsCountBeforeRelabel := getRowsCount(tss)
tss = rctx.applyRelabeling(tss, pcs)
rowsCountAfterRelabel := getRowsCount(tss)
@@ -1109,7 +1062,8 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
if rctx == nil {
rctx = getRelabelCtx()
// Make a copy of tss before dropping aggregated series
copyTimeSeriesIfNeeded()
v = tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
}
tss = dropAggregatedSeries(tss, matchIdxs.B, rwctx.streamAggrDropInput)
} else if rwctx.streamAggrDropInput {
@@ -1117,7 +1071,8 @@ func (rwctx *remoteWriteCtx) TryPushTimeSeries(tss []prompb.TimeSeries, forceDro
if rctx == nil {
rctx = getRelabelCtx()
// Make a copy of tss before dropping aggregated series
copyTimeSeriesIfNeeded()
v = tssPool.Get().(*[]prompb.TimeSeries)
tss = append(*v, tss...)
}
tss = dropUnaggregatedSeries(tss, matchIdxs.B)
}
@@ -1236,6 +1191,15 @@ func getRowsCount(tss []prompb.TimeSeries) int {
}
return rowsCount
}
func newMapFromStrings(a []string) map[string]struct{} {
m := make(map[string]struct{}, len(a))
for _, s := range a {
m[s] = struct{}{}
}
return m
}
func getMaxHourlySeries() int {
limit := *maxHourlySeries
if limit == -1 || limit > math.MaxInt32 {

View File

@@ -282,8 +282,7 @@ func processFlags() {
func setUp() {
const maxConcurrentRequests = 4
maxQueueDuration := 5 * time.Second
vmstorage.Init(maxConcurrentRequests, maxQueueDuration, promql.ResetRollupResultCacheIfNeeded)
vmstorage.Init(maxConcurrentRequests, promql.ResetRollupResultCacheIfNeeded)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
readyCheckFunc := func() bool {

View File

@@ -145,10 +145,10 @@ func TestRuleValidate(t *testing.T) {
}
func TestGroupValidate_Failure(t *testing.T) {
f := func(data []byte, validateExpressions bool, errStrExpected string) {
f := func(group *Group, validateExpressions bool, errStrExpected string) {
t.Helper()
_, err := parse(map[string][]byte{"test.yaml": data}, nil, validateExpressions)
err := group.Validate(nil, validateExpressions)
if err == nil {
t.Fatalf("expecting non-nil error")
}
@@ -158,238 +158,275 @@ func TestGroupValidate_Failure(t *testing.T) {
}
}
f([]byte(`
groups:
- name: ""
`), false, "group name must be set")
f(&Group{}, false, "group name must be set")
f([]byte(`
groups:
- name: both record and alert are not set
rules:
- expr: "sum(up == 0 ) by (host)"
for: 10ms
- expr: "sumSeries(time('foo.bar',10))"
`), false, "invalid rule")
f(&Group{
Name: "both record and alert are not set",
Rules: []Rule{
{
Expr: "sum(up == 0 ) by (host)",
For: promutil.NewDuration(10 * time.Millisecond),
},
{
Expr: "sumSeries(time('foo.bar',10))",
},
},
}, false, "invalid rule")
f([]byte(`
groups:
- name: negative interval
interval: -1ms
`), false, "interval shouldn't be lower than 0")
f(&Group{
Name: "negative interval",
Interval: promutil.NewDuration(-1),
}, false, "interval shouldn't be lower than 0")
f([]byte(`
groups:
- name: too big eval_offset
interval: 1m
eval_offset: 2m
`), false, "eval_offset should be smaller than interval")
f(&Group{
Name: "too big eval_offset",
Interval: promutil.NewDuration(time.Minute),
EvalOffset: promutil.NewDuration(2 * time.Minute),
}, false, "eval_offset should be smaller than interval")
f([]byte(`
groups:
- name: too big negative eval_offset
interval: 1m
eval_offset: -2m
`), false, "eval_offset should be smaller than interval")
f(&Group{
Name: "too big negative eval_offset",
Interval: promutil.NewDuration(time.Minute),
EvalOffset: promutil.NewDuration(-2 * time.Minute),
}, false, "eval_offset should be smaller than interval")
f([]byte(`
groups:
- name: wrong limit
limit: -1
`), false, "invalid limit")
limit := -1
f(&Group{
Name: "wrong limit",
Limit: &limit,
}, false, "invalid limit")
f([]byte(`
groups:
- name: wrong concurrency
concurrency: -1
`), false, "invalid concurrency")
f(&Group{
Name: "wrong concurrency",
Concurrency: -1,
}, false, "invalid concurrency")
f([]byte(`
groups:
- name: test
rules:
- alert: alert
expr: up == 1
- alert: alert
expr: up == 1
`), false, "duplicate")
f(&Group{
Name: "test",
Rules: []Rule{
{
Alert: "alert",
Expr: "up == 1",
},
{
Alert: "alert",
Expr: "up == 1",
},
},
}, false, "duplicate")
f([]byte(`
groups:
- name: test
rules:
- alert: alert
expr: up == 1
labels:
summary: "{{ value|query }}"
- alert: alert
expr: up == 1
labels:
summary: "{{ value|query }}"
`), false, "duplicate")
f(&Group{
Name: "test",
Rules: []Rule{
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
},
}, false, "duplicate")
f([]byte(`
groups:
- name: test
rules:
- record: record
expr: up == 1
labels:
summary: "{{ value|query }}"
- record: record
expr: up == 1
labels:
summary: "{{ value|query }}"
`), false, "duplicate")
f(&Group{
Name: "test",
Rules: []Rule{
{Record: "record", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
{Record: "record", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
},
}, false, "duplicate")
f([]byte(`
groups:
- name: test thanos
type: thanos
rules:
- alert: alert
expr: up == 1
labels:
description: "{{ value|query }}"
`), true, "unknown datasource type")
f(&Group{
Name: "test",
Rules: []Rule{
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"description": "{{ value|query }}",
}},
},
}, false, "duplicate")
f(&Group{
Name: "test",
Rules: []Rule{
{Record: "alert", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"summary": "{{ value|query }}",
}},
},
}, false, "duplicate")
f(&Group{
Name: "test thanos",
Type: NewRawType("thanos"),
Rules: []Rule{
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"description": "{{ value|query }}",
}},
},
}, true, "unknown datasource type")
// validate expressions
f([]byte(`
groups:
- name: test prometheus expr
type: prometheus
rules:
- record: record
expr: "up | 0"
`), true, "bad MetricsQL expr")
f(&Group{
Name: "test prometheus expr",
Type: NewPrometheusType(),
Rules: []Rule{
{
Record: "record",
Expr: "up | 0",
},
},
}, true, "bad MetricsQL expr")
f([]byte(`
groups:
- name: test graphite expr
type: graphite
rules:
- alert: alert
expr: up == 1
labels:
description: some-description
`), true, "bad GraphiteQL expr")
f(&Group{
Name: "test graphite expr",
Type: NewGraphiteType(),
Rules: []Rule{
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"description": "some-description",
}},
},
}, true, "bad GraphiteQL expr")
f([]byte(`
groups:
- name: test vlogs expr
type: vlogs
rules:
- alert: alert
expr: "stats count(*) as requests"
`), true, "bad LogsQL expr")
f(&Group{
Name: "test vlogs expr",
Type: NewVLogsType(),
Rules: []Rule{
{Alert: "alert", Expr: "stats count(*) as requests"},
},
}, true, "bad LogsQL expr")
f([]byte(`
groups:
- name: test vlogs expr multipart
type: vlogs
rules:
- alert: alert
expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"
`), true, "bad LogsQL expr")
f(&Group{
Name: "test vlogs expr",
Type: NewVLogsType(),
Rules: []Rule{
{Alert: "alert", Expr: "_time: 1m | stats by (path, _time: 1m) count(*) as requests"},
},
}, true, "bad LogsQL expr")
f([]byte(`
groups:
- name: test graphite with prometheus expr
type: graphite
rules:
- record: r1
expr: "sumSeries(time('foo.bar',10))"
for: 10ms
- record: r2
expr: "sum(up == 0 ) by (host)"
`), true, "bad GraphiteQL expr")
f(&Group{
Name: "test graphite with prometheus expr",
Type: NewGraphiteType(),
Rules: []Rule{
{
Record: "r1",
ID: 1,
Expr: "sumSeries(time('foo.bar',10))",
For: promutil.NewDuration(10 * time.Millisecond),
},
{
Record: "r2",
ID: 2,
Expr: "sum(up == 0 ) by (host)",
},
},
}, true, "bad GraphiteQL expr")
f([]byte(`
groups:
- name: test vlogs with prometheus expr
type: vlogs
rules:
- record: r1
expr: "sum(up == 0 ) by (host)"
for: 10ms
`), true, "bad LogsQL expr")
f(&Group{
Name: "test vlogs with prometheus exp",
Type: NewVLogsType(),
Rules: []Rule{
{
Record: "r1",
Expr: "sum(up == 0 ) by (host)",
For: promutil.NewDuration(10 * time.Millisecond),
},
},
}, true, "bad LogsQL expr")
f([]byte(`
groups:
- name: test prometheus with vlogs expr
type: prometheus
rules:
- record: r1
expr: "* | stats by (path) count()"
for: 10ms
`), true, "bad MetricsQL expr")
f(&Group{
Name: "test prometheus with vlogs exp",
Type: NewPrometheusType(),
Rules: []Rule{
{
Record: "r1",
Expr: "* | stats by (path) count()",
For: promutil.NewDuration(10 * time.Millisecond),
},
},
}, true, "bad MetricsQL expr")
}
func TestGroupValidate_Success(t *testing.T) {
f := func(data []byte, validateAnnotations, validateExpressions bool) {
f := func(group *Group, validateAnnotations, validateExpressions bool) {
t.Helper()
var validateTplFn ValidateTplFn
if validateAnnotations {
validateTplFn = notifier.ValidateTemplates
}
_, err := parse(map[string][]byte{"test.yaml": data}, validateTplFn, validateExpressions)
err := group.Validate(validateTplFn, validateExpressions)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
}
f([]byte(`
groups:
- name: test
rules:
- record: record
expr: "up | 0"
`), false, false)
f(&Group{
Name: "test",
Rules: []Rule{
{
Record: "record",
Expr: "up | 0",
},
},
}, false, false)
f([]byte(`
groups:
- name: test
rules:
- alert: alert
expr: up == 1
labels:
summary: "{{ value|query }}"
`), false, false)
f(&Group{
Name: "test",
Rules: []Rule{
{
Alert: "alert",
Expr: "up == 1",
Labels: map[string]string{
"summary": "{{ value|query }}",
},
},
},
}, false, false)
// validate annotations
f([]byte(`
groups:
- name: test
rules:
- alert: alert
expr: up == 1
labels:
summary: "\n{{ with printf \"node_memory_MemTotal{job='node',instance='%s'}\" \"localhost\" | query }}\n {{ . | first | value | humanize1024 }}B\n{{ end }}"
`), true, false)
f(&Group{
Name: "test",
Rules: []Rule{
{
Alert: "alert",
Expr: "up == 1",
Labels: map[string]string{
"summary": `
{{ with printf "node_memory_MemTotal{job='node',instance='%s'}" "localhost" | query }}
{{ . | first | value | humanize1024 }}B
{{ end }}`,
},
},
},
}, true, false)
// validate expressions
f([]byte(`
groups:
- name: test prometheus
type: prometheus
rules:
- alert: alert
expr: up == 1
labels:
description: "{{ value|query }}"
`), false, true)
f([]byte(`
groups:
- name: test victorialogs
type: vlogs
rules:
- alert: alert
expr: " _time: 1m | stats count(*) as requests"
labels:
description: "{{ value|query }}"
`), false, true)
f(&Group{
Name: "test prometheus",
Type: NewPrometheusType(),
Rules: []Rule{
{Alert: "alert", Expr: "up == 1", Labels: map[string]string{
"description": "{{ value|query }}",
}},
},
}, false, true)
f(&Group{
Name: "test victorialogs",
Type: NewVLogsType(),
Rules: []Rule{
{Alert: "alert", Expr: " _time: 1m | stats count(*) as requests", Labels: map[string]string{
"description": "{{ value|query }}",
}},
},
}, false, true)
}
func TestHashRule_NotEqual(t *testing.T) {

View File

@@ -140,18 +140,6 @@ users:
- "ProjectID: {{.MetricsProjectID}}"
url_prefix: "http://vminsert:8480/insert/prometheus"
# JWT-based routing that relies solely on custom claims.
# The `vm_access` claim is missing, default value will be used.
# e.g. {"role": "admin"}.
- name: jwt-custom-claims
jwt:
skip_verify: true
vm_default_access_claim:
metrics_account_id: 1
match_claims:
role: admin
url_prefix: "http://vmselect-admin:8481/select/0/prometheus"
# Requests without Authorization header are proxied according to `unauthorized_user` section.
# Requests are proxied in round-robin fashion between `url_prefix` backends.
# The deny_partial_response query arg is added to all the proxied requests.

View File

@@ -65,8 +65,6 @@ type JWTConfig struct {
MatchClaims map[string]string `yaml:"match_claims,omitempty"`
parsedMatchClaims []*jwt.Claim
DefaultVMAccessClaim *jwt.VMAccessClaim `yaml:"default_vm_access_claim,omitempty"`
// verifierPool is used to verify JWT tokens.
// It is initialized from PublicKeys and/or PublicKeyFiles.
// In this case, it is initialized once at config reload and never updated until next reload
@@ -434,6 +432,7 @@ func validateJWTPlaceholdersForURL(up *URLPrefix, isAllowed bool) error {
}
if strings.Contains(p, placeholderPrefix) {
return fmt.Errorf("invalid placeholder found in URL request path: %q, supported values are: %s", bu.Path, strings.Join(allPlaceholders, ", "))
}
}
for param, values := range bu.Query() {
@@ -488,6 +487,7 @@ func hasAnyPlaceholders(u *url.URL) bool {
return true
}
}
}
return false
}

View File

@@ -6,7 +6,6 @@ import (
"flag"
"fmt"
"io"
"math/rand/v2"
"net"
"net/http"
"net/textproto"
@@ -32,7 +31,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
)
var (
@@ -192,10 +190,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
if tkn == nil {
logger.Panicf("BUG: unexpected nil jwt token for user %q", ui.name())
}
if !tkn.HasVMAccessClaim() && ui.JWT.DefaultVMAccessClaim == nil {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return true
}
defer putToken(tkn)
processUserRequest(w, r, ui, tkn)
return true
@@ -208,7 +202,6 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
}
invalidAuthTokenRequests.Inc()
slowdownUnauthorizedResponse(r)
if *logInvalidAuthTokens {
err := fmt.Errorf("cannot authorize request with auth tokens %q", ats)
err = &httpserver.ErrorWithStatusCode{
@@ -431,12 +424,8 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
}
targetURL := bu.url
if tkn != nil {
vmac := tkn.VMAccess()
if !tkn.HasVMAccessClaim() {
vmac = ui.JWT.DefaultVMAccessClaim
}
// for security reasons allow templating only for configured url values and headers
targetURL, hc = replaceJWTPlaceholders(bu, hc, vmac)
targetURL, hc = replaceJWTPlaceholders(bu, hc, tkn.VMAccess())
}
if isDefault {
// Don't change path and add request_path query param for default route.
@@ -892,20 +881,3 @@ func debugInfo(u *url.URL, r *http.Request) string {
fmt.Fprint(s, ")")
return s.String()
}
// slowdownUnauthorizedResponse adds a random delay in the [2..3] seconds range before returning an unauthorized response.
// This reduces the effectiveness of brute-force.
//
// Recommended by OWASP Top10:
// https://owasp.org/Top10/2025/A07_2025-Authentication_Failures
func slowdownUnauthorizedResponse(r *http.Request) {
d := 2*time.Second + time.Duration(rand.IntN(1000))*time.Millisecond
t := timerpool.Get(d)
select {
case <-t.C:
case <-r.Context().Done():
}
timerpool.Put(t)
}

View File

@@ -739,12 +739,6 @@ users:
"vm_access": map[string]any{},
}, false)
// token without vm_access claim, but with a custom claim usable for routing
roleToken := genToken(t, map[string]any{
"exp": time.Now().Add(10 * time.Minute).Unix(),
"role": "admin",
}, true)
fullToken := genToken(t, map[string]any{
"exp": time.Now().Add(10 * time.Minute).Unix(),
"vm_access": map[string]any{
@@ -785,26 +779,6 @@ statusCode=401
Unauthorized`
f(simpleCfgStr, request, responseExpected)
// token without vm_access claim is accepted when it
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
request.Header.Set(`Authorization`, `Bearer `+roleToken)
responseExpected = `
statusCode=200
path: /foo/abc
query:
headers:`
f(fmt.Sprintf(`
users:
- jwt:
public_keys:
- %q
default_vm_access_claim:
metrics_account_id: 10
metrics_project_id: 10
match_claims:
role: admin
url_prefix: {BACKEND}/foo`, string(publicKeyPEM)), request, responseExpected)
// expired token
request = httptest.NewRequest(`GET`, "http://some-host.com/abc", nil)
request.Header.Set(`Authorization`, `Bearer `+expiredToken)

View File

@@ -175,19 +175,13 @@ func (ctx *InsertCtx) WriteMetadata(mmpbs []prompb.MetricMetadata) error {
}
mms := ctx.mms
mms = slicesutil.SetLength(mms, len(mmpbs))
var cnt int
for _, mmpb := range mmpbs {
if timeserieslimits.IsMetricMetadataExceeding(&mmpb) {
continue
}
mm := &mms[cnt]
for idx, mmpb := range mmpbs {
mm := &mms[idx]
mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.MetricFamilyName)
mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
mm.Type = mmpb.Type
mm.Unit = bytesutil.ToUnsafeBytes(mmpb.Unit)
cnt++
}
mms = mms[:cnt]
ctx.mms = mms
err := vmstorage.VMInsertAPI.WriteMetadata(mms)
@@ -207,19 +201,14 @@ func (ctx *InsertCtx) WritePromMetadata(mmps []prometheus.Metadata) error {
}
mms := ctx.mms
mms = slicesutil.SetLength(mms, len(mmps))
var cnt int
for _, mmpb := range mmps {
mm := &mms[cnt]
if timeserieslimits.IsPrometheusMetadataExceeding(&mmpb) {
continue
}
for idx, mmpb := range mmps {
mm := &mms[idx]
mm.MetricFamilyName = bytesutil.ToUnsafeBytes(mmpb.Metric)
mm.Help = bytesutil.ToUnsafeBytes(mmpb.Help)
mm.Type = mmpb.Type
cnt++
}
mms = mms[:cnt]
ctx.mms = mms
err := vmstorage.VMInsertAPI.WriteMetadata(mms)
if err != nil {
return &httpserver.ErrorWithStatusCode{

View File

@@ -49,7 +49,7 @@ func InsertHandlerForHTTP(req *http.Request) error {
}
q := req.URL.Query()
precision := q.Get("precision")
// Read db tag from https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite
// Read db tag from https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint
db := q.Get("db")
encoding := req.Header.Get("Content-Encoding")
isStreamMode := req.Header.Get("Stream-Mode") == "1"

View File

@@ -541,7 +541,7 @@ func handleStaticAndSimpleRequests(w http.ResponseWriter, r *http.Request, path
return true
case "/metric-relabel-debug":
promscrapeMetricRelabelDebugRequests.Inc()
promscrape.WriteMetricRelabelDebug(w, r)
promscrape.WriteMetricRelabelDebug(w, r, "", nil)
return true
case "/target-relabel-debug":
promscrapeTargetRelabelDebugRequests.Inc()

View File

@@ -956,7 +956,6 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
start, end, step int64, r *http.Request, ct int64, etfs [][]storage.TagFilter) error {
deadline := searchutil.GetDeadlineForQuery(r, startTime)
mayCache := !httputil.GetBool(r, "nocache")
optimizeRepeatedBinaryOpSubexprs := httputil.GetBool(r, "optimize_repeated_binary_op_subexprs")
lookbackDelta, err := getMaxLookback(r)
if err != nil {
return err
@@ -978,19 +977,18 @@ func queryRangeHandler(qt *querytracer.Tracer, startTime time.Time, w http.Respo
}
ec := &promql.EvalConfig{
Start: start,
End: end,
Step: step,
MaxPointsPerSeries: *maxPointsPerTimeseries,
MaxSeries: 0, // let vmstorage use maxUniqueTimeseries by default
QuotedRemoteAddr: httpserver.GetQuotedRemoteAddr(r),
Deadline: deadline,
MayCache: mayCache,
OptimizeRepeatedBinaryOpSubexprs: optimizeRepeatedBinaryOpSubexprs,
LookbackDelta: lookbackDelta,
RoundDigits: getRoundDigits(r),
EnforcedTagFilterss: etfs,
CacheTagFilters: etfs,
Start: start,
End: end,
Step: step,
MaxPointsPerSeries: *maxPointsPerTimeseries,
MaxSeries: 0, // let vmstorage use maxUniqueTimeseries by default
QuotedRemoteAddr: httpserver.GetQuotedRemoteAddr(r),
Deadline: deadline,
MayCache: mayCache,
LookbackDelta: lookbackDelta,
RoundDigits: getRoundDigits(r),
EnforcedTagFilterss: etfs,
CacheTagFilters: etfs,
GetRequestURI: func() string {
return httpserver.GetRequestURI(r)
},

View File

@@ -172,13 +172,7 @@ func newBinaryOpFunc(bf func(left, right float64, isBool bool) float64) binaryOp
left = removeEmptySeries(left)
right = removeEmptySeries(right)
}
if len(left) == 0 && len(right) == 0 {
return nil, nil
}
if len(left) == 0 && bfa.be.FillLeft == nil {
return nil, nil
}
if len(right) == 0 && bfa.be.FillRight == nil {
if len(left) == 0 || len(right) == 0 {
return nil, nil
}
left, right, dst, err := adjustBinaryOpTags(bfa.be, left, right)
@@ -232,7 +226,7 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
}
}
// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} {fill|fill_left|fill_right} b`
// Slow path: `vector op vector` or `a op {on|ignoring} {group_left|group_right} b`
var rvsLeft, rvsRight []*timeseries
mLeft, mRight := createTimeseriesMapByTagSet(be, left, right)
joinOp := strings.ToLower(be.JoinModifier.Op)
@@ -245,27 +239,10 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
// Add __name__ to groupTags if metric name must be preserved.
groupTags = append(groupTags[:len(groupTags):len(groupTags)], "__name__")
}
// Add missing keys from mRight to mLeft when fill_left()/fill() modifier is used
if be.FillLeft != nil {
for k := range mRight {
if _, ok := mLeft[k]; !ok {
mLeft[k] = nil
}
}
}
for k, tssLeft := range mLeft {
tssRight := mRight[k]
if len(tssLeft) == 0 {
if be.FillLeft == nil {
logger.Panicf("BUG: unexpected empty tssLeft for key %q when FillLeft is nil", k)
}
tssLeft = []*timeseries{newFillTimeseries(be, tssRight[0], be.FillLeft.N)}
}
if len(tssRight) == 0 {
if be.FillRight == nil {
continue
}
tssRight = []*timeseries{newFillTimeseries(be, tssLeft[0], be.FillRight.N)}
continue
}
switch joinOp {
case "group_left":
@@ -310,27 +287,6 @@ func adjustBinaryOpTags(be *metricsql.BinaryOpExpr, left, right []*timeseries) (
return rvsLeft, rvsRight, dst, nil
}
// newFillTimeseries returns a time series filled with fillValue for the fill_left()/fill_right()/fill() modifiers.
func newFillTimeseries(be *metricsql.BinaryOpExpr, src *timeseries, fillValue float64) *timeseries {
var ts timeseries
ts.CopyFromShallowTimestamps(src)
if !be.KeepMetricNames {
ts.MetricName.ResetMetricGroup()
}
groupTags := be.GroupModifier.Args
switch strings.ToLower(be.GroupModifier.Op) {
case "on":
ts.MetricName.RemoveTagsOn(groupTags)
default:
ts.MetricName.RemoveTagsIgnoring(groupTags)
}
values := ts.Values
for i := range values {
values[i] = fillValue
}
return &ts
}
func ensureSingleTimeseries(side string, be *metricsql.BinaryOpExpr, tss []*timeseries) error {
if len(tss) == 0 {
logger.Panicf("BUG: tss must contain at least one value")

View File

@@ -132,9 +132,6 @@ type EvalConfig struct {
// Whether the response can be cached.
MayCache bool
// Whether repeated cacheable binary op subexpressions can be optimized.
OptimizeRepeatedBinaryOpSubexprs bool
// LookbackDelta is analog to `-query.lookback-delta` from Prometheus.
LookbackDelta int64
@@ -174,7 +171,6 @@ func copyEvalConfig(src *EvalConfig) *EvalConfig {
ec.MaxPointsPerSeries = src.MaxPointsPerSeries
ec.Deadline = src.Deadline
ec.MayCache = src.MayCache
ec.OptimizeRepeatedBinaryOpSubexprs = src.OptimizeRepeatedBinaryOpSubexprs
ec.LookbackDelta = src.LookbackDelta
ec.RoundDigits = src.RoundDigits
ec.EnforcedTagFilterss = src.EnforcedTagFilterss
@@ -424,7 +420,18 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
if bf == nil {
return nil, fmt.Errorf(`unknown binary op %q`, be.Op)
}
tssLeft, tssRight, err := execBinaryOpArgs(qt, ec, be)
var err error
var tssLeft, tssRight []*timeseries
switch strings.ToLower(be.Op) {
case "and", "if":
// Fetch right-side series at first, since it usually contains
// lower number of time series for `and` and `if` operator.
// This should produce more specific label filters for the left side of the query.
// This, in turn, should reduce the time to select series for the left side of the query.
tssRight, tssLeft, err = execBinaryOpArgs(qt, ec, be.Right, be.Left, be)
default:
tssLeft, tssRight, err = execBinaryOpArgs(qt, ec, be.Left, be.Right, be)
}
if err != nil {
return nil, fmt.Errorf("cannot execute %q: %w", be.AppendString(nil), err)
}
@@ -440,29 +447,6 @@ func evalBinaryOp(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOp
return rv, nil
}
// binaryOpEvalOrder might change the order of evaluation of the left and right sides of a binary operation,
// when there is chance to push down common label filters from exprFirst to exprSecond in the following executions.
func binaryOpEvalOrder(be *metricsql.BinaryOpExpr) (exprFirst, exprSecond metricsql.Expr) {
exprFirst, exprSecond = be.Left, be.Right
switch strings.ToLower(be.Op) {
case "and", "if":
// For `and` and `if`, fetch the right-side series first, since it usually contains
// fewer time series and yields more specific filters for the left side.
exprFirst, exprSecond = be.Right, be.Left
}
if be.FillLeft != nil && be.FillRight == nil {
// For `fill_left(<value>)`, the unmatched series can only come from the right side, so evaluate it first.
exprFirst, exprSecond = be.Right, be.Left
}
return exprFirst, exprSecond
}
// canPushdownCommonFilters decides if common label filters can be pushed down from one side of a binary operation to the other.
//
// Common filters cannot be pushed down when:
// - the operator is `or` or `default`;
// - either side is an aggregation function without explicit grouping;
// - fill(<value>) modifier is used.
func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
switch strings.ToLower(be.Op) {
case "or", "default":
@@ -471,10 +455,6 @@ func canPushdownCommonFilters(be *metricsql.BinaryOpExpr) bool {
if isAggrFuncWithoutGrouping(be.Left) || isAggrFuncWithoutGrouping(be.Right) {
return false
}
// Filters cannot be propagated when fill(<value>) modifier is used.
if be.FillLeft != nil && be.FillRight != nil {
return false
}
return true
}
@@ -486,17 +466,8 @@ func isAggrFuncWithoutGrouping(e metricsql.Expr) bool {
return len(afe.Modifier.Args) == 0
}
func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
exprFirst, exprSecond := binaryOpEvalOrder(be)
canPushdown := canPushdownCommonFilters(be)
firstIsLeft := exprFirst == be.Left
sortResult := func(tssFirst, tssSecond []*timeseries) ([]*timeseries, []*timeseries, error) {
if firstIsLeft {
return tssFirst, tssSecond, nil
}
return tssSecond, tssFirst, nil
}
if !canPushdown && !shouldOptimizeRepeatedBinaryOpSubexprs(ec, exprFirst, exprSecond) {
func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, exprFirst, exprSecond metricsql.Expr, be *metricsql.BinaryOpExpr) ([]*timeseries, []*timeseries, error) {
if !canPushdownCommonFilters(be) {
// Execute exprFirst and exprSecond in parallel, since it is impossible to pushdown common filters
// from exprFirst to exprSecond.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2886
@@ -527,26 +498,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.Bina
if errSecond != nil {
return nil, nil, errSecond
}
return sortResult(tssFirst, tssSecond)
}
if !canPushdown {
qt = qt.NewChild("execute left and right sides of %q sequentially because repeated cacheable subexpression was found", be.Op)
defer qt.Done()
qtFirst := qt.NewChild("expr1")
tssFirst, err := evalExpr(qtFirst, ec, exprFirst)
qtFirst.Done()
if err != nil {
return nil, nil, err
}
qtSecond := qt.NewChild("expr2")
tssSecond, err := evalExpr(qtSecond, ec, exprSecond)
qtSecond.Done()
if err != nil {
return nil, nil, err
}
return sortResult(tssFirst, tssSecond)
return tssFirst, tssSecond, nil
}
// Execute binary operation in the following way:
@@ -589,79 +541,7 @@ func execBinaryOpArgs(qt *querytracer.Tracer, ec *EvalConfig, be *metricsql.Bina
if err != nil {
return nil, nil, err
}
return sortResult(tssFirst, tssSecond)
}
func shouldOptimizeRepeatedBinaryOpSubexprs(ec *EvalConfig, exprFirst, exprSecond metricsql.Expr) bool {
if !ec.OptimizeRepeatedBinaryOpSubexprs {
return false
}
if ec.Start == ec.End {
return false
}
if !ec.mayCache() {
return false
}
candidatesFirst := make(map[string]struct{}, 1)
var b []byte
visitOptimizedAggrs(exprFirst, func(ae *metricsql.AggrFuncExpr) {
if hasUnseededVolatileFunc(ae) {
return
}
b = ae.AppendString(b[:0])
candidatesFirst[string(b)] = struct{}{}
})
if len(candidatesFirst) == 0 {
return false
}
repeated := false
visitOptimizedAggrs(exprSecond, func(ae *metricsql.AggrFuncExpr) {
if repeated {
return
}
b = ae.AppendString(b[:0])
_, repeated = candidatesFirst[string(b)]
})
return repeated
}
func visitOptimizedAggrs(e metricsql.Expr, f func(ae *metricsql.AggrFuncExpr)) {
metricsql.VisitAll(e, func(expr metricsql.Expr) {
ae, ok := expr.(*metricsql.AggrFuncExpr)
if !ok {
return
}
if getIncrementalAggrFuncCallbacks(ae.Name) == nil {
return
}
fe, _ := tryGetArgRollupFuncWithMetricExpr(ae)
if fe == nil {
return
}
f(ae)
})
}
func hasUnseededVolatileFunc(e metricsql.Expr) bool {
found := false
metricsql.VisitAll(e, func(expr metricsql.Expr) {
if found {
return
}
fe, ok := expr.(*metricsql.FuncExpr)
if !ok {
return
}
switch strings.ToLower(fe.Name) {
case "now":
found = true
case "rand", "rand_normal", "rand_exponential":
found = len(fe.Args) == 0
}
})
return found
return tssFirst, tssSecond, nil
}
func getCommonLabelFilters(tss []*timeseries) []metricsql.LabelFilter {

View File

@@ -170,87 +170,3 @@ func TestGetSumInstantValues(t *testing.T) {
[]*timeseries{ts("foo", 100, 1)},
)
}
func TestShouldOptimizeRepeatedBinaryOpSubexprsGate(t *testing.T) {
e, err := metricsql.Parse(`count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`)
if err != nil {
t.Fatalf("unexpected error in metricsql.Parse(): %s", err)
}
be, ok := e.(*metricsql.BinaryOpExpr)
if !ok {
t.Fatalf("unexpected expr type; got %T; want *metricsql.BinaryOpExpr", e)
}
f := func(name string, ec *EvalConfig, resultExpected bool) {
t.Helper()
result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
if result != resultExpected {
t.Fatalf("unexpected result for %q; got %v; want %v", name, result, resultExpected)
}
}
f("disabled optimization", &EvalConfig{
Start: 1000,
End: 2000,
Step: 1000,
}, false)
f("disabled cache", &EvalConfig{
Start: 1000,
End: 2000,
Step: 1000,
OptimizeRepeatedBinaryOpSubexprs: true,
}, false)
f("instant query", &EvalConfig{
Start: 1000,
End: 1000,
Step: 1000,
MayCache: true,
OptimizeRepeatedBinaryOpSubexprs: true,
}, false)
f("repeated cacheable aggregate subexpression", &EvalConfig{
Start: 1000,
End: 2000,
Step: 1000,
MayCache: true,
OptimizeRepeatedBinaryOpSubexprs: true,
}, true)
f("unaligned range query", &EvalConfig{
Start: 1001,
End: 2000,
Step: 1000,
MayCache: true,
OptimizeRepeatedBinaryOpSubexprs: true,
}, false)
}
func TestShouldOptimizeRepeatedBinaryOpSubexprsExpressions(t *testing.T) {
f := func(name, q string, resultExpected bool) {
t.Helper()
e, err := metricsql.Parse(q)
if err != nil {
t.Fatalf("unexpected error in metricsql.Parse(%q) for %q: %s", q, name, err)
}
be, ok := e.(*metricsql.BinaryOpExpr)
if !ok {
t.Fatalf("unexpected expr type for %q; got %T; want *metricsql.BinaryOpExpr", name, e)
}
ec := &EvalConfig{Start: 1000, End: 2000, Step: 1000, MayCache: true, OptimizeRepeatedBinaryOpSubexprs: true}
result := shouldOptimizeRepeatedBinaryOpSubexprs(ec, be.Left, be.Right)
if result != resultExpected {
t.Fatalf("unexpected result for %q; got %v; want %v; query: %q", name, result, resultExpected, q)
}
}
f("original issue query", `count(count(vm_requests_total) by (action,addr,cluster,endpoint)) by (action,addr,cluster) / count(count(vm_requests_total) by (action,addr,cluster,endpoint))`, true)
f("right side contains repeated count aggregate", `count(foo) by (job) / (count(foo) by (job) + 1)`, true)
f("same sum aggregate", `sum(rate(foo[5m])) by (job) / sum(rate(foo[5m])) by (job)`, true)
f("same inner rollup but different aggregates", `sum(rate(foo[5m])) by (job) / count(rate(foo[5m])) by (job)`, false)
f("different count aggregates", `count(foo) by (job) / count(bar) by (job)`, false)
f("bare metric selector", `foo / foo`, false)
f("bare rollup function", `rate(a[5m]) / rate(a[5m])`, false)
f("now at modifier", `sum(rate(foo[5m] @ now())) by (job) / sum(rate(foo[5m] @ now())) by (job)`, false)
f("unseeded rand at modifier", `sum(rate(foo[5m] @ rand())) by (job) / sum(rate(foo[5m] @ rand())) by (job)`, false)
f("unseeded rand_normal at modifier", `sum(rate(foo[5m] @ rand_normal())) by (job) / sum(rate(foo[5m] @ rand_normal())) by (job)`, false)
f("unseeded rand_exponential at modifier", `sum(rate(foo[5m] @ rand_exponential())) by (job) / sum(rate(foo[5m] @ rand_exponential())) by (job)`, false)
f("seeded rand at modifier", `sum(rate(foo[5m] @ rand(1))) by (job) / sum(rate(foo[5m] @ rand(1))) by (job)`, true)
}

View File

@@ -4006,256 +4006,6 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r1, r2}
f(q, resultExpected)
})
t.Run(`vector + vector fill()`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label((
label_set(1, "foo", "common")
or label_set(2, "foo", "left_only")
) + fill(0) (
label_set(3, "foo", "common")
or label_set(4, "foo", "right_only")
), "foo")`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{4, 4, 4, 4, 4, 4},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("common"),
}}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{2, 2, 2, 2, 2, 2},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("left_only"),
}}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{4, 4, 4, 4, 4, 4},
Timestamps: timestampsExpected,
}
r3.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("right_only"),
}}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
t.Run(`vector + vector fill_left() fill_right()`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label((
label_set(1, "foo", "common")
or label_set(2, "foo", "left_only")
) + fill_left(10) fill_right(20) (
label_set(3, "foo", "common")
or label_set(4, "foo", "right_only")
), "foo")`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{4, 4, 4, 4, 4, 4},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("common"),
}}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{22, 22, 22, 22, 22, 22},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("left_only"),
}}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{14, 14, 14, 14, 14, 14},
Timestamps: timestampsExpected,
}
r3.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("right_only"),
}}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
t.Run(`vector + vector fill_right() only`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label((
label_set(1, "foo", "common")
or label_set(2, "foo", "left_only")
) + fill_right(20) (
label_set(3, "foo", "common")
or label_set(4, "foo", "right_only")
), "foo")`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{4, 4, 4, 4, 4, 4},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("common"),
}}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{22, 22, 22, 22, 22, 22},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("left_only"),
}}
resultExpected := []netstorage.Result{r1, r2}
f(q, resultExpected)
})
t.Run(`vector + vector on() fill()`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label((
label_set(1, "foo", "common", "extra", "l")
or label_set(2, "foo", "left_only", "extra", "l")
) + on(foo) fill(0) (
label_set(3, "foo", "common", "extra", "r")
or label_set(4, "foo", "right_only", "extra", "r")
), "foo")`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{4, 4, 4, 4, 4, 4},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("common"),
}}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{2, 2, 2, 2, 2, 2},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("left_only"),
}}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{4, 4, 4, 4, 4, 4},
Timestamps: timestampsExpected,
}
r3.MetricName.Tags = []storage.Tag{{
Key: []byte("foo"),
Value: []byte("right_only"),
}}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
t.Run(`vector + vector on() group_left() fill_right()`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label((
label_set(1, "method", "get", "code", "500")
or label_set(2, "method", "get", "code", "404")
or label_set(3, "method", "put", "code", "501")
) + on(method) group_left() fill_right(0) (
label_set(10, "method", "get")
), "method", "code")`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{12, 12, 12, 12, 12, 12},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{
{
Key: []byte("code"),
Value: []byte("404"),
},
{
Key: []byte("method"),
Value: []byte("get"),
},
}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{11, 11, 11, 11, 11, 11},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{
{
Key: []byte("code"),
Value: []byte("500"),
},
{
Key: []byte("method"),
Value: []byte("get"),
},
}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{3, 3, 3, 3, 3, 3},
Timestamps: timestampsExpected,
}
r3.MetricName.Tags = []storage.Tag{
{
Key: []byte("code"),
Value: []byte("501"),
},
{
Key: []byte("method"),
Value: []byte("put"),
},
}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
t.Run(`vector / vector ignoring() fill()`, func(t *testing.T) {
t.Parallel()
q := `sort_by_label((
label_set(6, "method", "get", "code", "500")
or label_set(1, "method", "put", "code", "500")
) / ignoring(code) fill(0) (
label_set(12, "method", "get")
or label_set(5, "method", "post")
or label_set(10, "method", "put")
), "method")`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0.5, 0.5, 0.5, 0.5, 0.5, 0.5},
Timestamps: timestampsExpected,
}
r1.MetricName.Tags = []storage.Tag{
{
Key: []byte("method"),
Value: []byte("get"),
},
}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0, 0, 0, 0, 0, 0},
Timestamps: timestampsExpected,
}
r2.MetricName.Tags = []storage.Tag{
{
Key: []byte("method"),
Value: []byte("post"),
},
}
r3 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{0.1, 0.1, 0.1, 0.1, 0.1, 0.1},
Timestamps: timestampsExpected,
}
r3.MetricName.Tags = []storage.Tag{
{
Key: []byte("method"),
Value: []byte("put"),
},
}
resultExpected := []netstorage.Result{r1, r2, r3}
f(q, resultExpected)
})
t.Run(`histogram_quantile(scalar)`, func(t *testing.T) {
t.Parallel()
q := `histogram_quantile(0.6, time())`

View File

@@ -1055,7 +1055,7 @@ func newRollupHoltWinters(args []any) (rollupFunc, error) {
return nan
}
// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29 .
// See https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing .
// TODO: determine whether this shit really works.
s0 := rfa.prevValue
if math.IsNaN(s0) {

View File

@@ -2,7 +2,6 @@ package promql
import (
"fmt"
"math"
"sort"
"strconv"
"sync"
@@ -289,9 +288,6 @@ func marshalMetricTagsSorted(dst []byte, mn *storage.MetricName) []byte {
}
func marshalBytesFast(dst []byte, s []byte) []byte {
if len(s) > math.MaxUint16 {
logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
}
dst = encoding.MarshalUint16(dst, uint16(len(s)))
dst = append(dst, s...)
return dst

View File

@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
#### holt_winters
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
Both `sf` and `tf` must be in the range `[0...1]`.
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
#### acosh
`acosh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
#### asinh
`asinh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
#### atanh
`atanh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.

View File

@@ -30,9 +30,6 @@ var (
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention. See also -retentionFilter")
futureRetention = flagutil.NewRetentionDuration("futureRetention", "2d", "Data with timestamps bigger than now+futureRetention is automatically deleted. "+
"The minimum futureRetention is 2 days. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention")
vmselectAddr = flag.String("vmselectAddr", "", "TCP address to accept connections from vmselect services")
vmselectDisableRPCCompression = flag.Bool("rpc.disableCompression", false, "Whether to disable compression of the data sent from vmstorage to vmselect. "+
"This reduces CPU usage at the cost of higher network bandwidth usage")
snapshotAuthKey = flagutil.NewPassword("snapshotAuthKey", "authKey, which must be passed in query string to /snapshot* pages. It overrides -httpAuth.*")
forceMergeAuthKey = flagutil.NewPassword("forceMergeAuthKey", "authKey, which must be passed in query string to /internal/force_merge pages. It overrides -httpAuth.*")
forceFlushAuthKey = flagutil.NewPassword("forceFlushAuthKey", "authKey, which must be passed in query string to /internal/force_flush pages. It overrides -httpAuth.*")
@@ -111,7 +108,7 @@ func DataPath() string {
}
// Init initializes vmstorage.
func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Duration, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
func Init(vmselectMaxConcurrentRequests int, resetCacheIfNeeded func(mrs []storage.MetricRow)) {
storage.SetDedupInterval(*minScrapeInterval)
storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
storage.LegacySetRetentionTimezoneOffset(*retentionTimezoneOffset)
@@ -172,21 +169,6 @@ func Init(vmselectMaxConcurrentRequests int, vmselectMaxQueueDuration time.Durat
storageMetrics.RegisterMetricsWriter(vmStorage.writeStorageMetrics)
metrics.RegisterSet(storageMetrics)
if *vmselectAddr != "" {
var err error
limits := vmselectapi.Limits{
MaxConcurrentRequests: vmselectMaxConcurrentRequests,
MaxConcurrentRequestsFlagName: "search.maxConcurrentRequests",
MaxQueueDuration: vmselectMaxQueueDuration,
MaxQueueDurationFlagName: "search.maxQueueDuration",
}
api := newVMStorageWithTenantID(vmStorage)
vmselectSrv, err = vmselectapi.NewServer(*vmselectAddr, api, limits, *vmselectDisableRPCCompression)
if err != nil {
logger.Fatalf("cannot create a server with -vmselectAddr=%s: %s", *vmselectAddr, err)
}
}
VMInsertAPI = vmStorage
VMSelectAPI = vmStorage
GetSearch = vmStorage.GetSearch
@@ -209,8 +191,6 @@ var (
// TODO(@rtm0): Remove this dependency from vmalert-tool unit tests.
DebugFlush func()
vmselectSrv *vmselectapi.Server
)
// Stop stops the vmstorage
@@ -221,10 +201,6 @@ func Stop() {
logger.Infof("gracefully closing the storage at %s", *storageDataPath)
startTime := time.Now()
if vmselectSrv != nil {
vmselectSrv.MustStop()
}
vmStorage.Stop()
logger.Infof("successfully closed the storage in %.3f seconds", time.Since(startTime).Seconds())

View File

@@ -164,10 +164,6 @@ func (vms *VMStorage) IsReadOnly() bool {
}
func (vms *VMStorage) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
return vms.initSearch(qt, sq, marshalDefault, deadline)
}
func (vms *VMStorage) initSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, marshal marshalFunc, deadline uint64) (vmselectapi.BlockIterator, error) {
vms.wg.Add(1)
tr := sq.GetTimeRange()
@@ -182,7 +178,6 @@ func (vms *VMStorage) initSearch(qt *querytracer.Tracer, sq *storage.SearchQuery
return nil, fmt.Errorf("missing tag filters")
}
bi := getBlockIterator()
bi.marshal = marshal
bi.wgDone = vms.wg.Done
bi.sr.Init(qt, vms.s, tfss, tr, maxMetrics, deadline)
if err := bi.sr.Error(); err != nil {
@@ -203,19 +198,11 @@ func (vms *VMStorage) getMaxMetrics(searchQueryLimit int) int {
return searchQueryLimit
}
type marshalFunc func(dst []byte, src *storage.MetricBlock) []byte
// marshalDefault is the default implementation of the MetricBlock marshaling.
func marshalDefault(dst []byte, src *storage.MetricBlock) []byte {
return src.Marshal(dst)
}
// blockIterator implements vmselectapi.BlockIterator
type blockIterator struct {
sr storage.Search
mb storage.MetricBlock
marshal marshalFunc
wgDone func()
sr storage.Search
mb storage.MetricBlock
wgDone func()
}
var blockIteratorsPool sync.Pool
@@ -244,7 +231,7 @@ func (bi *blockIterator) NextBlock(dst []byte) ([]byte, bool) {
mb := bi.mb
mb.MetricName = bi.sr.MetricBlockRef.MetricName
bi.sr.MetricBlockRef.BlockRef.MustReadBlock(&mb.Block)
dst = bi.marshal(dst[:0], &mb)
dst = mb.Marshal(dst[:0])
return dst, true
}

View File

@@ -1,282 +0,0 @@
package vmstorage
import (
"flag"
"fmt"
"math"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricnamestats"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage/metricsmetadata"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/vmselectapi"
)
var (
accountID = flag.Uint64("accountID", 0, "The accountID of the stored data")
projectID = flag.Uint64("projectID", 0, "The projectID of the stored data")
)
func newVMStorageWithTenantID(vms *VMStorage) *VMStorageWithTenantID {
if *accountID > math.MaxUint32 {
logger.Fatalf("-accountID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *accountID)
}
if *projectID > math.MaxUint32 {
logger.Fatalf("-projectID must be in the range [0, %d], got %d", uint32(math.MaxUint32), *projectID)
}
return &VMStorageWithTenantID{
vms: vms,
accountID: uint32(*accountID),
projectID: uint32(*projectID),
}
}
// VMStorageWithTenantID is a thin wrapper around VMStorage type that overrides
// its methods to properly serve requests coming from a vmselect (require
// tenantID).
//
// A new instance of this type should be created using
// newVMStorageWithTenantID(). The created instance does not require closing.
// The instance also does not take ownership of vms and it is the responsibility
// of the caller to close vms.
type VMStorageWithTenantID struct {
vms *VMStorage
accountID uint32
projectID uint32
}
// InitSearch initializes a storage search for a request initiated by a
// vmselect.
//
// The search is initialized only if the search query is either multitenant or
// its accountID and projectID match -accountID and -projectID flag values.
// Otherwise, the method returns an interator that will return no data.
//
// The method also overrides the data format of the data returned by the
// iterator by prepending accountID and projectID bytes to the metric name and
// the data block (a format used in vmcluster).
func (vmst *VMStorageWithTenantID) InitSearch(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (vmselectapi.BlockIterator, error) {
if !vmst.hasValidTenantID(sq) {
return emptyBI, nil
}
return vmst.vms.initSearch(qt, sq, vmst.marshalMetricBlock, deadline)
}
var emptyBI = &emptyBlockIterator{}
// emptyBlockIterator is an implementation of vmselectapi.BlockIterator that
// always returns no data.
type emptyBlockIterator struct{}
func (*emptyBlockIterator) MustClose() {}
func (*emptyBlockIterator) NextBlock(dst []byte) ([]byte, bool) {
return dst, false
}
func (*emptyBlockIterator) Error() error {
return nil
}
// marshalMetricBlock serializes a metric block in the format expected by
// vmselect.
//
// vmselect expects metric names and data blocks to have the tenantID but
// vmsingle does not have it. Therefore the tenantID needs to be included to
// every metric name and block.
func (vmst *VMStorageWithTenantID) marshalMetricBlock(dst []byte, src *storage.MetricBlock) []byte {
// Marshal metric name:
// 1. Marshal metric name length + accountID length + projectID length (in
// bytes).
// 2. append accountID and projectID bytes
// 3. Finally append metric name bytes
dst = encoding.MarshalVarUint64(dst, uint64(len(src.MetricName))+8)
dst = encoding.MarshalUint32(dst, vmst.accountID)
dst = encoding.MarshalUint32(dst, vmst.projectID)
dst = append(dst, src.MetricName...)
// Marshal data block.
dst = encoding.MarshalUint32(dst, vmst.accountID)
dst = encoding.MarshalUint32(dst, vmst.projectID)
dst = storage.MarshalBlock(dst, &src.Block)
return dst
}
// SearchMetricNames searches the storage for metric names that match the query.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, the method will return an
// empty result.
//
// Found metric names are prepended with accountID and projectID bytes (a format
// used in vmcluster).
func (vmst *VMStorageWithTenantID) SearchMetricNames(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) ([]string, error) {
if !vmst.hasValidTenantID(sq) {
return nil, nil
}
metricNames, err := vmst.vms.SearchMetricNames(qt, sq, deadline)
if err != nil {
return nil, err
}
// vmselect expects metric names to have the tenantID but vmsingle does not
// have it. Therefore the tenantID needs to be prepended to every metric
// name.
dst := make([]byte, 0, 8)
dst = encoding.MarshalUint32(dst, vmst.accountID)
dst = encoding.MarshalUint32(dst, vmst.projectID)
tenantID := string(dst)
for i, metricName := range metricNames {
metricNames[i] = tenantID + metricName
}
return metricNames, nil
}
// LabelValues searches the storage for label values that match the query and
// correspond to a label whose name is `labelName`. The returned result
// will contain not more than `maxLabelValues`.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, the method will return an
// empty result.
func (vmst *VMStorageWithTenantID) LabelValues(qt *querytracer.Tracer, sq *storage.SearchQuery, labelName string, maxLabelValues int, deadline uint64) ([]string, error) {
if !vmst.hasValidTenantID(sq) {
return nil, nil
}
return vmst.vms.LabelValues(qt, sq, labelName, maxLabelValues, deadline)
}
// TagValueSuffixes searches the storage for Graphite tag value suffixes. The
// returned result will contain not more than `maxSuffixes`.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, the method will return an
// empty result.
func (vmst *VMStorageWithTenantID) TagValueSuffixes(qt *querytracer.Tracer, accountID, projectID uint32, tr storage.TimeRange, tagKey, tagValuePrefix string, delimiter byte, maxSuffixes int, deadline uint64) ([]string, error) {
if !vmst.isValidTenantID(accountID, projectID) {
return nil, nil
}
return vmst.vms.TagValueSuffixes(qt, accountID, projectID, tr, tagKey, tagValuePrefix, delimiter, maxSuffixes, deadline)
}
// LabelNames searches the storage for label names that match the query.
// The returned result will contain not more than `maxLabelNames`.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, the method will return an
// empty result.
func (vmst *VMStorageWithTenantID) LabelNames(qt *querytracer.Tracer, sq *storage.SearchQuery, maxLabelNames int, deadline uint64) ([]string, error) {
if !vmst.hasValidTenantID(sq) {
return nil, nil
}
return vmst.vms.LabelNames(qt, sq, maxLabelNames, deadline)
}
// SeriesCount returns the total number of metrics stored in the database.
//
// The method may return inflated numbers. How inflated the count depends
// on the churn rate and the retention period. For example, if a metric lasts
// for 2 months, it will be counted twice.
//
// The method also counts the deleted metrics.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, the method will return 0.
func (vmst *VMStorageWithTenantID) SeriesCount(qt *querytracer.Tracer, accountID, projectID uint32, deadline uint64) (uint64, error) {
if !vmst.isValidTenantID(accountID, projectID) {
return 0, nil
}
return vmst.vms.SeriesCount(qt, accountID, projectID, deadline)
}
// Tenants returns just one tenant consisting of the -accountID and -projectID
// flag values.
func (vmst *VMStorageWithTenantID) Tenants(qt *querytracer.Tracer, tr storage.TimeRange, deadline uint64) ([]string, error) {
tenantID := fmt.Sprintf("%d:%d", vmst.accountID, vmst.projectID)
return []string{tenantID}, nil
}
// TSDBStatus retrieves the status for metrics that match to the search query.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, the method will return empty
// status.
func (vmst *VMStorageWithTenantID) TSDBStatus(qt *querytracer.Tracer, sq *storage.SearchQuery, focusLabel string, topN int, deadline uint64) (*storage.TSDBStatus, error) {
if !vmst.hasValidTenantID(sq) {
return &storage.TSDBStatus{}, nil
}
return vmst.vms.TSDBStatus(qt, sq, focusLabel, topN, deadline)
}
// DeleteSeries marks as deleted metrics that match the search query.
// The method returns the number of deleted metrics.
//
// If the query is not multitenant or the query accountID and projectID do not
// match the -accoutID and -projectID flag values, no metrics will be deleted
// and the method will return 0.
func (vmst *VMStorageWithTenantID) DeleteSeries(qt *querytracer.Tracer, sq *storage.SearchQuery, deadline uint64) (int, error) {
if !vmst.hasValidTenantID(sq) {
return 0, nil
}
return vmst.vms.DeleteSeries(qt, sq, deadline)
}
// RegisterMetricNames registers metric names in the index, the sample values
// and timestamps are ignored.
func (vmst *VMStorageWithTenantID) RegisterMetricNames(qt *querytracer.Tracer, mrs []storage.MetricRow, deadline uint64) error {
return vmst.vms.RegisterMetricNames(qt, mrs, deadline)
}
// GetMetricNamesUsageStats retrieves the usage stats for metrics whose name
// matches the pattern.
//
// If the request is not multitenant or the request accountID and projectID do
// not match the -accoutID and -projectID flag values, no metrics will be
// deleted and the method will return 0.
func (vmst *VMStorageWithTenantID) GetMetricNamesUsageStats(qt *querytracer.Tracer, tt *storage.TenantToken, limit, le int, matchPattern string, deadline uint64) (metricnamestats.StatsResult, error) {
if !vmst.isValidTenantToken(tt) {
return metricnamestats.StatsResult{}, nil
}
return vmst.vms.GetMetricNamesUsageStats(qt, tt, limit, le, matchPattern, deadline)
}
// ResetMetricNamesUsageStats resets the metric name usage stats.
func (vmst *VMStorageWithTenantID) ResetMetricNamesUsageStats(qt *querytracer.Tracer, deadline uint64) error {
return vmst.vms.ResetMetricNamesUsageStats(qt, deadline)
}
// GetMetadataRecords retrieves the metadata for the metricName.
//
// If the request is not multitenant or the request accountID and projectID do
// not match the -accoutID and -projectID flag values, no metrics will be
// deleted and the method will return 0.
func (vmst *VMStorageWithTenantID) GetMetadataRecords(qt *querytracer.Tracer, tt *storage.TenantToken, limit int, metricName string, deadline uint64) ([]*metricsmetadata.Row, error) {
if !vmst.isValidTenantToken(tt) {
return nil, nil
}
return vmst.vms.GetMetadataRecords(qt, tt, limit, metricName, deadline)
}
// hasValidTenantID returns true if the search query is either multitenant or
// its accountID and projectID match -accountID and -projectID flag values.
func (vmst *VMStorageWithTenantID) hasValidTenantID(sq *storage.SearchQuery) bool {
return sq.IsMultiTenant || vmst.isValidTenantID(sq.AccountID, sq.ProjectID)
}
// isValidTenantToken returns true if the TenantToken is either multitenant or
// its accountID and projectID match -accountID and -projectID flag values.
func (vmst *VMStorageWithTenantID) isValidTenantToken(tt *storage.TenantToken) bool {
return tt == nil || vmst.isValidTenantID(tt.AccountID, tt.ProjectID)
}
// isValidTenantID returns true if the accountID and projectID match -accountID
// and -projectID flag values.
func (vmst *VMStorageWithTenantID) isValidTenantID(accountID, projectID uint32) bool {
return accountID == vmst.accountID && projectID == vmst.projectID
}

View File

@@ -6,7 +6,7 @@ COPY web/ /build/
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o web-amd64 github.com/VictoriMetrics/vmui/ && \
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -o web-windows github.com/VictoriMetrics/vmui/
FROM alpine:3.24.1
FROM alpine:3.23.4
USER root
COPY --from=build-web-stage /build/web-amd64 /app/web

View File

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
Labels from the `on()` list aren't copied.
* [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
* [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
#### holt_winters
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
Both `sf` and `tf` must be in the range `[0...1]`.
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
#### acosh
`acosh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
#### asinh
`asinh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
#### atanh
`atanh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.

View File

@@ -183,7 +183,7 @@ const StepConfigurator: FC = () => {
<div className="vm-step-control-popper-info">
<p>
<code>step</code> - the <Hyperlink
href="https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations"
href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations"
text="interval"
/> between datapoints, which must be returned from the range query.
The <code>query</code> is executed

View File

@@ -16,7 +16,7 @@ const supportedValuesOf = Intl.supportedValuesOf;
export const supportedTimezones = supportedValuesOf ? supportedValuesOf("timeZone") as string[] : timezones;
// The list of supported units could be the following -
// https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations
// https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations
export const supportedDurations = [
{ long: "years", short: "y", possible: "year" },
{ long: "weeks", short: "w", possible: "week" },

View File

@@ -1,358 +0,0 @@
package apptest
import (
"fmt"
"slices"
)
type TestData struct {
Samples []string
Step int64
WantSeries []map[string]string
WantLabels []string
WantLabelValues []string
WantQueryResults []*QueryResult
WantMetadata map[string][]MetadataEntry
WantMetricNamesStats []MetricNamesStatsRecord
}
func GenerateTestData(prefix string, numMetrics, start, end int64) TestData {
d := TestData{
Samples: []string{},
Step: (end - start) / numMetrics,
WantSeries: make([]map[string]string, numMetrics),
WantLabels: make([]string, numMetrics),
WantLabelValues: make([]string, numMetrics),
WantQueryResults: make([]*QueryResult, numMetrics),
WantMetadata: make(map[string][]MetadataEntry),
WantMetricNamesStats: make([]MetricNamesStatsRecord, numMetrics),
}
for i := range numMetrics {
metricName := fmt.Sprintf("%s_%04d", prefix, i)
metricHelp := fmt.Sprintf("# HELP %s some help message", metricName)
metricType := fmt.Sprintf("# TYPE %s gauge", metricName)
labelName := fmt.Sprintf("label_%04d", i)
labelValue := fmt.Sprintf("value_%04d", i)
value := i
timestamp := start + i*d.Step
sample := fmt.Sprintf(`%s{%s="value", label="%s"} %d %d`, metricName, labelName, labelValue, value, timestamp)
d.Samples = append(d.Samples, metricHelp, metricType, sample)
d.WantSeries[i] = map[string]string{
"__name__": metricName,
labelName: "value",
"label": labelValue,
}
d.WantLabels[i] = labelName
d.WantLabelValues[i] = labelValue
d.WantQueryResults[i] = &QueryResult{
Metric: map[string]string{
"__name__": metricName,
labelName: "value",
"label": labelValue,
},
Samples: []*Sample{{Timestamp: timestamp, Value: float64(value)}},
}
d.WantMetadata[metricName] = []MetadataEntry{{Help: "some help message", Type: "gauge"}}
d.WantMetricNamesStats[i].MetricName = metricName
}
d.WantLabels = append(d.WantLabels, "__name__", "label")
slices.Sort(d.WantLabels)
return d
}
// AssertSeries retrieves metric names from the storage and compares the result
// with the expected one.
func AssertSeries(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []map[string]string) {
tc.T().Helper()
query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/series response",
Got: func() any {
tc.T().Helper()
return app.PrometheusAPIV1Series(tc.T(), query, QueryOpts{
Tenant: tenantID,
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
}).Sort()
},
Want: &PrometheusAPIV1SeriesResponse{
Status: "success",
Data: want,
},
Retries: 1000,
FailNow: true,
})
}
// AssertSeriesCount retrieves series count and compares it with expected one.
func AssertSeriesCount(tc *TestCase, app PrometheusQuerier, tenantID string, start, end int64, want uint64) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/series/count response",
Got: func() any {
tc.T().Helper()
return app.PrometheusAPIV1SeriesCount(tc.T(), QueryOpts{
Tenant: tenantID,
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
})
},
Want: &PrometheusAPIV1SeriesCountResponse{
Status: "success",
Data: []uint64{want},
},
FailNow: true,
})
}
// AssertLabels retrieves label names from the storage and compares the result
// with the expected one.
func AssertLabels(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end int64, want []string) {
tc.T().Helper()
query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/labels response",
Got: func() any {
tc.T().Helper()
res := app.PrometheusAPIV1Labels(tc.T(), query, QueryOpts{
Tenant: tenantID,
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
})
slices.Sort(res.Data)
return res
},
Want: &PrometheusAPIV1LabelsResponse{
Status: "success",
Data: want,
},
FailNow: true,
})
}
// AssertLabelValues retrieves values for the label whose name is labelName for
// the series whose name mathes metricNameRE, compares the result with the
// expected one.
func AssertLabelValues(tc *TestCase, app PrometheusQuerier, metricNameRE, labelName, tenantID string, start, end int64, want []string) {
tc.T().Helper()
query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/labels/.../values response",
Got: func() any {
tc.T().Helper()
res := app.PrometheusAPIV1LabelValues(tc.T(), labelName, query, QueryOpts{
Tenant: tenantID,
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
})
slices.Sort(res.Data)
return res
},
Want: &PrometheusAPIV1LabelValuesResponse{
Status: "success",
Data: want,
},
FailNow: true,
})
}
// AssertQueryResults sends a data query to storage and compares the query
// result with the expected one.
func AssertQueryResults(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, start, end, step int64, want []*QueryResult) {
tc.T().Helper()
query := fmt.Sprintf(`{__name__=~"%s"}`, metricNameRE)
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/query_range response",
Got: func() any {
tc.T().Helper()
return app.PrometheusAPIV1QueryRange(tc.T(), query, QueryOpts{
Tenant: tenantID,
Start: fmt.Sprintf("%d", start),
End: fmt.Sprintf("%d", end),
Step: fmt.Sprintf("%dms", step),
MaxLookback: fmt.Sprintf("%dms", step-1),
NoCache: "1",
})
},
Want: &PrometheusAPIV1QueryResponse{
Status: "success",
Data: &QueryData{
ResultType: "matrix",
Result: want,
},
},
FailNow: true,
})
}
func AssertMetadata(tc *TestCase, app PrometheusQuerier, metricName, tenantID string, want map[string][]MetadataEntry) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/metadata response",
Got: func() any {
tc.T().Helper()
return app.PrometheusAPIV1Metadata(tc.T(), metricName, 0, QueryOpts{
Tenant: tenantID,
})
},
Want: &PrometheusAPIV1Metadata{
Status: "success",
Data: want,
},
FailNow: true,
})
}
func AssertMetricNamesStats(tc *TestCase, app PrometheusQuerier, metricNameRE, tenantID string, want []MetricNamesStatsRecord) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /prometheus/api/v1/status/metric_names_stats response",
Got: func() any {
tc.T().Helper()
return app.PrometheusAPIV1StatusMetricNamesStats(tc.T(), "", "", metricNameRE, QueryOpts{
Tenant: tenantID,
})
},
Want: MetricNamesStatsResponse{
Records: want,
},
FailNow: true,
})
}
// GraphiteTestData holds the data samples in Graphite Pickle format, distance
// between samples in milliseconds and expected responses for various Graphite
// API endpoints.
type GraphiteTestData struct {
Samples []string
Step int64
WantMetricsIndex []string
WantMetricsFind []GraphiteMetric
WantMetricsExpand []string
WantRenderedTargets []GraphiteRenderedTarget
}
// GenerateGraphiteTestData generates Graphite test data.
func GenerateGraphiteTestData(prefix string, numMetrics, start, end int64) GraphiteTestData {
d := GraphiteTestData{
Samples: make([]string, numMetrics),
Step: (end - start) / numMetrics,
WantMetricsIndex: make([]string, numMetrics),
WantMetricsFind: make([]GraphiteMetric, numMetrics),
WantMetricsExpand: make([]string, numMetrics),
WantRenderedTargets: make([]GraphiteRenderedTarget, numMetrics),
}
datapoints := make([][2]float64, numMetrics)
for i := range numMetrics {
timestamp := (start + i*d.Step) / 1000
datapoints[i][1] = float64(timestamp)
}
for i := range numMetrics {
suffix := fmt.Sprintf("%04d", i)
metricName := fmt.Sprintf("%s.%s", prefix, suffix)
value := i
timestamp := (start + i*d.Step) / 1000
sample := fmt.Sprintf(`%s %d %d`, metricName, value, timestamp)
d.Samples[i] = sample
d.WantMetricsIndex[i] = metricName
d.WantMetricsFind[i].Id = metricName
d.WantMetricsFind[i].Text = suffix
d.WantMetricsFind[i].Leaf = 1
d.WantMetricsExpand[i] = metricName
d.WantRenderedTargets[i].Target = metricName
d.WantRenderedTargets[i].Datapoints = slices.Clone(datapoints)
d.WantRenderedTargets[i].Datapoints[i][0] = float64(value)
}
return d
}
// AssertGraphiteMetricsIndex retrieves all metrics by sending a request to
// /graphite/metrics/index.json and compares the result with the expected one.
func AssertGraphiteMetricsIndex(tc *TestCase, app PrometheusQuerier, tenantID string, want []string) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /graphite/metrics/index.json response",
Got: func() any {
tc.T().Helper()
return app.GraphiteMetricsIndex(tc.T(), QueryOpts{
Tenant: tenantID,
})
},
Want: want,
Retries: 30,
FailNow: true,
})
}
// AssertGraphiteMetricsFind finds metric names by sending a request to
// /graphite/metrics/find and compares the result with the expected one.
func AssertGraphiteMetricsFind(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []GraphiteMetric) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /graphite/metrics/find response",
Got: func() any {
tc.T().Helper()
return app.GraphiteMetricsFind(tc.T(), query, QueryOpts{
Tenant: tenantID,
})
},
Want: want,
FailNow: true,
})
}
// AssertGraphiteMetricsFind expands metric names by sending a request to
// /graphite/metrics/expand and compares the result with the expected one.
func AssertGraphiteMetricsExpand(tc *TestCase, app PrometheusQuerier, query, tenantID string, want []string) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /graphite/metrics/expand response",
Got: func() any {
tc.T().Helper()
return app.GraphiteMetricsExpand(tc.T(), query, QueryOpts{
Tenant: tenantID,
})
},
Want: want,
FailNow: true,
})
}
// AssertGraphiteRender retieves metric raw data by sending a request to
// /graphite/render and compares the result with the expected one.
func AssertGraphiteRender(tc *TestCase, app PrometheusQuerier, target, tenantID string, from, until, step int64, want []GraphiteRenderedTarget) {
tc.T().Helper()
tc.Assert(&AssertOptions{
Msg: "unexpected /graphite/render response",
Got: func() any {
tc.T().Helper()
return app.GraphiteRender(tc.T(), target, QueryOpts{
Tenant: tenantID,
From: fmt.Sprintf("%d", from/1000),
Until: fmt.Sprintf("%d", until/1000),
StorageStep: fmt.Sprintf("%dms", step),
})
},
Want: want,
FailNow: true,
})
}

View File

@@ -2,7 +2,6 @@ package tests
import (
"fmt"
"math"
"testing"
"github.com/google/go-cmp/cmp"
@@ -26,11 +25,7 @@ func TestSingleMetricsMetadata(t *testing.T) {
if len(resp.Data) != 0 {
t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
}
generateValueExceedLimit := func(prefix string) string {
buf := make([]byte, math.MaxUint16+len(prefix))
copy(buf, prefix)
return string(buf)
}
const ingestTimestamp = 1707123456700
prometheusTextDataSet := []string{
`# HELP metric_name_1 some help message`,
@@ -45,12 +40,6 @@ func TestSingleMetricsMetadata(t *testing.T) {
`# TYPE metric_name_3 gauge`,
`metric_name_3{label="baz"} 30`,
}
prometheusTextDataSet = append(prometheusTextDataSet,
`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
`# TYPE metric_name_4 gauge`,
`metric_name_4{label="baz"} 30`,
)
prometheusRemoteWriteDataSet := prompb.WriteRequest{
Timeseries: []prompb.TimeSeries{
{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
@@ -63,9 +52,6 @@ func TestSingleMetricsMetadata(t *testing.T) {
{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
},
}
@@ -151,11 +137,6 @@ func TestClusterMetricsMetadata(t *testing.T) {
if len(resp.Data) != 0 {
t.Fatalf("unexpected resp Records: %d, want: %d", len(resp.Data), 0)
}
generateValueExceedLimit := func(prefix string) string {
buf := make([]byte, math.MaxUint16+len(prefix))
copy(buf, prefix)
return string(buf)
}
const ingestTimestamp = 1707123456700
prometheusTextDataSet := []string{
@@ -171,11 +152,6 @@ func TestClusterMetricsMetadata(t *testing.T) {
`# TYPE metric_name_3 gauge`,
`metric_name_3{label="baz"} 30`,
}
prometheusTextDataSet = append(prometheusTextDataSet,
`# HELP metric_name_4 `+generateValueExceedLimit("large help"),
`# TYPE metric_name_4 gauge`,
`metric_name_4{label="baz"} 30`,
)
prometheusRemoteWriteDataSet := prompb.WriteRequest{
Timeseries: []prompb.TimeSeries{
{Labels: []prompb.Label{{Name: "__name__", Value: "metric_name_4"}}, Samples: []prompb.Sample{{Value: 40, Timestamp: ingestTimestamp}}},
@@ -188,9 +164,6 @@ func TestClusterMetricsMetadata(t *testing.T) {
{MetricFamilyName: "metric_name_5", Help: "some help message", Type: prompb.MetricTypeSummary},
{MetricFamilyName: "metric_name_6", Help: "some help message", Type: prompb.MetricTypeStateset},
{MetricFamilyName: `metric_name_7_!@"_suffix`, Help: "some help message", Type: prompb.MetricTypeStateset},
{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset},
{MetricFamilyName: "metric_name_9", Help: "some help message", Type: prompb.MetricTypeStateset, Unit: generateValueExceedLimit("large_unit")},
{MetricFamilyName: generateValueExceedLimit("metric_name_10"), Help: "some help message", Type: prompb.MetricTypeStateset},
},
}

View File

@@ -517,15 +517,10 @@ func TestClusterVMAgentForwardMetricsMetadata(t *testing.T) {
"-remoteWrite.tmpDataPath=" + tc.Dir() + "/vmagent",
fmt.Sprintf(`-remoteWrite.url=http://%s/insert/multitenant/prometheus/api/v1/write`, sut.Vminsert.HTTPAddr()),
})
generateValueExceedLimit := func(prefix string) string {
buf := make([]byte, math.MaxUint16+len(prefix))
copy(buf, prefix)
return string(buf)
}
prometheusRemoteWriteDataSet := prompb.WriteRequest{
Metadata: []prompb.MetricMetadata{
{MetricFamilyName: "metric_name_4", Help: "some help message", Type: prompb.MetricTypeSummary, AccountID: 100},
{MetricFamilyName: "metric_name_8", Help: generateValueExceedLimit("large_help"), Type: prompb.MetricTypeStateset, AccountID: 100},
},
}
vmagent.PrometheusAPIV1Write(t, prometheusRemoteWriteDataSet, apptest.QueryOpts{Tenant: "multitenant"})

View File

@@ -1,216 +0,0 @@
package tests
import (
"fmt"
"path/filepath"
"slices"
"strconv"
"testing"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/google/go-cmp/cmp"
)
func TestMixedPrometheusQueries(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
const (
accountID1 = 12
projectID1 = 34
accountID2 = 56
projectID2 = 78
numMetrics = 10
)
tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
data := apptest.GenerateTestData("metric", numMetrics, start, end)
emptySeries := []map[string]string{}
emptyLabels := []string{}
emptyLabelValues := []string{}
emptyQueryResults := []*apptest.QueryResult{}
emptyMetadata := map[string][]apptest.MetadataEntry{}
emptyMetricNamesStats := []apptest.MetricNamesStatsRecord{}
vmsingle := tc.MustStartVmsingle("vmsingle", []string{
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
"-retentionPeriod=100y",
fmt.Sprintf("-accountID=%d", accountID1),
fmt.Sprintf("-projectID=%d", projectID1),
})
vmselect := tc.MustStartVmselect("vmselect", []string{
"-storageNode=" + vmsingle.VmselectAddr(),
})
vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data.Samples, apptest.QueryOpts{})
vmsingle.ForceFlush(t)
// Ensure vmsingle returns data.
apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data.WantSeries)
apptest.AssertSeriesCount(tc, vmsingle, "", start, end, numMetrics)
apptest.AssertLabels(tc, vmsingle, "metric.*", "", start, end, data.WantLabels)
apptest.AssertLabelValues(tc, vmsingle, "metric.*", "label", "", start, end, data.WantLabelValues)
apptest.AssertQueryResults(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantQueryResults)
apptest.AssertMetadata(tc, vmsingle, "", "", data.WantMetadata)
for i := range data.WantMetricNamesStats {
data.WantMetricNamesStats[i].QueryRequestsCount = 1
}
apptest.AssertMetricNamesStats(tc, vmsingle, "", "", data.WantMetricNamesStats)
// Check that current vmsingle tenant (configured via flags) is tenant1.
gotAdminTenantsResponse := vmselect.APIV1AdminTenants(t, apptest.QueryOpts{})
wantAdminTenantsResponse := &apptest.AdminTenantsResponse{
Status: "success",
Data: []string{tenantID1},
}
if diff := cmp.Diff(wantAdminTenantsResponse, gotAdminTenantsResponse); diff != "" {
t.Fatalf("unexpected tenants (-want, +got):\n%s", diff)
}
// Ensure vmselect returns data for tenant1.
apptest.AssertSeries(tc, vmselect, "metric.*", tenantID1, start, end, data.WantSeries)
apptest.AssertSeriesCount(tc, vmselect, tenantID1, start, end, numMetrics)
apptest.AssertLabels(tc, vmselect, "metric.*", tenantID1, start, end, data.WantLabels)
apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", tenantID1, start, end, data.WantLabelValues)
apptest.AssertQueryResults(tc, vmselect, "metric.*", tenantID1, start, end, data.Step, data.WantQueryResults)
apptest.AssertMetadata(tc, vmselect, "", tenantID1, data.WantMetadata)
for i := range data.WantMetricNamesStats {
data.WantMetricNamesStats[i].QueryRequestsCount = 2
}
apptest.AssertMetricNamesStats(tc, vmselect, "", tenantID1, data.WantMetricNamesStats)
// Ensure vmselect does not return any data for tenant2.
apptest.AssertSeries(tc, vmselect, "metric.*", tenantID2, start, end, emptySeries)
apptest.AssertSeriesCount(tc, vmselect, tenantID2, start, end, 0)
apptest.AssertLabels(tc, vmselect, "metric.*", tenantID2, start, end, emptyLabels)
apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", tenantID2, start, end, emptyLabelValues)
apptest.AssertQueryResults(tc, vmselect, "metric.*", tenantID2, start, end, data.Step, emptyQueryResults)
apptest.AssertMetadata(tc, vmselect, "", tenantID2, emptyMetadata)
apptest.AssertMetricNamesStats(tc, vmselect, "", tenantID2, emptyMetricNamesStats)
// Ensure vmselect returns data for multitenant.
for _, v := range data.WantSeries {
v["vm_account_id"] = strconv.Itoa(accountID1)
v["vm_project_id"] = strconv.Itoa(projectID1)
}
apptest.AssertSeries(tc, vmselect, "metric.*", "multitenant", start, end, data.WantSeries)
data.WantLabels = append(data.WantLabels, "vm_account_id", "vm_project_id")
apptest.AssertLabels(tc, vmselect, "metric.*", "multitenant", start, end, data.WantLabels)
apptest.AssertLabelValues(tc, vmselect, "metric.*", "label", "multitenant", start, end, data.WantLabelValues)
for _, v := range data.WantQueryResults {
v.Metric["vm_account_id"] = strconv.Itoa(accountID1)
v.Metric["vm_project_id"] = strconv.Itoa(projectID1)
}
apptest.AssertQueryResults(tc, vmselect, "metric.*", "multitenant", start, end, data.Step, data.WantQueryResults)
apptest.AssertMetadata(tc, vmselect, "", "multitenant", data.WantMetadata)
for i := range data.WantMetricNamesStats {
data.WantMetricNamesStats[i].QueryRequestsCount = 3
}
apptest.AssertMetricNamesStats(tc, vmselect, "", "multitenant", data.WantMetricNamesStats)
}
func TestMixedDeleteSeries(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
const (
accountID1 = 12
projectID1 = 34
accountID2 = 56
projectID2 = 78
numMetrics = 10
)
tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
data1 := apptest.GenerateTestData("metric1", numMetrics, start, end)
data2 := apptest.GenerateTestData("metric2", numMetrics, start, end)
emptySeries := []map[string]string{}
vmsingle := tc.MustStartVmsingle("vmsingle", []string{
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
"-retentionPeriod=100y",
fmt.Sprintf("-accountID=%d", accountID1),
fmt.Sprintf("-projectID=%d", projectID1),
})
vmselect := tc.MustStartVmselect("vmselect", []string{
"-storageNode=" + vmsingle.VmselectAddr(),
})
vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data1.Samples, apptest.QueryOpts{})
vmsingle.PrometheusAPIV1ImportPrometheus(tc.T(), data2.Samples, apptest.QueryOpts{})
vmsingle.ForceFlush(t)
wantSeries12 := slices.Concat(data1.WantSeries, data2.WantSeries)
apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, wantSeries12)
vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric1.*"}`, apptest.QueryOpts{
Tenant: tenantID1,
})
apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data2.WantSeries)
vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric2.*"}`, apptest.QueryOpts{
Tenant: tenantID2,
})
apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, data2.WantSeries)
vmselect.PrometheusAPIV1AdminTSDBDeleteSeries(tc.T(), `{__name__=~"metric2.*"}`, apptest.QueryOpts{
Tenant: "multitenant",
})
apptest.AssertSeries(tc, vmsingle, "metric.*", "", start, end, emptySeries)
}
func TestMixedGraphiteQueries(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
const (
accountID1 = 12
projectID1 = 34
accountID2 = 56
projectID2 = 78
numMetrics = 10
)
tenantID1 := fmt.Sprintf("%d:%d", accountID1, projectID1)
tenantID2 := fmt.Sprintf("%d:%d", accountID2, projectID2)
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).UnixMilli()
end := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC).UnixMilli()
data := apptest.GenerateGraphiteTestData("metric", numMetrics, start, end)
emptyMetricsIndex := []string{}
emptyMetricsFind := []apptest.GraphiteMetric{}
emptyMetricsExpand := []string{}
emptyRenderedTargets := []apptest.GraphiteRenderedTarget{}
vmsingle := tc.MustStartVmsingle("vmsingle", []string{
"-storageDataPath=" + filepath.Join(tc.Dir(), "vmsingle"),
"-retentionPeriod=100y",
fmt.Sprintf("-accountID=%d", accountID1),
fmt.Sprintf("-projectID=%d", projectID1),
})
vmselect := tc.MustStartVmselect("vmselect", []string{
"-storageNode=" + vmsingle.VmselectAddr(),
})
vmsingle.GraphiteWrite(tc.T(), data.Samples, apptest.QueryOpts{})
vmsingle.ForceFlush(t)
// Ensure vmsingle returns data.
apptest.AssertGraphiteMetricsIndex(tc, vmsingle, "", data.WantMetricsIndex)
apptest.AssertGraphiteMetricsFind(tc, vmsingle, "metric.*", "", data.WantMetricsFind)
apptest.AssertGraphiteMetricsExpand(tc, vmsingle, "metric.*", "", data.WantMetricsExpand)
apptest.AssertGraphiteRender(tc, vmsingle, "metric.*", "", start, end, data.Step, data.WantRenderedTargets)
// Ensure vmselect returns data for tenant1.
apptest.AssertGraphiteMetricsIndex(tc, vmselect, tenantID1, data.WantMetricsIndex)
apptest.AssertGraphiteMetricsFind(tc, vmselect, "metric.*", tenantID1, data.WantMetricsFind)
apptest.AssertGraphiteMetricsExpand(tc, vmselect, "metric.*", tenantID1, data.WantMetricsExpand)
apptest.AssertGraphiteRender(tc, vmselect, "metric.*", tenantID1, start, end, data.Step, data.WantRenderedTargets)
// Ensure vmselect does not return any data for tenant2.
apptest.AssertGraphiteMetricsIndex(tc, vmselect, tenantID2, emptyMetricsIndex)
apptest.AssertGraphiteMetricsFind(tc, vmselect, "metric.*", tenantID2, emptyMetricsFind)
apptest.AssertGraphiteMetricsExpand(tc, vmselect, "metric.*", tenantID2, emptyMetricsExpand)
apptest.AssertGraphiteRender(tc, vmselect, "metric.*", tenantID2, start, end, data.Step, emptyRenderedTargets)
}

View File

@@ -25,14 +25,12 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
"-httpListenAddr": "127.0.0.1:0",
"-graphiteListenAddr": "127.0.0.1:0",
"-opentsdbListenAddr": "127.0.0.1:0",
"-vmselectAddr": "127.0.0.1:0",
},
extractREs: []*regexp.Regexp{
storageDataPathRE,
httpListenAddrRE,
graphiteListenAddrRE,
openTSDBListenAddrRE,
vmselectAddrRE,
},
output: output,
})
@@ -45,7 +43,6 @@ func StartVmsingle(instance string, flags []string, cli *Client, output io.Write
httpListenAddr: stderrExtracts[1],
graphiteListenAddr: stderrExtracts[2],
openTSDBListenAddr: stderrExtracts[3],
vmselectAddr: stderrExtracts[4],
}), nil
}
@@ -54,7 +51,6 @@ type vmsingleRuntimeValues struct {
httpListenAddr string
graphiteListenAddr string
openTSDBListenAddr string
vmselectAddr string
}
func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
@@ -89,7 +85,6 @@ func newVmsingle(app *app, cli *Client, rt vmsingleRuntimeValues) *Vmsingle {
},
storageDataPath: rt.storageDataPath,
httpListenAddr: rt.httpListenAddr,
vmselectAddr: rt.vmselectAddr,
}
}
@@ -104,7 +99,6 @@ type Vmsingle struct {
storageDataPath string
httpListenAddr string
vmselectAddr string
}
// HTTPAddr returns the address at which the vminsert process is
@@ -113,12 +107,6 @@ func (app *Vmsingle) HTTPAddr() string {
return app.httpListenAddr
}
// VmselectAddr returns the address at which the vmsingle process is listening
// for vmselect connections.
func (app *Vmsingle) VmselectAddr() string {
return app.vmselectAddr
}
// String returns the string representation of the vmsingle app state.
func (app *Vmsingle) String() string {
return fmt.Sprintf("{app: %s storageDataPath: %q httpListenAddr: %q}", []any{

View File

@@ -3,9 +3,9 @@
DOCKER_REGISTRIES ?= docker.io quay.io
DOCKER_NAMESPACE ?= victoriametrics
ROOT_IMAGE ?= alpine:3.24.1
ROOT_IMAGE ?= alpine:3.23.4
ROOT_IMAGE_SCRATCH ?= scratch
CERTS_IMAGE := alpine:3.24.1
CERTS_IMAGE := alpine:3.23.4
GO_BUILDER_IMAGE := golang:1.26.4

View File

@@ -283,7 +283,7 @@ Released: 2025-06-13
## v1.23.2
Released: 2025-06-09
- IMPROVEMENT: Increased convergence speed for [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score), [ZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score), [MADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#mad-median-absolute-deviation), and [OnlineMADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) models. Now it works better for tight optimization budgets (n_trials < 10, timeout < 1s)
- IMPROVEMENT: Increased convergence speed for [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score), [ZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score), [MADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#mad), and [OnlineMADModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-mad) models. Now it works better for tight optimization budgets (n_trials < 10, timeout < 1s)
- BUGFIX: Now mean and variance of [OnlineZScoreModel](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score) with exponential `decay` < 1 [arg](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay) are properly calculated for unbiased predictions.
@@ -538,7 +538,7 @@ Released: 2024-10-01
> A bug was discovered in this release that causes the service to crash. Please use the patch [v1.16.1](#v1161) to resolve this issue.
- FEATURE: Introduced data dumps to a host filesystem for [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader). Resource-intensive setups (multiple queries returning many metrics, bigger `fit_window` arg) will have RAM consumption reduced during fit calls.
- FEATURE: Introduced data dumps to a host filesystem for [VmReader](https://docs.victoriametrics.com/anomaly-detection/#vm-reader). Resource-intensive setups (multiple queries returning many metrics, bigger `fit_window` arg) will have RAM consumption reduced during fit calls.
- IMPROVEMENT: Added a `groupby` argument for logical grouping in [multivariate models](https://docs.victoriametrics.com/anomaly-detection/components/models/#multivariate-models). When specified, a separate multivariate model is trained for each unique combination of label values in the `groupby` columns. For example, to perform multivariate anomaly detection on metrics at the machine level without cross-entity interference, you can use `groupby: [host]` or `groupby: [instance]`, ensuring one model per entity being trained (e.g., per host). Please find more details [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#group-by).
- IMPROVEMENT: Improved performance of [VmReader](https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader) on multicore instances for reading and data processing.
- IMPROVEMENT: Introduced new CLI argument aliases to enhance compatibility with [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md) (i.e. using secrets) and better align with [VictoriaMetrics flags](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#list-of-command-line-flags):
@@ -685,7 +685,7 @@ Released: 2024-02-15
## v1.9.2
Released: 2024-01-29
- BUGFIX: now multivariate models (like [`IsolationForestMultivariateModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-forest-multivariate)) are properly handled throughout fit/infer phases.
- BUGFIX: now multivariate models (like [`IsolationForestMultivariateModel`](https://docs.victoriametrics.com/anomaly-detection/components/models/#isolation-foresthttpsenwikipediaorgwikiisolation_forest-multivariate)) are properly handled throughout fit/infer phases.
## v1.9.1

View File

@@ -553,7 +553,7 @@ preset: ui
# other optional server/settings parameters, e.g. port, max_concurrent_tasks, n_workers, logger_levels, etc.
```
using one of the [deployment methods](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly) in a [QuickStart guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/), e.g. via Docker.
using one of the [deployment methods](https://docs.victoriametrics.com/anomaly-detection/quickstart/#how-to-install-and-run-vmanomaly) in a [QuickStart guide](https://docs.victoriametrics.com/anomaly-detection/quickstart/#quickstart), e.g. via Docker.
Retrieve the UI at `http://<vmanomaly-host>:<port>` (e.g. at `http://localhost:8490` if running locally with default port) and start exploring anomaly detection models and their configurations interactively.

View File

@@ -32,7 +32,7 @@ Server component of VictoriaMetrics Anomaly Detection (`vmanomaly`) is responsib
### Example Configuration
> [!TIP]
> If [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/#hot-reload) is enabled in vmanomaly service, the server will automatically pick up changes made to the configuration file without requiring a restart.
> If [hot-reloading](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#hot-reloading) is enabled in vmanomaly service, the server will automatically pick up changes made to the configuration file without requiring a restart.
```yaml
server:
@@ -63,4 +63,4 @@ reader:
After starting the `vmanomaly` server with the above configuration, UI can be accessed at `<vmanomaly-host>:8490/vmanomaly/vmui/` (e.g. `http://localhost:8490/vmanomaly/vmui/`).
Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).
Rest API endpoints (e.g. `/metrics`) can be accessed at `<vmanomaly-host>:8490/vmanomaly/metrics` (e.g. `http://localhost:8490/vmanomaly/metrics`).

View File

@@ -39,7 +39,7 @@ Each subsequent section of this guide presents an architecture designed to handl
### The decision tree
<p align="center">
<img src="/guides/vm-architectures/decision-tree.webp" alt="Decision Tree" width="80%">
<img src="decision-tree.webp" alt="Decision Tree" width="80%">
</p>
## Basic
@@ -62,7 +62,7 @@ Installation guide reference: [VictoriaMetrics Single](https://docs.victoriametr
**Schema:**
<p align="center">
<img src="/guides/vm-architectures/basic-architecture.webp" alt="Basic Architecture" width="40%">
<img src="basic-architecture.webp" alt="Basic Architecture" width="40%">
</p>
### Unavailability Scenarios
@@ -93,7 +93,7 @@ High availability implementation: [HA VictoriaMetrics Cluster](https://docs.vict
**Schema:**
<p align="center">
<img src="/guides/vm-architectures/single-az-architecture.webp" alt="Single AZ Architecture" width="60%">
<img src="single-az-architecture.webp" alt="Single AZ Architecture" width="60%">
</p>
### Application vs. Storage Replication
@@ -210,7 +210,7 @@ To ensure reliability, vmagent implements the bulkhead pattern: each destination
**Schema:**
<p align="center">
<img src="/guides/vm-architectures/multi-az-architecture.webp" alt="Multi-AZ Architecture" width="65%">
<img src="multi-az-architecture.webp" alt="Multi-AZ Architecture" width="65%">
</p>
### Unavailability Scenarios
@@ -257,7 +257,7 @@ For complete disaster recovery, this entire cell-based architecture is duplicate
A global, stateless layer of routing cells (vmagent, vmauth) sits on top. It routes traffic to several logical groups of storage cells. Each storage group contains multiple AZs, and data is replicated or sharded across them. There are several approaches to implementing it.
<p align="center">
<img src="/guides/vm-architectures/hyperscale-architecture.webp" alt="Hyperscale Architecture" width="85%">
<img src="hyperscale-architecture.webp" alt="Hyperscale Architecture" width="85%">
</p>
### Choosing Your Read Path Strategy
@@ -375,7 +375,7 @@ This multitenancy approach gives us another trade-off in the isolation implement
**Schema:**
<p align="center">
<img src="/guides/vm-architectures/logical-layers-architecture.webp" alt="Logical Layers Architecture" width="80%">
<img src="logical-layers-architecture.webp" alt="Logical Layers Architecture" width="80%">
</p>
**Path A: Shared resources.** We have a single, shared pool of all cluster components.

View File

@@ -49,7 +49,7 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
* [Brewblox: InfluxDB to Victoria Metrics](https://www.brewblox.com/dev/decisions/20210718_victoria_metrics.html)
* [Techetio: Evaluating Backend Options For Prometheus Metrics](https://www.techetio.com/2022/08/21/evaluating-backend-options-for-prometheus-metrics/)
* [Asserts: Announcing Asserts](https://www.asserts.ai/blog/announcing-asserts/)
* [Optimizing Linkerd metrics in Prometheus](https://itnext.io/optimizing-linkerd-metrics-in-prometheus-de607ec10f6b)
* [Optimizing Linkerd metrics in Prometheus](https://aatarasoff.medium.com/optimizing-linkerd-metrics-in-prometheus-de607ec10f6b)
* [VictoriaMetrics vs. OpenTSDB](https://blg.robot-house.us/posts/tsdbs-grow/)
* [Monitoring of multiple OpenShift clusters with VictoriaMetrics](https://medium.com/ibm-garage/monitoring-of-multiple-openshift-clusters-with-victoriametrics-d4f0979e2544)
* [Ultra Monitoring with Victoria Metrics](https://dev.to/aws-builders/ultra-monitoring-with-victoria-metrics-1p2)

View File

@@ -482,7 +482,7 @@ Across our production VictoriaMetrics clusters, in a 12 months period we go beyo
## Roblox
[Roblox](https://www.roblox.com/) builds the tools and platform that empower people to create their own immersive experiences,
[Roblox](https://roblox.com/) builds the tools and platform that empower people to create their own immersive experiences,
so that any world they can imagine can be brought to life.
With more than 200 million active monthly users, Roblox is one of the most popular gaming platforms on the Internet.
@@ -688,7 +688,8 @@ Thanos, Cortex and VictoriaMetrics were evaluated as a long-term storage for Pro
- The API is compatible with Prometheus and nearly all standard PromQL queries work well out of the box.
- Handles storage well, with periodic compaction which makes it easy to take snapshots.
Please see [Monitoring K8S with VictoriaMetrics](https://youtu.be/ZJQYW-cFOms) video and [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/) blog post for more details.
Please see [Monitoring K8S with VictoriaMetrics](https://docs.google.com/presentation/d/1g7yUyVEaAp4tPuRy-MZbPXKqJ1z78_5VKuV841aQfsg/edit) slides,
[video](https://youtu.be/ZJQYW-cFOms) and [Infrastructure monitoring with Prometheus at Zerodha](https://zerodha.tech/blog/infra-monitoring-at-zerodha/) blog post for more details.
## zhihu

View File

@@ -639,7 +639,7 @@ Also in the cluster version the `/prometheus/api/v1` endpoint ingests `jsonl`,
- `prometheus/api/v1/import` - for importing data obtained via `api/v1/export` at `vmselect` (see below), JSON line format.
- `prometheus/api/v1/import/native` - for importing data obtained via `api/v1/export/native` on `vmselect` (see below).
- `prometheus/api/v1/import/csv` - for importing arbitrary CSV data. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data) for details.
- `prometheus/api/v1/import/prometheus` - for importing data in [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format) and in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md). This endpoint also supports [Pushgateway protocol](https://github.com/prometheus/pushgateway#url). See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
- `prometheus/api/v1/import/prometheus` - for importing data in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format) and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md). This endpoint also supports [Pushgateway protocol](https://github.com/prometheus/pushgateway#url). See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
- `opentelemetry/v1/metrics` - for ingesting data via [OpenTelemetry protocol for metrics](https://github.com/open-telemetry/opentelemetry-specification/blob/97c826b70e2f89cfdf655d5150791f3f0c2bae19/specification/metrics/data-model.md). See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/).
- `datadog/api/v1/series` - for ingesting data with DataDog submit metrics API v1. See [these docs](https://docs.victoriametrics.com/victoriametrics/url-examples/#datadogapiv1series) for details.
- `datadog/api/v2/series` - for ingesting data with [DataDog submit metrics API](https://docs.datadoghq.com/api/latest/metrics/#submit-metrics). See [these docs](https://docs.victoriametrics.com/victoriametrics/integrations/datadog/) for details.

View File

@@ -247,7 +247,7 @@ See [How to migrate from InfluxDB to VictoriaMetrics](https://docs.victoriametri
* TimescaleDB insists on using SQL as a query language. While SQL is more powerful than PromQL, this power is rarely required during typical usages of a TSDB. Real-world queries usually [look clearer and simpler when written in PromQL than in SQL](https://medium.com/@valyala/promql-tutorial-for-beginners-9ab455142085).
* VictoriaMetrics requires [up to 70x less storage space compared to TimescaleDB](https://medium.com/@valyala/when-size-matters-benchmarking-victoriametrics-vs-timescale-and-influxdb-6035811952d4) for storing the same amount of time series data. The gap in storage space usage can be decreased from 70x to 3x if [compression in TimescaleDB is properly configured](https://docs.timescale.com/use-timescale/latest/compression/) (it isn't an easy task in general :)).
* VictoriaMetrics requires up to 10x less CPU and RAM resources than TimescaleDB for processing production data. See [this article](https://abiosgaming.com/press/high-cardinality-aggregations/) for details.
* TimescaleDB is [harder to set up, configure and operate](https://www.tigerdata.com/docs/get-started/choose-your-path/install-timescaledb#tab=ubuntu) than VictoriaMetrics (see [how to run VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics)).
* TimescaleDB is [harder to set up, configure and operate](https://docs.timescale.com/timescaledb/latest/how-to-guides/install-timescaledb/self-hosted/ubuntu/installation-apt-ubuntu/) than VictoriaMetrics (see [how to run VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics)).
* VictoriaMetrics accepts data in multiple popular data ingestion protocols InfluxDB, OpenTSDB, Graphite, CSV while TimescaleDB supports only SQL inserts.
* VictoriaMetrics can be queried via [Graphite's API](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#graphite-api-usage).
@@ -269,7 +269,7 @@ We provide commercial support for both versions. [Contact us](https://victoriame
[VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_faq) the most cost-efficient hosted monitoring platform, operated by VictoriaMetrics core team.
<!-- Links inside the paragraph break navigation in the right-side menu. To fix this, an explicit anchor definition has been added. -->
## Why doesn't VictoriaMetrics support the [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read)? {#why-doesnrsquot-victoriametrics-support-the-prometheus-remote-read-api-}
## Why doesn't VictoriaMetrics support the [Prometheus remote read API](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_read%3E)? {#why-doesnrsquot-victoriametrics-support-the-prometheus-remote-read-api-}
The remote read API requires transferring all the raw data for all the requested metrics over the given time range. For instance,
if a query covers 1000 metrics with 10K values each, then the remote read API has to return `1000*10K`=10M metric values to Prometheus.

View File

@@ -91,9 +91,9 @@ The list of MetricsQL features on top of PromQL:
Labels from the `on()` list aren't copied.
* [Aggregate functions](#aggregate-functions) accept arbitrary number of args.
For example, `avg(q1, q2, q3)` would return the average values for every point across time series returned by `q1`, `q2` and `q3`.
* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) can be put anywhere in the query.
* [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) can be put anywhere in the query.
For example, `sum(foo) @ end()` calculates `sum(foo)` at the `end` timestamp of the selected time range `[start ... end]`.
* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier).
* Arbitrary subexpression can be used as [@ modifier](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier).
For example, `foo @ (end() - 1h)` calculates `foo` at the `end - 1 hour` timestamp on the selected time range `[start ... end]`.
* [offset](https://prometheus.io/docs/prometheus/latest/querying/basics/#offset-modifier), lookbehind window in square brackets
and `step` value for [subquery](#subqueries) may refer to the current step aka `$__interval` value from Grafana with `[Ni]` syntax.
@@ -482,7 +482,7 @@ See also [hoeffding_bound_lower](#hoeffding_bound_lower).
#### holt_winters
`holt_winters(series_selector[d], sf, tf)` is a [rollup function](#rollup-functions), which calculates Holt-Winters value
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing_%28Holt_linear%29)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
(aka [double exponential smoothing](https://en.wikipedia.org/wiki/Exponential_smoothing#Double_exponential_smoothing)) for [raw samples](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#raw-samples)
over the given lookbehind window `d` using the given smoothing factor `sf` and the given trend factor `tf`.
Both `sf` and `tf` must be in the range `[0...1]`.
@@ -1154,7 +1154,7 @@ See also [asin](#asin) and [cos](#cos).
#### acosh
`acosh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic cosine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_cosine) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -1176,7 +1176,7 @@ See also [acos](#acos) and [sin](#sin).
#### asinh
`asinh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic sine](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
@@ -1198,7 +1198,7 @@ See also [tan](#tan).
#### atanh
`atanh(q)` is a [transform function](#transform-functions), which returns
[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Definitions_in_terms_of_logarithms) for every point of every time series returned by `q`.
[inverse hyperbolic tangent](https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_tangent) for every point of every time series returned by `q`.
Metric names are stripped from the resulting series. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.

View File

@@ -36,6 +36,7 @@ VictoriaMetrics is available as:
* [Ansible Roles](https://github.com/VictoriaMetrics/ansible-playbooks)
* [Source code](https://github.com/VictoriaMetrics/VictoriaMetrics).
See [How to build from sources](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-build-from-sources)
* [VictoriaMetrics on Linode](https://www.linode.com/marketplace/apps/victoriametrics/victoriametrics/)
* [VictoriaMetrics on DigitalOcean](https://marketplace.digitalocean.com/apps/victoriametrics-single)
Just download VictoriaMetrics and follow [these instructions](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-start-victoriametrics).

View File

@@ -245,7 +245,7 @@ The following steps must be performed during the upgrade / downgrade procedure:
* Wait until the process stops. This can take a few seconds.
* Start the upgraded VictoriaMetrics.
Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/whats-new-in-prometheus-2-8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
> If you'd prefer not to manage upgrades yourself, [VictoriaMetrics Cloud](https://console.victoriametrics.cloud/signUp?utm_source=website&utm_campaign=docs_vm_single_upgrade)
> performs version upgrades automatically during maintenance windows with no action required on your part.
@@ -417,7 +417,7 @@ VictoriaMetrics is configured via command-line flags, so it must be restarted wh
* Wait until the process stops. This can take a few seconds.
* Start VictoriaMetrics with the new command-line flags.
Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/whats-new-in-prometheus-2-8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
Prometheus doesn't drop data during VictoriaMetrics restart. See [this article](https://grafana.com/blog/2019/03/25/whats-new-in-prometheus-2.8-wal-based-remote-write/) for details. The same applies also to [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/).
## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter)
@@ -478,11 +478,6 @@ and [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyco
to the given number of digits after the decimal point.
For example, `/api/v1/query?query=avg_over_time(temperature[1h])&round_digits=2` would round response values to up to two digits after the decimal point.
VictoriaMetrics accepts `optimize_repeated_binary_op_subexprs=1` query arg for [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query)
handler. It allows `vmselect` to execute left and right sides of binary operators sequentially when they contain the same
optimized aggregate rollup result expression, so the second side may reuse the rollup result cache populated by the first side.
The optimization is disabled by default and applies only when rollup result cache can be used for the request.
VictoriaMetrics accepts `limit` query arg for [/api/v1/labels](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labels)
and [`/api/v1/label/<labelName>/values`](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1labelvalues) handlers for limiting the number of returned entries.
For example, the query to `/api/v1/labels?limit=5` returns a sample of up to 5 unique labels, while ignoring the rest of labels.
@@ -999,7 +994,7 @@ Note that it could be required to flush response cache after importing historica
### How to import data in Prometheus exposition format
VictoriaMetrics accepts data in [Prometheus exposition format](https://github.com/prometheus/docs/blob/main/docs/instrumenting/exposition_formats.md),
in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md)
in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md)
and in [Pushgateway format](https://github.com/prometheus/pushgateway#url) via `/api/v1/import/prometheus` path.
For example, the following command imports a single line in Prometheus exposition format into VictoriaMetrics:
@@ -1838,7 +1833,7 @@ trivy image --sbom-sources oci \
The only option is increasing the limit on [the number of open files in the OS](https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a).
The recommendation is not specific to VictoriaMetrics only, but also for any service that handles many HTTP connections and stores data on disk.
* VictoriaMetrics is a write-heavy application, and its performance depends on disk performance. So be careful with other
applications or utilities (like [fstrim](https://manpages.ubuntu.com/manpages/noble/en/man8/fstrim.8.html))
applications or utilities (like [fstrim](https://manpages.ubuntu.com/manpages/lunar/en/man8/fstrim.8.html))
which could [exhaust disk resources](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1521).
* The recommended filesystem is `ext4`, the recommended persistent storage is [persistent HDD-based disk on GCP](https://cloud.google.com/compute/docs/disks/#pdspecs),
since it is protected from hardware failures via internal replication, and it can be [resized on the fly](https://cloud.google.com/compute/docs/disks/add-persistent-disk#resize_pd).

View File

@@ -26,23 +26,11 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for selecting relabel configurations from `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` in the [metrics relabel debug UI](https://docs.victoriametrics.com/victoriametrics/relabeling/#relabel-debugging). See [#9918](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9918).
* SECURITY: upgrade base docker image (Alpine) from 3.23.4 to 3.24.1. See [Alpine 3.24.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.24.1-released.html).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): add `default_vm_access_claim` field into `jwt` section of auth config. It could be used at [JWT claim placeholders](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), if `JWT` token doesn't have `vm_access` claim. See [#11054](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11054).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): reduces CPU usage by 10% at [sharding among remote storages](https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages). See [#11113](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11113). Thanks to @bennf for contribution.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce `64KiB` size limit for `metric metadata` fields - `Unit`, `Help` and `MetricFamilyName`. See [#11128](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `optimize_repeated_binary_op_subexprs=1` query arg to [/api/v1/query_range](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#range-query) for executing binary operator sides sequentially when they share the same optimized aggregate rollup result expression. This allows the second side to reuse rollup result cache populated by the first side. See [#10575](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10575). Thanks to @xhebox for the contribution.
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): prevent possible password brute-force attacks with an artificial 2-3 second delay as recommended by [OWASP](https://owasp.org/Top10/2025/A07_2025-Authentication_Failures). See [#11180](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11180).
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support `fill` modifiers to allow missing series on either side of a binary operation to be filled with a provided default value. See [#10598](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10598).
* BUGFIX: all VictoriaMetrics components: cancel in-flight HTTP requests shortly before `-http.maxGracefulShutdownDuration` elapses during graceful shutdown, so they can drain and the shutdown completes cleanly within that window instead of timing out and exiting via `logger.Fatalf` -> `os.Exit`. This prevents skipping the storage flush and losing in-memory data when long-lived requests are in flight (such as VictoriaLogs live tailing). See [#1502](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502).
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fixes unexpected rare rerouting. See [#11162](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11162).
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): propagate cache reset operation to `selectNode` when `/internal/resetRollupResultCache` is called. Previously, the propagation only happened when the `delete_series` API was called. See [#11112](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11112).
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix possible unexpected increases in `rate_avg` and `rate_sum` if an out-of-order sample is ingested after the previous flush. See [#11140](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11140).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): Add the support of vmselect RPC to vmsingle so that single node can be queried by a vmselect from a vmcluster deployment. See [4328](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4328) and [10926](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10926).
## [v1.146.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.146.0)
Released at 2026-06-22
@@ -56,7 +44,6 @@ Released at 2026-06-22
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add `-vm-headers` and `-vm-bearer-token` flags for authenticating requests to the VictoriaMetrics import destination. The flags are available in `opentsdb`, `influx`, `remote-read`, `prometheus`, `mimir`, and `thanos` vmctl sub-commands. See [#8897](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8897).
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): log calls to [/api/v1/admin/tsdb/delete_series](https://docs.victoriametrics.com/victoriametrics/url-examples/#apiv1admintsdbdelete_series) API handler. This should help to identify events of metrics deletion from the database. See [#11104](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11104).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add the `last` value to graph legend statistics. See [#10759](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10759).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add support for [Monitoring Data eXchange (MDX)](https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange): the ability to route only metrics from VictoriaMetrics services to a specific `-remoteWrite.url`. MDX is useful for building monitoring-of-monitoring where one remote storage should receive the full metric stream and another should receive only VictoriaMetrics metrics. Enable per destination with `-remoteWrite.mdx.enable=true`. See [#10600](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10600).
* BUGFIX: [enterprise](https://docs.victoriametrics.com/enterprise/) [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly expose metric `vm_retention_filters_partitions_scheduled_rows`. See [#11138](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11138)
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): fix issue with producing aggregated samples with identical timestamps between flushes. See [#10808](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10808).
@@ -71,7 +58,6 @@ Released at 2026-06-22
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `metricFamilyName` at metrics metadata response. See [#11129](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11129). Thanks for @fxrlv for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent more cases of panic during directory deletion on `NFS`-based mounts. See [#11060](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11060).
## [v1.145.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.145.0)
Released at 2026-06-08
@@ -234,7 +220,7 @@ Released at 2026-03-27
**Update Note 1:** [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): a bug in [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) parsing caused the `Unit` suffix of the previously parsed metric to be incorrectly applied to subsequent metrics that have no `Unit` field, when `-opentelemetry.usePrometheusNaming` is enabled. For example, if `http_requests` has `Unit: seconds` and the next metric `cpu_usage` has no `Unit`, `cpu_usage` would be ingested as `cpu_usage_seconds`. The bug was introduced in [v1.132.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11320). See [#10889](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10889).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): show `seriesCountByMetricName` table when a label is in focus in the [Cardinality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer). See [#10630](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10630). Thanks to @Roshan1299 for the contribution.
* FEATURE: [dashboards/metrics-explorer](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/metrics-explorer.json): add a new dashboard for exploring stored metrics based on [Caridnality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) and [ingested metrics usage API](https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage). The dashboard requires [Infinity Grafana plugin](https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/) to be installed. See [#10617](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10617) for details.
* FEATURE: [dashboards/unused-metrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/dashboards/unused-metrics.json): add a new dashboard for exploring stored metrics based on [Caridnality Explorer](https://docs.victoriametrics.com/victoriametrics/#cardinality-explorer) and [ingested metrics usage API](https://docs.victoriametrics.com/victoriametrics/#track-ingested-metrics-usage). The dashboard requires [Infinity Grafana plugin](https://grafana.com/grafana/plugins/yesoreyeram-infinity-datasource/) to be installed. See [#10617](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10617) for details.
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `search` parameter and pagination support in `/api/v1/rules` API. See [#10046](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10046).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): add default pagination to improve the Alerting Rules page experience when vmalert loads thousands of rules. See [#10046](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10046).
* FEATURE: all VictoriaMetrics components: log a warning when an IPv6 listen address (e.g. `[::]:6969`) is specified but `-enableTCP6` is not set. Previously, the server silently listened on IPv4 only. See [#6858](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6858). Thanks to @andriibeee for the contribution.
@@ -248,7 +234,7 @@ Released at 2026-03-27
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix autocomplete dropdown not closing on the Raw Query page. See [#10665](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10665)
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly handle JWKS keys per [RFC 7517](https://datatracker.ietf.org/doc/html/rfc7517#section-4.2) during [OIDC discovery](https://docs.victoriametrics.com/victoriametrics/vmauth/#oidc-discovery): skip keys with `use=enc`, reject `use=sig` keys with unsupported `alg`, and warn-skip keys with empty `use` that have unsupported `alg`. See [#10663](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10663). Thanks to @andriibeee for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enforce `datasource_type=prometheus` when [proxying](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert) Grafana requests to [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/). Grafana supports only `prometheus` and `loki` alerts. Without this fix, Grafana shows `Error loading alerts` when non-Prometheus alert types are returned. See [victoriametrics-datasource#329](https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic during directory deletion on `NFS`-based mounts. The bug was introduced in [83da33d8](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/83da33d8cfe8352fd0022d05a8b6346ebb48420d) and included in [v1.123.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11230). See [#9842](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842).
## [v1.138.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.138.0)
@@ -482,7 +468,7 @@ The v1.136.x line will be supported for at least 12 months since [v1.136.0](http
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): retry RPC by dialing a new connection instead of reusing a pooled one when the previous attempt fails with `io.EOF`, `broken pipe` or `reset by peer`. This reduces query failures caused by stale connections to restarted vmstorage nodes. See [#10314](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10314)
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix autocomplete dropdown not closing on the Raw Query page. See [#10665](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10665)
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enforce `datasource_type=prometheus` when [proxying](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmalert) Grafana requests to [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/). Grafana supports only `prometheus` and `loki` alerts. Without this fix, Grafana shows `Error loading alerts` when non-Prometheus alert types are returned. See [victoriametrics-datasource#329](https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/329).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): stop logging `error`-level messages when scraping targets that expose OpenMetrics `info`, `gaugehistogram`, `stateset`, or `unknown` metric types. These are valid [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) types and should be parsed without error. See [#10685](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10685). Thanks to @tsarna for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent panic during directory deletion on `NFS`-based mounts. The bug was introduced in [83da33d8](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/83da33d8cfe8352fd0022d05a8b6346ebb48420d) and included in [v1.123.0](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/victoriametrics/changelog/CHANGELOG_2025.md#v11230). See [#9842](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9842).
## [v1.136.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.136.2)
@@ -850,7 +836,7 @@ Released at 2026-02-13
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.122.x line will be supported for at least 12 months since [v1.122.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11220) release**
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.9-3.21.6-3.22.3-3.23.3-released.html).
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.3-released.html).
* SECURITY: upgrade Go builder from Go1.24.12 to Go1.24.13. See [the list of issues addressed in Go1.24.13](https://github.com/golang/go/issues?q=milestone%3AGo1.24.13+label%3ACherryPickApproved).
* BUGFIX: all VictoriaMetrics components: respect default http client proxy env variables (`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`). See [#10385](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10385). Thanks to @zane-deg for the contribution.
@@ -1012,7 +998,7 @@ Released at 2026-02-13
All these fixes are also included in [the latest community release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/latest).
The v1.110.x line will be supported for at least 12 months since [v1.110.0](https://docs.victoriametrics.com/victoriametrics/changelog/#v11100) release**
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.20.9-3.21.6-3.22.3-3.23.3-released.html).
* SECURITY: upgrade base docker image (Alpine) from 3.23.2 to 3.23.3. See [Alpine 3.23.3 release notes](https://www.alpinelinux.org/posts/Alpine-3.23.3-released.html).
* SECURITY: upgrade Go builder from Go1.24.12 to Go1.24.13. See [the list of issues addressed in Go1.24.13](https://github.com/golang/go/issues?q=milestone%3AGo1.24.13+label%3ACherryPickApproved).
* BUGFIX: VictoriaMetrics [enterprise](https://docs.victoriametrics.com/enterprise/) [vmagent](https://docs.victoriametrics.com/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): introduce timebased manual offset commit for [kafka consumer](https://docs.victoriametrics.com/victoriametrics/integrations/kafka/) to fix performance degradation with enabled manual commit. After this change, the consumer will commit partition offsets in batch per second to avoid high commit QPS on the Kafka broker. It's no longer recommended to set `enable.auto.commit=true` in `-kafka.consumer.topic.options`, as `vmagent` will automatically manage it. See [#10395](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10395).

View File

@@ -37,7 +37,7 @@ Released at 2020-12-19
* BUGFIX: vmalert: properly populate template variables. This has been broken in v1.50.0. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/974>
* BUGFIX: properly parse negative combined duration in MetricsQL such as `-1h3m4s`. It must be parsed as `-(1h + 3m + 4s)`. Previously it was parsed as `-1h + 3m + 4s`.
* BUGFIX: properly parse lines in [Prometheus exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format) and in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md) with whitespace after the timestamp. For example, `foo 123 456 ## some comment here`. See <https://github.com/VictoriaMetrics/VictoriaMetrics/pull/970>
* BUGFIX: properly parse lines in [Prometheus exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md) and in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md) with whitespace after the timestamp. For example, `foo 123 456 ## some comment here`. See <https://github.com/VictoriaMetrics/VictoriaMetrics/pull/970>
## [v1.50.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.50.1)
@@ -86,7 +86,7 @@ Released at 2020-12-05
* FEATURE: upgrade Go builder from v1.15.5 to v1.15.6 . This fixes [issues found in Go since v1.15.5](https://github.com/golang/go/issues?q=milestone%3AGo1.15.6+label%3ACherryPickApproved).
* BUGFIX: properly parse timestamps in OpenMetrics format - they are exposed as floating-point number in seconds instead of integer milliseconds
unlike in Prometheus exposition format. See [the docs](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#timestamps).
unlike in Prometheus exposition format. See [the docs](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md#timestamps).
* BUGFIX: return `nan` for `a >bool b` query when `a` equals to `nan` like Prometheus does. Previously `0` was returned in this case. This applies to any comparison operation
with `bool` modifier. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) for details.
* BUGFIX: properly parse hex numbers in MetricsQL. Previously hex numbers with non-decimal digits such as `0x3b` couldn't be parsed.
@@ -119,8 +119,8 @@ Released at 2020-11-26
* FEATURE: log metric name plus all its labels when the metric timestamp is out of the configured retention. This should simplify detecting the source of metrics with unexpected timestamps.
* FEATURE: add `-dryRun` command-line flag to single-node VictoriaMetrics in order to check config file pointed by `-promscrape.config`.
* BUGFIX: properly parse Prometheus metrics with [exemplars](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) such as `foo 123 ## {bar="baz"} 1`.
* BUGFIX: properly parse "infinity" values in [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#abnf).
* BUGFIX: properly parse Prometheus metrics with [exemplars](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#exemplars-1) such as `foo 123 ## {bar="baz"} 1`.
* BUGFIX: properly parse "infinity" values in [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#abnf).
See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924>
## [v1.47.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.47.0)

View File

@@ -275,7 +275,7 @@ Released at 2021-07-15
* FEATURE: support durations anywhere in [MetricsQL queries](https://docs.victoriametrics.com/victoriametrics/metricsql/). For example, `sum_over_time(m[1h]) / 1h` is a valid query, which is equivalent to `sum_over_time(m[1h]) / 3600`.
* FEATURE: support durations without suffixes in [MetricsQL queries](https://docs.victoriametrics.com/victoriametrics/metricsql/). For example, `rate(m[3600])` is a valid query, which is equivalent to `rate(m[1h])`.
* FEATURE: export `vmselect_request_duration_seconds` and `vminsert_request_duration_seconds` [VictoriaMetrics histograms](https://valyala.medium.com/improving-histogram-usability-for-prometheus-and-grafana-bc7e5df0e350) at `/metrics` page. These histograms can be used for determining latency distribution and SLI/SLO for the served requests. For example, the following query would return the percent of queries that took less than 500ms during the last hour: `histogram_share(500ms, sum(rate(vmselect_request_duration_seconds_bucket[1h])) by (vmrange))`.
* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/access-controls/service-credentials/mutual-tls-authentication/). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
* FEATURE: log http request path plus all the query args on errors during request processing. Previously only http request path was logged without query args, so it could be hard debugging such errors.
* FEATURE: add `is_set` label to `flag` metrics. This allows determining explicitly set command-line flags with the query `flag{is_set="true"}`.
* FEATURE: add ability to remove caches stored inside `<-storageDataPath>/cache` on startup if `reset_cache_on_startup` file is present there. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1447).
@@ -590,7 +590,7 @@ Released at 2021-01-13
* FEATURE: add ability to pass multiple labels to `sort_by_label()` and `sort_by_label_desc()` functions. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/992> .
* FEATURE: enforce at least TLS v1.2 when accepting HTTPS requests if `-tls`, `-tlsCertFile` and `-tlsKeyFile` command-line flags are set, because older TLS protocols such as v1.0 and v1.1 have been deprecated due to security vulnerabilities.
* FEATURE: support `extra_label` query arg for all HTTP-based [data ingestion protocols](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-time-series-data). This query arg can be used for specifying extra labels which should be added for the ingested data.
* FEATURE: vmbackup: increase backup chunk size from 128MB to 1GB. This should reduce the number of Object storage API calls during backups by 8x. This may also reduce costs, since object storage API calls usually have non-zero costs. See <https://aws.amazon.com/s3/pricing/> and <https://cloud.google.com/storage/pricing#operation-charges> .
* FEATURE: vmbackup: increase backup chunk size from 128MB to 1GB. This should reduce the number of Object storage API calls during backups by 8x. This may also reduce costs, since object storage API calls usually have non-zero costs. See <https://aws.amazon.com/s3/pricing/> and <https://cloud.google.com/storage/pricing#operations-pricing> .
* BUGFIX: properly parse escaped unicode chars in MetricsQL metric names, label names and function names. See <https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990>
* BUGFIX: override user-provided labels with labels set in `extra_label` query args during data ingestion over HTTP-based protocols.

View File

@@ -978,7 +978,7 @@ Released at 2022-02-14
Released at 2022-01-18
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add support for `@` modifier, which is enabled by default in Prometheus starting from [Prometheus v2.33.0](https://github.com/prometheus/prometheus/pull/10121). See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#-modifier) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1348). VictoriaMetrics extends `@` modifier with the following additional features:
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add support for `@` modifier, which is enabled by default in Prometheus starting from [Prometheus v2.33.0](https://github.com/prometheus/prometheus/pull/10121). See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1348). VictoriaMetrics extends `@` modifier with the following additional features:
* It can contain arbitrary expression. For example, `foo @ (end() - 1h)` would return `foo` value at `end - 1 hour` timestamp on the selected time range `[start ... end]`. Another example: `foo @ (now() - 10m)` would return `foo` value 10 minutes ago from the current time.
* It can be put everywhere in the query. For example, `sum(foo) @ start()` would calculate `sum(foo)` at `start` timestamp on the selected time range `[start ... end]`.
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): add support for optional `keep_metric_names` modifier, which can be applied to all the [rollup functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#rollup-functions) and [transform functions](https://docs.victoriametrics.com/victoriametrics/metricsql/#transform-functions). This modifier prevents from deleting metric names from function results. For example, `rate({__name__=~"foo|bar"}[5m]) keep_metric_names` leaves `foo` and `bar` metric names in `rate()` results. This feature provides an additional workaround for [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/949).

View File

@@ -370,7 +370,7 @@ The v1.93.x line will be supported for at least 12 months since [v1.93.0](https:
* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add `Concurrent inserts` panel to vmagent's dashboard. The new panel supposed to show whether the number of concurrent inserts processed by vmagent isn't reaching the limit.
* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): add panels for absolute Mem and CPU usage by vmalert. See related issue [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4627).
* FEATURE: [Official Grafana dashboards for VictoriaMetrics](https://grafana.com/orgs/victoriametrics): correctly calculate `Bytes per point` value for single-server and cluster VM dashboards. Before, the calculation mistakenly accounted for the number of entries in indexdb in denominator, which could have shown lower values than expected.
* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-single-node.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) as it could be related to many VictoriaMetrics components.
* FEATURE: [Alerting rules for VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#alerts): `ConcurrentFlushesHitTheLimit` alerting rule was moved from [single-server](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts.yml) and [cluster](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-cluster.yml) alerts to the [list of "health" alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-health.yml) as it could be related to many VictoriaMetrics components.
* BUGFIX: [storage](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly set next retention time for indexDB. Previously it may enter into endless retention loop. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4873) for details.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): return human readable error if opentelemetry has json encoding. Follow-up after [PR](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2570).
@@ -558,7 +558,7 @@ Released at 2023-05-18
* BUGFIX: change the max allowed value for `-memory.allowedPercent` from 100 to 200. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4171).
* BUGFIX: properly limit the number of [OpenTSDB HTTP](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) concurrent requests specified via `-maxConcurrentInserts` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4204). Thanks to @zouxiang1993 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4208).
* BUGFIX: do not ignore trailing empty field in CSV lines when [importing data in CSV format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048).
* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#line-format). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-format-example). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent from possible panic when the number of vmstorage nodes increases when [automatic vmstorage discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery) is enabled.
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix a panic when the duration in the query contains uppercase `M` suffix. Such a suffix isn't allowed to use in durations, since it clashes with `a million` suffix, e.g. it isn't clear whether `rate(metric[5M])` means rate over 5 minutes, 5 months or 5 million seconds. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3589) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4120) issues.
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): properly handle the `vm_promscrape_config_last_reload_successful` metric after config reload. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4260).
@@ -825,7 +825,7 @@ The v1.87.x line will be supported for at least 12 months since [v1.87.0](https:
* BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.
* BUGFIX: reduce the possibility of increased CPU usage when data with timestamps older than one hour is ingested into VictoriaMetrics. This reduces spikes for the graph `sum(rate(vm_slow_per_day_index_inserts_total))`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4258).
* BUGFIX: do not ignore trailing empty field in CSV lines when [importing data in CSV format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048).
* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#line-format). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
* BUGFIX: disallow `"` chars when parsing Prometheus label names, since they aren't allowed by [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-format-example). Previously this could result in silent incorrect parsing of incorrect Prometheus labels such as `foo{"bar"="baz"}` or `{foo:"bar",baz="aaa"}`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4284).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): fix a panic when the duration in the query contains uppercase `M` suffix. Such a suffix isn't allowed to use in durations, since it clashes with `a million` suffix, e.g. it isn't clear whether `rate(metric[5M])` means rate over 5 minutes, 5 months or 5 million seconds. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3589) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4120) issues.
* BUGFIX: [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): prevent from possible panic when the number of vmstorage nodes increases when [automatic vmstorage discovery](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery) is enabled.
* BUGFIX: properly limit the number of [OpenTSDB HTTP](https://docs.victoriametrics.com/victoriametrics/integrations/opentsdb/#sending-data-via-http) concurrent requests specified via `-maxConcurrentInserts` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4204). Thanks to @zouxiang1993 for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4208).

View File

@@ -443,7 +443,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
* `position` - the position of the aggregation rule in the corresponding streaming aggregation config file
* FEATURE: [streaming aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): prevent having duplicated aggregation function as `outputs` in one [aggregation config](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/configuration/#stream-aggregation-config). It also prevents using `outputs: ["quantiles(0.5)", "quantiles(0.9)"]` instead of `outputs: ["quantiles(0.5, 0.9)"]`, as the former has higher computation cost for producing the same result.
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [Single-node VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): add `-graphite.sanitizeMetricName` command-line flag for sanitizing metrics ingested via [Graphite protocol](https://docs.victoriametrics.com/victoriametrics/integrations/graphite/#ingesting). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6077).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs): add support for obtaining IAM token in [GCE format](https://yandex.cloud/en/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm) additionally to the [deprecated Amazon EC2 IMDSv1 format](https://yandex.cloud/en/docs/security/standard/authentication#aws-token). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5513).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): [`yandexcloud_sd_configs`](https://docs.victoriametrics.com/victoriametrics/sd_configs/#yandexcloud_sd_configs): add support for obtaining IAM token in [GCE format](https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm) additionally to the [deprecated Amazon EC2 IMDSv1 format](https://yandex.cloud/en/docs/security/standard/authentication#aws-token). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5513).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): make `-replay.timeTo` optional in [replay mode](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling). When omitted, the current timestamp will be used. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6492).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): reduce CPU usage when proxying data ingestion requests.
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): allow disabling request body caching with `-maxRequestBodySizeToRetry=0`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6445). Thanks to @shichanglin5 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6533).
@@ -674,7 +674,7 @@ Released at 2024-03-01
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): preserve [`WITH` templates](https://play.victoriametrics.com/select/0/prometheus/graph/#/expand-with-exprs) when clicking the `prettify query` button at the right side of query input field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5383).
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): allow filling gaps on graphs with interpolated lines as Grafana does. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5152) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5862).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmalert): support filtering by group, rule or labels in [vmalert's UI](https://docs.victoriametrics.com/victoriametrics/vmalert/#web) for `/groups` and `/alerts` pages. See [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5791) by @victoramsantos.
* FEATURE: [docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#docker-compose-environment-for-victoriametrics): create a separate [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaLogs/tree/master/deployment/docker#victorialogs-server) for VictoriaLogs installation, including fluentbit and [VictoriaLogs Grafana datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). See [these docs](https://github.com/VictoriaMetrics/VictoriaLogs/tree/master/deployment/docker#victorialogs-server) for details.
* FEATURE: [docker-compose](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#docker-compose-environment-for-victoriametrics): create a separate [docker-compose environment](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#victoriaLogs-server) for VictoriaLogs installation, including fluentbit and [VictoriaLogs Grafana datasource](https://github.com/VictoriaMetrics/victorialogs-datasource). See [these docs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#victoriaLogs-server) for details.
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): wait for up 30 seconds before making a [snapshot](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-work-with-snapshots) for backup if `vmstorage` is temporarily unavailable. This should prevent from `vmbackupmanager` termination in this case. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5859).
* BUGFIX: downgrade Go builder from `1.22.0` to `1.21.7`, since `1.22.0` contains [the bug](https://github.com/golang/go/issues/65705), which can lead to deadlocked HTTP connections to remote storage systems, scrape targets and service discovery endpoints at [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/). This may result in incorrect service discovery, target scraping and failed sending samples to remote storage.

View File

@@ -1251,7 +1251,7 @@ Released at 2025-01-14
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victorialogs/vmalert/): do not append tenant info to VictoriaLogs datasource request path in [clusterMode](https://docs.victoriametrics.com/victoriametrics/vmalert/#multitenancy). See [this doc](https://docs.victoriametrics.com/victorialogs/vmalert/#how-to-use-multitenancy-in-rules) for how to use multitenancy in VictoriaLogs.
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly set `host` field at debug information formatted with `dump_request_on_errors: true` setting.
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly handle discovery for ipv6 addresses. Thanks to @badie for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7955).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7935). Thanks to @bitbidu for reporting.
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly ingest `influx` line protocol metrics with empty tags. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7933) for details.
* BUGFIX: [vmselect](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): allow to override the default unique time series limit in vmstorage with command-line flags like `-search.maxUniqueTimeseries`, `-search.maxLabelsAPISeries`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7852).
@@ -1515,7 +1515,7 @@ The v1.102.x line will be supported for at least 12 months since [v1.102.0](http
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): make instant query results consistent. VictoriaMetrics detects and adjusts scrape interval and while this is very useful for range queries (i.e. this eliminates gaps on the graph), it may cause instant queries to return a non-empty result when no result is expected. The fix is to disable scrape interval detection and always use the step as the scrape interval in instant queries. This will guarantee that the samples are searched within the (time-step, time] interval. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5796) for details.
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): fix cursor reset in query input field. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7288).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly handle discovery for ipv6 addresses. Thanks to @badie for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7955).
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7935). Thanks to @bitbidu for reporting.
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
## [v1.102.9](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.9)
@@ -1622,7 +1622,7 @@ The v1.97.x line will be supported for at least 12 months since [v1.97.0](https:
* FEATURE: all VictoriaMetrics [enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) components: add support of hot-reload for license key supplied by `-licenseFile` command-line flag.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/), `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): make instant query results consistent. VictoriaMetrics detects and adjusts scrape interval and while this is very useful for range queries (i.e. this eliminates gaps on the graph), it may cause instant queries to return a non-empty result when no result is expected. The fix is to disable scrape interval detection and always use the step as the scrape interval in instant queries. This will guarantee that the samples are searched within the (time-step, time] interval. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5796) for details.
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/7935). Thanks to @bitbidu for reporting.
* BUGFIX: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): fix support for migrating influx series without any tag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7921). Thanks to @bitbidu for reporting.
* BUGFIX: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): storage nodes defined in `-storageNode` are now sorted, ensuring that varying node orders across different vminsert instances do not result in inconsistent replication.
## [v1.97.14](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.14)

View File

@@ -11,7 +11,7 @@ menu:
OpenShift uses Prometheus as a core monitoring solution. It cannot be replaced without violating the OpenShift support policy. However, OpenShift can be configured to use VictoriaMetrics as a remote write target.
According to [remote write configuration in the OpenShift documentation](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_core_platform_monitoring/configuring-metrics#configuring-remote-write-storage_configuring-metrics), the following manifest needs to be applied to send platform metrics to VictoriaMetrics:
According to [remote write configuration in the OpenShift documentation](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-core-platform-monitoring#configuring-remote-write-storage_configuring-metrics), the following manifest needs to be applied to send platform metrics to VictoriaMetrics:
```yaml
apiVersion: v1
@@ -91,7 +91,7 @@ data:
key: token
```
Along with core platform monitoring, OpenShift also supports collecting user workload metrics. See [this guide](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_user_workload_monitoring/index) for more information. In order to send user workload metrics to VictoriaMetrics, the following manifest needs to be applied:
Along with core platform monitoring, OpenShift also supports collecting user workload metrics. See [this guide](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-user-workload-monitoring) for more information. In order to send user workload metrics to VictoriaMetrics, the following manifest needs to be applied:
```yaml
apiVersion: v1
kind: ConfigMap
@@ -166,6 +166,6 @@ data:
## References
- [OpenShift Documentation: Core Platform Monitoring](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_core_platform_monitoring/index)
- [OpenShift Documentation: User Workload Monitoring](https://docs.redhat.com/en/documentation/monitoring_stack_for_red_hat_openshift/4.18/html/configuring_user_workload_monitoring/index)
- [OpenShift Documentation: Core Platform Monitoring](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-core-platform-monitoring)
- [OpenShift Documentation: User Workload Monitoring](https://docs.redhat.com/en/documentation/openshift_container_platform/4.18/html/monitoring/configuring-user-workload-monitoring)
- [OpenShift Documentation: Hosted Control Planes Overview](https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/hosted_control_planes/hosted-control-planes-overview)

View File

@@ -46,7 +46,7 @@ If you don't see an option to create a data source - try contacting system admin
## Prometheus datasource
Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure/)
Create [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/configure-prometheus-data-source/)
in Grafana. Follow the same connection instructions as for [VictoriaMetrics datasource](#VictoriaMetrics-datasource).
In the "Type and version" section set the type to "Prometheus" and the version to at least "2.24.x".

View File

@@ -81,7 +81,7 @@ The `/api/v1/export` endpoint should return the following response:
## Data transformations
VictoriaMetrics performs the following transformations to the ingested InfluxDB data:
* [db query arg](https://docs.influxdata.com/influxdb/v1/api/write/#operation/PostWrite) is mapped into `db`
* [db query arg](https://docs.influxdata.com/influxdb/v1.7/tools/api/#write-http-endpoint) is mapped into `db`
[label](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#labels) value unless `db` tag exists in the InfluxDB line.
The `db` label name can be overridden via `-influxDBLabel` command-line flag. If more strict data isolation is required,
read more about multi-tenancy [here](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#multi-tenancy).

View File

@@ -19,8 +19,8 @@ Use `-kafka.consumer.topic.defaultFormat` or `-kafka.consumer.topic.format` comm
* `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
Messages in this format can be sent by vmagent - see [these docs](#writing-metrics).
* `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/).
* `prometheus` - [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format)
and [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md).
* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
* `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
* `jsonline` - [JSON line format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-json-line-format).
* `opentelemetry`{{% available_from "v1.128.0" %}} - [Opentelemetry format](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/)

View File

@@ -18,8 +18,8 @@ Use `-gcp.pubsub.subscribe.defaultMessageFormat` and `-gcp.pubsub.subscribe.topi
* `promremotewrite` - [Prometheus remote_write](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write).
Messages in this format can be sent by vmagent - see [these docs](#writing-metrics).
* `influx` - [InfluxDB line protocol format](https://docs.influxdata.com/influxdb/cloud/reference/syntax/line-protocol/).
* `prometheus` - [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format)
and [OpenMetrics format](https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md).
* `prometheus` - [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format)
and [OpenMetrics format](https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md).
* `graphite` - [Graphite plaintext format](https://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol).
* `jsonline` - [JSON line format](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-json-line-format).

View File

@@ -84,7 +84,7 @@ If the value has more than 12 significant decimal digits, then the less signific
The `timestamp` is a [Unix timestamp](https://en.wikipedia.org/wiki/Unix_time) with millisecond precision.
Below is an example of a single raw sample
in [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format):
in [Prometheus text exposition format](https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md#text-based-format):
```
requests_total{path="/", code="200"} 123 4567890
@@ -456,7 +456,7 @@ The basic monitoring setup of VictoriaMetrics and vmagent is described
in the [example docker-compose manifest](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker#readme).
In this example vmagent [scrapes a list of targets](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/prometheus-vm-single.yml)
and [forwards collected data to VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/9751ea10983d42068487624849cac7ad6fd7e1d8/deployment/docker/compose-vm-single.yml#L16).
VictoriaMetrics is then used as a [datasource for Grafana](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/provisioning/datasources/prometheus/single.yml)
VictoriaMetrics is then used as a [datasource for Grafana](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/provisioning/datasources/prometheus-datasource/single.yml)
installation for querying collected data.
VictoriaMetrics components allow building more advanced topologies. For example, vmagents can push metrics from separate datacenters to the central VictoriaMetrics:
@@ -492,7 +492,7 @@ Params:
* `time` - optional, [timestamp](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats)
in millisecond precision to evaluate the `query` at. If omitted, `time` is set to `now()` (current timestamp).
The `time` param can be specified in [multiple allowed formats](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats).
* `step` - optional [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations)
* `step` - optional [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations)
for searching for raw samples in the past when executing the `query` (used when a sample is missing at the specified `time`).
For example, the request `/api/v1/query?query=up&step=1m` looks for the last written raw sample for the metric `up`
in the `(now()-1m, now()]` interval (the first millisecond is not included). If omitted, `step` is set to `5m` (5 minutes)
@@ -590,7 +590,7 @@ Params:
* `end` - the ending [timestamp](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#timestamp-formats)
of the time range for `query` evaluation.
If the `end` isn't set, then the `end` is automatically set to the current time.
* `step` - the [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations)
* `step` - the [interval](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations)
between data points, which must be returned from the range query.
The `query` is executed at `start`, `start+step`, `start+2*step`, ..., `start+N*step` timestamps,
where `N` is the whole number of steps that fit between `start` and `end`.
@@ -929,7 +929,7 @@ rate(node_network_receive_bytes_total)
By default, VictoriaMetrics calculates the `rate` over [raw samples](#raw-samples) on the lookbehind window specified in the `step` param
passed either to [instant query](#instant-query) or to [range query](#range-query).
The interval on which `rate` needs to be calculated can be specified explicitly
as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#float-literals-and-time-durations) in square brackets:
as [duration](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-durations) in square brackets:
```metricsql
rate(node_network_receive_bytes_total[5m])

View File

@@ -342,7 +342,7 @@ See [these docs](https://docs.victoriametrics.com/victoriametrics/sd_configs/#ht
The following [`-promscrape.config`](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-scrape-prometheus-exporters-such-as-node-exporter)
instructs discovering and scraping metrics for all the containers with the name `my-super-app`.
It is expected that these containers expose only a single TCP port, which serves its metrics at `/metrics` page
according to [Prometheus text exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/#prometheus-text-format):
according to [Prometheus text exposition format](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#text-based-format):
```yaml
scrape_configs:

View File

@@ -32,7 +32,7 @@ supports the following Prometheus-compatible service discovery options for Prome
* `http_sd_configs` is for discovering and scraping targets provided by external http-based service discovery. See [these docs](#http_sd_configs).
* `kubernetes_sd_configs` is for discovering and scraping [Kubernetes](https://kubernetes.io/) targets. See [these docs](#kubernetes_sd_configs).
* `kuma_sd_configs` is for discovering and scraping [Kuma](https://kuma.io) targets. See [these docs](#kuma_sd_configs).
* `marathon_sd_configs` is for discovering and scraping [Marathon](https://github.com/d2iq-archive/marathon) targets. See [these docs](#marathon_sd_configs).
* `marathon_sd_configs` is for discovering and scraping [Marathon](https://mesosphere.github.io/marathon/) targets. See [these docs](#marathon_sd_configs).
* `nomad_sd_configs` is for discovering and scraping targets registered in [HashiCorp Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs).
* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs).
* `ovhcloud_sd_configs` is for discovering and scraping OVH Cloud VPS and dedicated server targets. See [these docs](#ovhcloud_sd_configs).
@@ -324,7 +324,7 @@ The list of discovered Consul Agent targets is refreshed at the interval, which
## digitalocean_sd_configs
DigitalOcean SD configuration allows retrieving scrape targets from [DigitalOcean's Droplets API](https://docs.digitalocean.com/reference/api/reference/droplets/).
DigitalOcean SD configuration allows retrieving scrape targets from [DigitalOcean's Droplets API](https://docs.digitalocean.com/reference/api/api-reference/#tag/Droplets).
Configuration example:
@@ -1315,7 +1315,7 @@ The list of discovered Kuma targets is refreshed at the interval, which can be c
## marathon_sd_configs
Marathon SD configuration {{% available_from "v1.109.0" %}} allows retrieving scrape targets from [Marathon](https://github.com/d2iq-archive/marathon) REST API.
Marathon SD configuration {{% available_from "v1.109.0" %}} allows retrieving scrape targets from [Marathon](https://mesosphere.github.io/marathon/) REST API.
Configuration example:
@@ -1350,7 +1350,7 @@ The list of discovered Marathon targets is refreshed at the interval, which can
## nomad_sd_configs
Nomad SD configuration allows retrieving scrape targets from [HashiCorp Nomad Services](https://www.hashicorp.com/en/blog/nomad-service-discovery).
Nomad SD configuration allows retrieving scrape targets from [HashiCorp Nomad Services](https://www.hashicorp.com/blog/nomad-service-discovery).
Configuration example:

View File

@@ -268,39 +268,6 @@ for the collected samples. Examples:
```sh
./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
```
### Monitoring Data eXchange
The Monitoring Data eXchange (MDX){{% available_from "#" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
To enable MDX, set `-remoteWrite.mdx.enable=true` for the target URL and `-remoteWrite.mdx.enable=false` for other URLs:
```sh
./vmagent \
-remoteWrite.url=http://service-to-keep-all-metrics:8428/api/v1/write \
-remoteWrite.mdx.enable=false \
-remoteWrite.url=http://service-to-keep-only-vm-metrics:8428/api/v1/write \
-remoteWrite.mdx.enable=true
```
When MDX is enabled for a `-remoteWrite.url`, `vmagent` forwards only metrics that:
- come from the target that exposes the `vm_app_version` metric (emitted by all VictoriaMetrics components)
- contain the `victoriametrics_app=true` label, which will be added automatically to the metrics if the instance was deployed via [VictoriaMetrics Operator](https://docs.victoriametrics.com/operator/).
`victoriametrics_app=true` label will be added to all metrics that are preserved by MDX if it's absent.
- contain the label specified via `-mdx.label`.
```sh
./vmagent \
-remoteWrite.url=http://service-to-keep-only-vm-metrics:8428/api/v1/write \
-remoteWrite.mdx.enable=true \
-mdx.label="service=victoriametrics"
```
In this configuration, metrics with the label `service=victoriametrics` are preserved even if their scrape targets do not expose `vm_app_version` metric.
The number of VictoriaMetrics metrics preserved by MDX is exposed as `vmagent_remotewrite_mdx_rows_preserved_total`.
The scope of MDX is at the per-url level, so it works after global level mechanisms, such as stream aggregation, relabeling, complexity limiter, and cardinality limiter. See [Life of a sample](https://docs.victoriametrics.com/victoriametrics/vmagent/#life-of-a-sample).
### Life of a sample
@@ -318,20 +285,18 @@ flowchart TB
F --> G[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#replication-and-high-availability">replicate</a> to each <b>-remoteWrite.url</b><br/>or <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#sharding-among-remote-storages">shard</a> if <b>-remoteWrite.shardByURL</b> is set]
%% Left branch
G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange/">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
H2 --> H3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
H3 --> H4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
H4 --> H5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
H5 --> H6[[push to <b>-remoteWrite.url</b>]]
G --> H1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
H1 --> H2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
H2 --> H3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
H3 --> H4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
H4 --> H5[[push to <b>-remoteWrite.url</b>]]
%% Right branch
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#monitoring-data-exchange">mdx filter</a><br><b>-remoteWrite.mdx.enable</b>]
R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
R2 --> R3[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
R3 --> R4["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
R4 --> R5[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
R5 --> R6[[push to <b>-remoteWrite.url</b>]]
G --> R1[per-url <a href="https://docs.victoriametrics.com/victoriametrics/relabeling/">relabeling</a><br><b>-remoteWrite.urlRelabelConfig</b>]
R1 --> R2[per-url <a href="https://docs.victoriametrics.com/victoriametrics/stream-aggregation">aggregation</a><br><b>-remoteWrite.streamAggr.config</b><br><b>-remoteWrite.streamAggr.dedupInterval</b>]
R2 --> R3["per-url <a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#calculating-disk-space-for-persistence-queue">queue</a> (default: enabled)<br><b>-remoteWrite.disableOnDiskQueue</b>"]
R3 --> R4[<a href="https://docs.victoriametrics.com/victoriametrics/vmagent/#adding-labels-to-metrics">add extra labels</a><br><b>-remoteWrite.label</b>]
R4 --> R5[[push to <b>-remoteWrite.url</b>]]
```
Scraping has additional settings that can be applied before samples are pushed to the processing pipeline above:

View File

@@ -768,7 +768,7 @@ to set the `-datasource.appendTypePrefix` flag to `true`, so vmalert can adjust
###### Prometheus
vmalert uses [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/) for querying
vmalert uses [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/#http-api) for querying
and [Prometheus Remote Write v1 protocol](https://prometheus.io/docs/specs/prw/remote_write_spec/) for persisting
recording rules results and alerting state. Hence, it can be integrated with any Prometheus-compatible storage
that supports these protocols.

View File

@@ -270,7 +270,7 @@ users:
url_prefix: "http://victoria-metrics:8428/"
```
The `vm_access` claim is optional starting from {{% available_from "#" %}}: when present it is used for [request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating), and when absent the default tenant `0:0` is assumed for any `vm_access`-based placeholders. Routing can rely solely on other token claims via [JWT claim matching](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-matching).
JWT tokens must contain a `"vm_access": {}` claim, more on that in [JWT claim-based request templating](https://docs.victoriametrics.com/victoriametrics/vmauth/#jwt-claim-based-request-templating)
For testing, skip signature verification with `skip_verify: true` (not recommended for production).
@@ -520,8 +520,7 @@ for dynamic URL rewriting based on `vm_access` claim fields.
`vmauth` can dynamically rewrite{{% available_from "v1.137.0" %}} upstream URLs and request headers using values from the JWT `vm_access` claim.
This enables routing different users to different backends or tenants based solely on the JWT token,
without maintaining separate user configs per tenant. In addition `vm_access` claim could be defined at `jwt` section with `default_vm_access_claim` {{% available_from "#" %}}.
In this case, if JWT token doesn't have `vm_access` claim defined, value from `default_vm_access_claim` will be used for templaing.
without maintaining separate user configs per tenant.
Example: minimal valid JWT. If vm_access is empty, tenant `0:0` is assumed and no additional filters are applied.
```json
@@ -576,28 +575,6 @@ Placeholders are supported in the following locations:
Placeholders are **not** supported in response headers.
They are also only valid for JWT-authenticated users — using them in configs for `username`/`password` or `bearer_token` users causes a configuration error.
Example: default `vm_access` claim:
```yaml
users:
- jwt:
default_vm_access_claim:
metrics_account_id: 10
metrics_project_id: 10
metrics_extra_filters:
- '{instance="sandbox"}'
metrics_extra_labels:
- team=dev
- env=dev
public_keys:
- |
-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
-----END PUBLIC KEY-----
url_prefix: "http://vminsert:8480/insert/{{.MetricsAccountID}}:{{.MetricsProjectID}}/prometheus/?extra_filters={{.MetricsExtraFilters}}&extra_label={{.MetricsExtraLabels}}"
```
Example: route requests to the VictoriaMetrics single-node:
```yaml

View File

@@ -88,7 +88,7 @@ Here's an example of importing timeseries for one day only:
--influx-filter-time-end "2020-01-01T15:07:00Z"
```
See more about [time filtering in InfluxDB](https://docs.influxdata.com/influxdb/v1/query_language/explore-schema/#filter-meta-queries-by-time).
See more about [time filtering in InfluxDB](https://docs.influxdata.com/influxdb/v1.7/query_language/schema_exploration#filter-meta-queries-by-time).
## InfluxDB v2
@@ -103,7 +103,7 @@ Increase `--influx-concurrency` to execute more read requests concurrently. But
during migration.
The flag `--influx-chunk-size` controls the max amount of datapoints to return in single chunk from fetch requests.
Please see more details [here](https://archive.docs.influxdata.com/influxdb/v1.2/guides/querying_data/#chunking).
Please see more details [here](https://docs.influxdata.com/influxdb/v1.7/guides/querying_data/#chunking).
The chunk size is used to control InfluxDB memory usage, so it won't OOM on processing large timeseries with
billions of datapoints.
@@ -111,4 +111,4 @@ See general [vmctl migration tips](https://docs.victoriametrics.com/victoriametr
See `./vmctl influx --help` for details and full list of flags:
{{% content "vmctl_influx_flags.md" %}}
{{% content "vmctl_influx_flags.md" %}}

View File

@@ -663,7 +663,7 @@ Below is the list of configuration flags (it can be viewed by running `./vmgatew
## Troubleshooting
* Access control:
* incorrect `jwt` format, try <https://www.jwt.io/> with our tokens
* incorrect `jwt` format, try <https://jwt.io/#debugger-io> with our tokens
* expired token, check `exp` field.
* Rate Limiting:
* `scrape_interval` at the datasource, reduce it to apply limits faster.

View File

@@ -1,140 +0,0 @@
package handshake
import (
"bufio"
"fmt"
"io"
"net"
"os"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding/zstd"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
)
type bufferedWriter interface {
Write(p []byte) (int, error)
Flush() error
}
// BufferedConn is a net.Conn with Flush suport.
type BufferedConn struct {
net.Conn
// IsLegacy defines if BufferedConn operates in legacy mode
// and doesn't support RPC protocol
IsLegacy bool
br io.Reader
bw bufferedWriter
readDeadline time.Time
writeDeadline time.Time
}
const bufferSize = 64 * 1024
// newBufferedConn returns buffered connection with the given compression level.
func newBufferedConn(c net.Conn, compressionLevel int, isReadCompressed bool) *BufferedConn {
bc := &BufferedConn{
Conn: c,
}
if compressionLevel <= 0 {
bc.bw = bufio.NewWriterSize(c, bufferSize)
} else {
bc.bw = zstd.NewWriterLevel(c, compressionLevel)
}
if !isReadCompressed {
bc.br = bufio.NewReaderSize(c, bufferSize)
} else {
bc.br = zstd.NewReader(c)
}
return bc
}
// SetDeadline sets read and write deadlines for bc to t.
//
// Deadline is checked on each Read and Write call.
func (bc *BufferedConn) SetDeadline(t time.Time) error {
bc.readDeadline = t
bc.writeDeadline = t
return bc.Conn.SetDeadline(t)
}
// SetReadDeadline sets read deadline for bc to t.
//
// Deadline is checked on each Read call.
func (bc *BufferedConn) SetReadDeadline(t time.Time) error {
bc.readDeadline = t
return bc.Conn.SetReadDeadline(t)
}
// SetWriteDeadline sets write deadline for bc to t.
//
// Deadline is checked on each Write call.
func (bc *BufferedConn) SetWriteDeadline(t time.Time) error {
bc.writeDeadline = t
return bc.Conn.SetWriteDeadline(t)
}
// Read reads up to len(p) from bc to p.
func (bc *BufferedConn) Read(p []byte) (int, error) {
startTime := fasttime.UnixTimestamp()
if deadlineExceeded(bc.readDeadline, startTime) {
return 0, os.ErrDeadlineExceeded
}
n, err := bc.br.Read(p)
if err != nil && err != io.EOF {
err = fmt.Errorf("cannot read data in %d seconds: %w", fasttime.UnixTimestamp()-startTime, err)
}
return n, err
}
// Write writes p to bc.
//
// Do not forget to call Flush if needed.
func (bc *BufferedConn) Write(p []byte) (int, error) {
startTime := fasttime.UnixTimestamp()
if deadlineExceeded(bc.writeDeadline, startTime) {
return 0, os.ErrDeadlineExceeded
}
n, err := bc.bw.Write(p)
if err != nil {
err = fmt.Errorf("cannot write data in %d seconds: %w", fasttime.UnixTimestamp()-startTime, err)
}
return n, err
}
func deadlineExceeded(deadline time.Time, currentTimestamp uint64) bool {
if deadline.IsZero() {
return false
}
return currentTimestamp > uint64(deadline.Unix())
}
// Close closes bc.
func (bc *BufferedConn) Close() error {
// Close the Conn at first. It is expected that all the required data
// is already flushed to the Conn.
err := bc.Conn.Close()
bc.Conn = nil
if zr, ok := bc.br.(*zstd.Reader); ok {
zr.Release()
}
bc.br = nil
if zw, ok := bc.bw.(*zstd.Writer); ok {
// Do not call zw.Close(), since we already closed the underlying conn.
zw.Release()
}
bc.bw = nil
bc.IsLegacy = false
return err
}
// Flush flushes internal write buffers to the underlying conn.
func (bc *BufferedConn) Flush() error {
return bc.bw.Flush()
}

View File

@@ -1,319 +0,0 @@
package handshake
import (
"errors"
"flag"
"fmt"
"io"
"net"
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
)
var rpcHandshakeTimeout = flag.Duration("rpc.handshakeTimeout", 5*time.Second, "Timeout for RPC handshake between vminsert/vmselect and vmstorage. Increase this value if transient handshake failures occur. See https://docs.victoriametrics.com/victoriametrics/troubleshooting/#cluster-instability section for more details.")
const (
vminsertHelloLegacyVersion = "vminsert.02"
vminsertHello = "vminsert.03"
vmselectHello = "vmselect.01"
successResponse = "ok"
)
// Func must perform handshake on the given c using the given compressionLevel.
//
// It must return BufferedConn wrapper for c on successful handshake.
type Func func(c net.Conn, compressionLevel int) (*BufferedConn, error)
// VMInsertClientWithDialer performs client-side handshake for vminsert protocol.
//
// it uses provided dial func to establish connection to the server.
// compressionLevel is a legacy option which defines the level used for compression of the data sent
// to the server.
// compressionLevel <= 0 means 'no compression'
func VMInsertClientWithDialer(dial func() (net.Conn, error), compressionLevel int) (*BufferedConn, error) {
c, err := dial()
if err != nil {
return nil, fmt.Errorf("dial error: %w", err)
}
bc, err := vminsertClient(c, 0)
if err == nil {
return bc, nil
}
_ = c.Close()
// fallback only if vmstorage closed connection at read success response
if !errors.Is(err, io.EOF) && !strings.Contains(err.Error(), "cannot read success response after sending hello") {
return nil, err
}
// try to fallback to the prev non-RPC API version
// we cannot re-use exist connection, since vmstorage already closed it
c, err = dial()
if err != nil {
return nil, fmt.Errorf("dial error: %w", err)
}
bc, err = genericClient(c, vminsertHelloLegacyVersion, compressionLevel)
if err != nil {
_ = c.Close()
return nil, fmt.Errorf("legacy handshake error: %w", err)
}
bc.IsLegacy = true
logger.Infof("server=%q doesn't support new RPC version, fallback to the legacy format", c.RemoteAddr())
return bc, nil
}
func vminsertClient(c net.Conn, compressionLevel int) (*BufferedConn, error) {
return genericClient(c, vminsertHello, compressionLevel)
}
// VMInsertClientWithHello performs client-side handshake for vminsert protocol.
//
// should be used for testing only
func VMInsertClientWithHello(c net.Conn, helloMsg string, compressionLevel int) (*BufferedConn, error) {
return genericClient(c, helloMsg, compressionLevel)
}
// VMInsertServer performs server-side handshake for vminsert protocol.
//
// compressionLevel is the level used for compression of the data sent
// to the client.
// compressionLevel <= 0 means 'no compression'
func VMInsertServer(c net.Conn, compressionLevel int) (*BufferedConn, error) {
var isRPCSupported bool
bc, err := genericServer(c, compressionLevel, func(c net.Conn) error {
buf, err := readData(c, len(vminsertHello))
if err != nil {
if errors.Is(err, io.EOF) {
// This is likely a TCP healthcheck, which must be ignored in order to prevent logs pollution.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1762
return errTCPHealthcheck
}
return fmt.Errorf("cannot read hello: %w", err)
}
isRPCSupported = string(buf) == vminsertHello
if !isRPCSupported {
// try to fallback to the previous protocol version
if string(buf) != vminsertHelloLegacyVersion {
return fmt.Errorf("unexpected message obtained; got %q; want %q", buf, vminsertHello)
}
logger.Infof("client=%q doesn't support new RPC version, fallback to the legacy format", c.RemoteAddr())
}
return nil
})
if err != nil {
return nil, err
}
bc.IsLegacy = !isRPCSupported
return bc, nil
}
// VMInsertServerWithLegacyHello performs server-side handshake for vminsert protocol
// with legacy hello message
//
// should be used for testing only
func VMInsertServerWithLegacyHello(c net.Conn, compressionLevel int) (*BufferedConn, error) {
bc, err := genericServer(c, compressionLevel, func(c net.Conn) error {
return readMessage(c, vminsertHelloLegacyVersion)
})
if err != nil {
return nil, err
}
bc.IsLegacy = true
return bc, nil
}
// VMSelectClient performs client-side handshake for vmselect protocol.
//
// compressionLevel is the level used for compression of the data sent
// to the server.
// compressionLevel <= 0 means 'no compression'
func VMSelectClient(c net.Conn, compressionLevel int) (*BufferedConn, error) {
return genericClient(c, vmselectHello, compressionLevel)
}
// VMSelectServer performs server-side handshake for vmselect protocol.
//
// compressionLevel is the level used for compression of the data sent
// to the client.
// compressionLevel <= 0 means 'no compression'
func VMSelectServer(c net.Conn, compressionLevel int) (*BufferedConn, error) {
return genericServer(c, compressionLevel, func(c net.Conn) error {
err := readMessage(c, vmselectHello)
if errors.Is(err, io.EOF) {
// This is likely a TCP healthcheck, which must be ignored in order to prevent logs pollution.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1762 and https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10786
return errTCPHealthcheck
}
return err
})
}
// errTCPHealthcheck indicates that the connection was opened as part of a TCP health check
// and was closed immediately after being established.
//
// This is expected behavior and can be safely ignored.
var errTCPHealthcheck = fmt.Errorf("TCP health check connection safe to ignore")
// IsTCPHealthcheck determines whether the provided error is a TCP health check
func IsTCPHealthcheck(err error) bool {
return errors.Is(err, errTCPHealthcheck)
}
// IsClientNetworkError determines whether the provided error is a client-side network error,
// such as io.EOF, io.ErrUnexpectedEOF, or a timeout.
// These errors typically occur when a client disconnects abruptly or fails during the handshake,
// and are generally non-actionable from the server point of view.
// This function helps distinguish such errors from critical ones during the handshake process
// and adjust logging accordingly.
//
// See: https://github.com/VictoriaMetrics/VictoriaMetrics-enterprise/pull/880
func IsClientNetworkError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
if IsTimeoutNetworkError(err) {
return true
}
if errMsg := err.Error(); strings.Contains(errMsg, "broken pipe") || strings.Contains(errMsg, "reset by peer") {
return true
}
return false
}
// IsTimeoutNetworkError determines whether the provided error is a network error with a timeout.
func IsTimeoutNetworkError(err error) bool {
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
return true
}
return false
}
func genericServer(c net.Conn, compressionLevel int, readHelloMessage func(c net.Conn) error) (*BufferedConn, error) {
if err := c.SetDeadline(time.Now().Add(*rpcHandshakeTimeout)); err != nil {
return nil, fmt.Errorf("cannot set deadline: %w", err)
}
if err := readHelloMessage(c); err != nil {
return nil, fmt.Errorf("cannot read hello message : %w", err)
}
if err := writeMessage(c, successResponse); err != nil {
return nil, fmt.Errorf("cannot write success response on isCompressed: %w", err)
}
isRemoteCompressed, err := readIsCompressed(c)
if err != nil {
return nil, fmt.Errorf("cannot read isCompressed flag: %w", err)
}
if err := writeMessage(c, successResponse); err != nil {
return nil, fmt.Errorf("cannot write success response on isCompressed: %w", err)
}
if err := writeIsCompressed(c, compressionLevel > 0); err != nil {
return nil, fmt.Errorf("cannot write isCompressed flag: %w", err)
}
if err := readMessage(c, successResponse); err != nil {
return nil, fmt.Errorf("cannot read success response on isCompressed: %w", err)
}
if err := c.SetDeadline(time.Time{}); err != nil {
return nil, fmt.Errorf("cannot reset deadline: %w", err)
}
bc := newBufferedConn(c, compressionLevel, isRemoteCompressed)
return bc, nil
}
func genericClient(c net.Conn, msg string, compressionLevel int) (*BufferedConn, error) {
if err := c.SetDeadline(time.Now().Add(*rpcHandshakeTimeout)); err != nil {
return nil, fmt.Errorf("cannot set deadline: %w", err)
}
if err := writeMessage(c, msg); err != nil {
return nil, fmt.Errorf("cannot write hello: %w", err)
}
if err := readMessage(c, successResponse); err != nil {
return nil, fmt.Errorf("cannot read success response after sending hello: %w", err)
}
if err := writeIsCompressed(c, compressionLevel > 0); err != nil {
return nil, fmt.Errorf("cannot write isCompressed flag: %w", err)
}
if err := readMessage(c, successResponse); err != nil {
return nil, fmt.Errorf("cannot read success response on isCompressed: %w", err)
}
isRemoteCompressed, err := readIsCompressed(c)
if err != nil {
return nil, fmt.Errorf("cannot read isCompressed flag: %w", err)
}
if err := writeMessage(c, successResponse); err != nil {
return nil, fmt.Errorf("cannot write success response on isCompressed: %w", err)
}
if err := c.SetDeadline(time.Time{}); err != nil {
return nil, fmt.Errorf("cannot reset deadline: %w", err)
}
bc := newBufferedConn(c, compressionLevel, isRemoteCompressed)
return bc, nil
}
func writeIsCompressed(c net.Conn, isCompressed bool) error {
var buf [1]byte
if isCompressed {
buf[0] = 1
}
return writeMessage(c, string(buf[:]))
}
func readIsCompressed(c net.Conn) (bool, error) {
buf, err := readData(c, 1)
if err != nil {
return false, err
}
isCompressed := buf[0] != 0
return isCompressed, nil
}
func writeMessage(c net.Conn, msg string) error {
if _, err := io.WriteString(c, msg); err != nil {
return fmt.Errorf("cannot write %q to server: %w", msg, err)
}
if fc, ok := c.(flusher); ok {
if err := fc.Flush(); err != nil {
return fmt.Errorf("cannot flush %q to server: %w", msg, err)
}
}
return nil
}
type flusher interface {
Flush() error
}
func readMessage(c net.Conn, msg string) error {
buf, err := readData(c, len(msg))
if err != nil {
return err
}
if string(buf) != msg {
return fmt.Errorf("unexpected message obtained; got %q; want %q", buf, msg)
}
return nil
}
func readData(c net.Conn, dataLen int) ([]byte, error) {
data := make([]byte, dataLen)
if n, err := io.ReadFull(c, data); err != nil {
return nil, fmt.Errorf("cannot read message with size %d: %w; read only %d bytes", dataLen, err, n)
}
return data, nil
}

View File

@@ -1,83 +0,0 @@
package handshake
import (
"fmt"
"net"
"testing"
"time"
)
func TestVMInsertHandshake(t *testing.T) {
testHandshake(t, vminsertClient, VMInsertServer)
}
func TestVMSelectHandshake(t *testing.T) {
testHandshake(t, VMSelectClient, VMSelectServer)
}
func TestVMSelectServerTCPHealthcheck(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("cannot start listener: %s", err)
}
c, err := net.Dial("tcp", ln.Addr().String())
if err != nil {
t.Fatalf("cannot dial: %s", err)
}
if err := c.Close(); err != nil {
t.Fatalf("cannot close client conn: %s", err)
}
s, err := ln.Accept()
if err != nil {
t.Fatalf("cannot accept conn: %s", err)
}
if _, err := VMSelectServer(s, 0); !IsTCPHealthcheck(err) {
t.Fatalf("unexpected error; got %v; want TCP healthcheck error", err)
}
}
func testHandshake(t *testing.T, clientFunc, serverFunc Func) {
t.Helper()
c, s := net.Pipe()
ch := make(chan error, 1)
go func() {
bcs, err := serverFunc(s, 3)
if err != nil {
ch <- fmt.Errorf("error on outer handshake: %w", err)
return
}
bcc, err := clientFunc(bcs, 3)
if err != nil {
ch <- fmt.Errorf("error on inner handshake: %w", err)
return
}
if bcc == nil {
ch <- fmt.Errorf("expecting non-nil conn")
return
}
ch <- nil
}()
bcc, err := clientFunc(c, 0)
if err != nil {
t.Fatalf("error on outer handshake: %s", err)
}
bcs, err := serverFunc(bcc, 0)
if err != nil {
t.Fatalf("error on inner handshake: %s", err)
}
if bcs == nil {
t.Fatalf("expecting non-nil conn")
}
select {
case <-time.After(5 * time.Second):
t.Fatalf("timeout")
case err := <-ch:
if err != nil {
t.Fatalf("unexpected error on the server side: %s", err)
}
}
}

View File

@@ -58,13 +58,10 @@ var (
disableKeepAlive = flag.Bool("http.disableKeepAlive", false, "Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr")
disableResponseCompression = flag.Bool("http.disableResponseCompression", false, "Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth")
maxGracefulShutdownDuration = flag.Duration("http.maxGracefulShutdownDuration", 7*time.Second, "The maximum duration for a graceful shutdown of the HTTP server. "+
"During this period the server stops accepting new connections, but it will continue serving existing connections. "+
"The remaining in-flight requests are canceled before the deadline, so the shutdown can finish within this duration. "+
"A highly loaded server may require increased value for a graceful shutdown")
shutdownDelay = flag.Duration("http.shutdownDelay", 0, `Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers`)
idleConnTimeout = flag.Duration("http.idleConnTimeout", time.Minute, "Timeout for incoming idle http connections")
connTimeout = flag.Duration("http.connTimeout", 2*time.Minute, "Incoming connections to -httpListenAddr are closed after the configured timeout. "+
maxGracefulShutdownDuration = flag.Duration("http.maxGracefulShutdownDuration", 7*time.Second, `The maximum duration for a graceful shutdown of the HTTP server. A highly loaded server may require increased value for a graceful shutdown`)
shutdownDelay = flag.Duration("http.shutdownDelay", 0, `Optional delay before http server shutdown. During this delay, the server returns non-OK responses from /health page, so load balancers can route new requests to other servers`)
idleConnTimeout = flag.Duration("http.idleConnTimeout", time.Minute, "Timeout for incoming idle http connections")
connTimeout = flag.Duration("http.connTimeout", 2*time.Minute, "Incoming connections to -httpListenAddr are closed after the configured timeout. "+
"This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections")
headerHSTS = flag.String("http.header.hsts", "", "Value for 'Strict-Transport-Security' header, recommended: 'max-age=31536000; includeSubDomains'")
@@ -83,7 +80,6 @@ var (
type server struct {
shutdownDelayDeadline atomic.Int64
s *http.Server
cancel context.CancelFunc
}
// RequestHandler must serve the given request r and write response to w.
@@ -160,11 +156,7 @@ func serve(addr string, rh RequestHandler, idx int, opts ServeOptions) {
func serveWithListener(addr string, ln net.Listener, rh RequestHandler, disableBuiltinRoutes bool) {
var s server
ctx, cancel := context.WithCancel(context.Background())
s.s = &http.Server{
BaseContext: func(l net.Listener) context.Context {
return ctx
},
// Disable http/2, since it doesn't give any advantages for VictoriaMetrics services.
TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
@@ -178,7 +170,6 @@ func serveWithListener(addr string, ln net.Listener, rh RequestHandler, disableB
ErrorLog: log.New(&tlsErrorSkipLogger{}, "", 0),
}
s.s.SetKeepAlivesEnabled(!*disableKeepAlive)
s.cancel = cancel
if *connTimeout > 0 {
s.s.ConnContext = func(ctx context.Context, _ net.Conn) context.Context {
timeoutSec := connTimeout.Seconds()
@@ -274,18 +265,8 @@ func stop(addr string) error {
logger.Infof("Starting shutdown for http server %q", addr)
}
// Cancel in-flight requests shortly before the deadline, reserving up to 2s (or 20%
// of the window, whichever is smaller) for them to unwind, so Shutdown returns cleanly
// within -http.maxGracefulShutdownDuration instead of timing out and dying via
// logger.Fatalf -> os.Exit, which skips the storage flush and loses data.
// See https://github.com/VictoriaMetrics/VictoriaLogs/issues/1502
cancelInflightAfter := *maxGracefulShutdownDuration - min(*maxGracefulShutdownDuration/5, 2*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), *maxGracefulShutdownDuration)
defer cancel()
t := time.AfterFunc(cancelInflightAfter, s.cancel)
defer t.Stop()
if err := s.s.Shutdown(ctx); err != nil {
return fmt.Errorf("cannot gracefully shutdown http server at %q in %.3fs; "+
"probably, `-http.maxGracefulShutdownDuration` command-line flag value must be increased; error: %s", addr, maxGracefulShutdownDuration.Seconds(), err)

View File

@@ -105,10 +105,6 @@ type body struct {
Scope string `json:"scope,omitempty"`
vmAccessClaim VMAccessClaim
// hasVMAccess is set to true when the token body contains a `vm_access` claim.
// Presence enforcement is left to the caller via Token.HasVMAccess.
hasVMAccess bool
buf []byte
p *fastjson.Parser
@@ -125,6 +121,7 @@ type body struct {
}
func (b *body) parse(src string) error {
var err error
b.buf, err = decodeB64(b.buf[:0], src)
if err != nil {
@@ -135,9 +132,6 @@ func (b *body) parse(src string) error {
if err != nil {
return err
}
if jv.Type() != fastjson.TypeObject {
return fmt.Errorf("unexpected non json object; type: %q", jv.Type())
}
if expObject := jv.Get("exp"); expObject != nil {
b.Exp, err = expObject.Int64()
if err != nil {
@@ -159,31 +153,30 @@ func (b *body) parse(src string) error {
}
vaObject := jv.Get("vm_access")
switch {
case vaObject == nil || vaObject.Type() == fastjson.TypeNull:
b.hasVMAccess = false
default:
// some IDPs encode custom claims as a string
// try parsing as an object and fallback to a string
switch vaObject.Type() {
case fastjson.TypeObject:
if err := b.vmAccessClaim.parseFrom(vaObject); err != nil {
return err
}
case fastjson.TypeString:
b.claimsParser = parserPool.Get()
va, err := b.claimsParser.ParseBytes(vaObject.GetStringBytes())
if err != nil {
return fmt.Errorf("cannot parse `vm_access` string json: %w", err)
}
if err := b.vmAccessClaim.parseFrom(va); err != nil {
return fmt.Errorf("cannot parse `vm_access` values from string json: %w", err)
}
b.vmAccessClaimObject = va
default:
return fmt.Errorf("unexpected type for `vm_access` field; got: %q, want object {}", vaObject.Type())
if vaObject == nil {
return ErrVMAccessFieldMissing
}
// some IDPs encode custom claims as a string
// try parsing as an object and fallback to a string
switch vaObject.Type() {
case fastjson.TypeObject:
if err := b.vmAccessClaim.parseFrom(vaObject); err != nil {
return err
}
b.hasVMAccess = true
case fastjson.TypeString:
b.claimsParser = parserPool.Get()
va, err := b.claimsParser.ParseBytes(vaObject.GetStringBytes())
if err != nil {
return fmt.Errorf("cannot parse `vm_access` string json: %w", err)
}
if err := b.vmAccessClaim.parseFrom(va); err != nil {
return fmt.Errorf("cannot parse `vm_access` values from string json: %w", err)
}
b.vmAccessClaimObject = va
case fastjson.TypeNull:
return ErrVMAccessFieldMissing
default:
return fmt.Errorf("unexpected type for `vm_access` field; got: %q, want object {}", vaObject.Type())
}
b.Jti = bytesutil.ToUnsafeString(jv.GetStringBytes("jti"))
@@ -225,7 +218,6 @@ func (b *body) reset() {
b.buf = b.buf[:0]
b.allClaims = nil
b.vmAccessClaim.reset()
b.hasVMAccess = false
if b.p != nil {
parserPool.Put(b.p)
b.p = nil
@@ -237,9 +229,11 @@ func (b *body) reset() {
if b.vmAccessClaimObject != nil {
b.vmAccessClaimObject = nil
}
}
// Parse parses JWT token from given source string
//
// Token field is valid until src is reachable
func (t *Token) Parse(src string, enforceAuthPrefix bool) error {
if enforceAuthPrefix && (len(src) < len(prefix) || !strings.EqualFold(src[:len(prefix)], prefix)) {
@@ -274,11 +268,6 @@ func (t *Token) Parse(src string, enforceAuthPrefix bool) error {
return nil
}
// HasVMAccessClaim reports whether the parsed token contains a `vm_access` claim.
func (t *Token) HasVMAccessClaim() bool {
return t.body.hasVMAccess
}
// Issuer returns `iss` claim value from token body
func (t *Token) Issuer() string {
return t.body.Iss
@@ -382,30 +371,30 @@ func (t *Token) Reset() {
// VMAccessClaim represent JWT claim object
type VMAccessClaim struct {
MetricsExtraFilters []string `json:"metrics_extra_filters,omitempty" yaml:"metrics_extra_filters,omitempty"`
MetricsExtraLabels []string `json:"metrics_extra_labels,omitempty" yaml:"metrics_extra_labels,omitempty"`
LogsExtraFilters []string `json:"logs_extra_filters,omitempty" yaml:"logs_extra_filters,omitempty"`
LogsExtraStreamFilters []string `json:"logs_extra_stream_filters,omitempty" yaml:"logs_extra_stream_filters,omitempty"`
MetricsExtraFilters []string `json:"metrics_extra_filters,omitempty"`
MetricsExtraLabels []string `json:"metrics_extra_labels,omitempty"`
LogsExtraFilters []string `json:"logs_extra_filters,omitempty"`
LogsExtraStreamFilters []string `json:"logs_extra_stream_filters,omitempty"`
MetricsAccountID uint32 `json:"metrics_account_id,omitempty" yaml:"metrics_account_id,omitempty"`
MetricsProjectID uint32 `json:"metrics_project_id,omitempty" yaml:"metrics_project_id,omitempty"`
MetricsAccountID uint32 `json:"metrics_account_id,omitempty"`
MetricsProjectID uint32 `json:"metrics_project_id,omitempty"`
LogsAccountID uint32 `json:"logs_account_id,omitempty" yaml:"logs_account_id,omitempty"`
LogsProjectID uint32 `json:"logs_project_id,omitempty" yaml:"logs_project_id,omitempty"`
LogsAccountID uint32 `json:"logs_account_id,omitempty"`
LogsProjectID uint32 `json:"logs_project_id,omitempty"`
// Properties below are deprecated and retained only for compatibility with vmgateway, which is itself deprecated.
// promql filters applied to each select query
// Deprecated
ExtraFilters []string `json:"extra_filters,omitempty" yaml:"-"`
ExtraFilters []string `json:"extra_filters,omitempty"`
// Deprecated
Tenant TenantID `json:"tenant_id" yaml:"-"`
Tenant TenantID `json:"tenant_id"`
// role can be denied as 1 = read, 2 = write, 3 = read and write
// 0 = unconfigured - read and write
// Deprecated
Mode int `json:"mode,omitempty" yaml:"-"`
Mode int `json:"mode,omitempty"`
// Deprecated
Labels []string `json:"extra_labels,omitempty" yaml:"-"`
Labels []string `json:"extra_labels,omitempty"`
// labelsBuf holds allocated memory for Labels
// Deprecated
labelsBuf []byte
@@ -436,6 +425,7 @@ func (vac *VMAccessClaim) reset() {
}
func (vac *VMAccessClaim) parseFrom(jv *fastjson.Value) error {
if err := vac.Tenant.parseFrom(jv); err != nil {
return err
}
@@ -579,9 +569,6 @@ func NewToken(auth string, enforceAuthPrefix bool) (*Token, error) {
if err := t.parse(jwt[0], jwt[1], jwt[2]); err != nil {
return nil, err
}
if !t.body.hasVMAccess {
return nil, ErrVMAccessFieldMissing
}
return &t, nil
}

View File

@@ -168,10 +168,17 @@ func TestParseJWTBody_Failure(t *testing.T) {
true,
)
// non-object body type
// invalid body type json
f(
`[]`,
`unexpected non json object; type: "array"`,
"missing `vm_access` claim",
true,
)
// missing vm_access claim
f(
`{}`,
"missing `vm_access` claim",
true,
)
@@ -182,6 +189,13 @@ func TestParseJWTBody_Failure(t *testing.T) {
true,
)
// vm_access claim null
f(
`{"vm_access": null}`,
"missing `vm_access` claim",
true,
)
// invalid vm_access: account_id type mismatch
f(
`{"vm_access": {"tenant_id": {"account_id": "1", "project_id": 5}}}`,
@@ -541,33 +555,6 @@ func TestParseJWTBody_Success(t *testing.T) {
)
}
func TestParseJWTBody_VMAccessPresence(t *testing.T) {
f := func(data string, wantHasVMAccess bool) {
t.Helper()
encodedLen := base64.RawURLEncoding.EncodedLen(len(data))
encoded := make([]byte, encodedLen)
base64.RawURLEncoding.Encode(encoded, []byte(data))
var b body
if err := b.parse(string(encoded)); err != nil {
t.Fatalf("unexpected error: %s", err)
}
if b.hasVMAccess != wantHasVMAccess {
t.Fatalf("unexpected hasVMAccess; got %v; want %v", b.hasVMAccess, wantHasVMAccess)
}
}
// vm_access claim is present
f(`{"vm_access": {}}`, true)
f(`{"vm_access": {"metrics_account_id": 1}}`, true)
// vm_access claim is absent or null - parsing must succeed with hasVMAccess=false
f(`{}`, false)
f(`{"vm_access": null}`, false)
f(`{"role": "admin"}`, false)
}
func TestNewTokenFromRequest_Failure(t *testing.T) {
f := func(r *http.Request) {
t.Helper()
@@ -879,6 +866,7 @@ func TestNewTokenFromRequest_Success(t *testing.T) {
}
func TestTokenMatchClaims(t *testing.T) {
/*
{
"iss": "https://login.microsoftonline.com/-6691-4868-a77b-1b0f9bbe5f43/v2.0",

View File

@@ -1,311 +0,0 @@
package mdx
import (
"flag"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
var (
vmLabel = flag.String("mdx.label", "", "Optional label value in the form 'name=value' used to identify VictoriaMetrics metrics for MDX. "+
"Metrics containing the specified label are forwarded to `-remoteWrite.url` endpoints configured with `-remoteWrite.mdx.enable=true`.")
)
const (
vmAppLabelName = "victoriametrics_app"
vmAppLabelValue = "true"
vmAppVersionMetricName = "vm_app_version"
)
// Ctx defines filtering context
type Ctx struct {
// labels hold modified timeseries labels
// valid until PutContext call
labels []prompb.Label
buf []byte
hasVMAppLabel bool
hasVMAppVersionLabel bool
hasFilterLabelValue bool
jobLabelValue string
instanceLabelValue string
}
func (ctx *Ctx) reset() {
// do not reset labels intentionally
// it must live until PutContext call
ctx.buf = ctx.buf[:0]
ctx.hasVMAppLabel = false
ctx.hasVMAppVersionLabel = false
ctx.hasFilterLabelValue = false
ctx.jobLabelValue = ""
ctx.instanceLabelValue = ""
}
var ctxPool = &sync.Pool{
New: func() any {
return &Ctx{}
},
}
// GetContext returns filtering context
func GetContext() *Ctx {
return ctxPool.Get().(*Ctx)
}
// PutContext resets context
func PutContext(ctx *Ctx) {
clear(ctx.labels)
ctx.labels = ctx.labels[:0]
ctx.reset()
ctxPool.Put(ctx)
}
// Filter manages the list of VictoriaMetrics instances grouped by job:instance labels.
// job and instance must present at timeseries.
//
// Filter keeps timeseries with any of the following conditions:
// * vm_app_version present
// * victoriametrics_app=true label present at timeseries
// * if labels has label value defined with flag `-mdx.label`
//
// Filter track entries with TTL of 1 hour
type Filter struct {
tracker *instanceTracker
filterByLabelName string
label string
}
// NewFilter returns new Filter instance
func NewFilter() *Filter {
filter := &Filter{
tracker: newInstanceTracker(),
}
if len(*vmLabel) > 0 {
n := strings.IndexByte(*vmLabel, '=')
if n < 0 {
logger.Fatalf("missing '=' in `-mdx.label`. It must contain label in the form `name=value`; got %q", *vmLabel)
}
filter.filterByLabelName = (*vmLabel)[:n]
filter.label = (*vmLabel)[n+1:]
if len(filter.filterByLabelName) == 0 || len(filter.label) == 0 {
logger.Fatalf("label name and value cannot be empty in `-mdx.label`. It must contain label in the form `name=value`; got %q", *vmLabel)
}
}
return filter
}
// VMInstancesCount returns amount of currently tracked instances
func (filter *Filter) VMInstancesCount() int {
return filter.tracker.len()
}
// MustStop stops filter instance
func (filter *Filter) MustStop() {
filter.tracker.mustStop()
}
// Filter filters provided timeseries with given context.
//
// Returned timeseries is valid as long as Ctx is valid
func (filter *Filter) Filter(ctx *Ctx, tss []prompb.TimeSeries) []prompb.TimeSeries {
dstTss := tss[:0]
for _, ts := range tss {
ctx.prepare(ts.Labels, filter.filterByLabelName, filter.label)
key := ctx.formatTimeSeriesKey()
if len(key) == 0 {
// metrics with empty job or instance labels must be always dropped
// despite any other conditions
continue
}
if ctx.hasVMAppLabel {
filter.trackInstance(key)
dstTss = append(dstTss, ts)
continue
}
if ctx.hasFilterLabelValue || ctx.hasVMAppVersionLabel {
ts.Labels = ctx.addVMAppLabel(ts.Labels)
filter.trackInstance(key)
dstTss = append(dstTss, ts)
continue
}
ok := filter.tracker.has(key)
if ok {
ts.Labels = ctx.addVMAppLabel(ts.Labels)
dstTss = append(dstTss, ts)
}
}
return dstTss
}
func (filter *Filter) trackInstance(key string) {
if filter.tracker.has(key) {
return
}
key = strings.Clone(key)
filter.tracker.register(key)
}
func (ctx *Ctx) prepare(labels []prompb.Label, filterByLabelName, label string) {
ctx.reset()
// always use the last label=value pair
// because in case of possible label duplicates,
// the last added label must win
for _, l := range labels {
switch l.Name {
case "job":
ctx.jobLabelValue = l.Value
case "instance":
ctx.instanceLabelValue = l.Value
case vmAppLabelName:
if l.Value == vmAppLabelValue {
ctx.hasVMAppLabel = true
}
case "__name__":
if l.Value == vmAppVersionMetricName {
ctx.hasVMAppVersionLabel = true
}
}
if len(filterByLabelName) > 0 {
if l.Name == filterByLabelName && l.Value == label {
ctx.hasFilterLabelValue = true
}
}
}
}
// formatTimeSeriesKey returns timeseries key after ctx.prepare call
// if it catched job and instances labels
//
// returned string is valid until next ctx.prepare
func (ctx *Ctx) formatTimeSeriesKey() string {
if len(ctx.jobLabelValue) == 0 || len(ctx.instanceLabelValue) == 0 {
return ""
}
buf := ctx.buf[:0]
buf = strconv.AppendQuote(buf, ctx.jobLabelValue)
buf = append(buf, ':')
buf = strconv.AppendQuote(buf, ctx.instanceLabelValue)
ctx.buf = buf
return bytesutil.ToUnsafeString(buf)
}
func (ctx *Ctx) addVMAppLabel(labels []prompb.Label) []prompb.Label {
// unconditionally add vmAppLabelValue at the end of labels list
// it will overwrite any exist vmAppLabelName labels with a value different to vmAppLabelValue
// it's guaranteed by VictoriaMetrics ingestion contract
poolLabels := ctx.labels
poolLabelsLen := len(poolLabels)
poolLabels = append(poolLabels, labels...)
poolLabels = append(poolLabels, prompb.Label{Name: vmAppLabelName, Value: vmAppLabelValue})
ctx.labels = poolLabels
return poolLabels[poolLabelsLen:len(poolLabels):len(poolLabels)]
}
type instanceTracker struct {
mu sync.RWMutex
lastAccessByKey map[string]*atomic.Uint64
wg sync.WaitGroup
stop chan struct{}
}
func newInstanceTracker() *instanceTracker {
c := &instanceTracker{
lastAccessByKey: make(map[string]*atomic.Uint64),
stop: make(chan struct{}),
}
c.wg.Add(1)
go c.startStaleWatcher()
return c
}
func (it *instanceTracker) len() int {
it.mu.RLock()
s := len(it.lastAccessByKey)
it.mu.RUnlock()
return s
}
func (it *instanceTracker) has(key string) bool {
it.mu.RLock()
lat, ok := it.lastAccessByKey[key]
it.mu.RUnlock()
if ok {
lat.Store(fasttime.UnixTimestamp())
}
return ok
}
func (it *instanceTracker) register(key string) {
it.mu.Lock()
// key could be registered by concurrent goroutine
lat, ok := it.lastAccessByKey[key]
if !ok {
lat = &atomic.Uint64{}
it.lastAccessByKey[key] = lat
}
it.mu.Unlock()
lat.Store(fasttime.UnixTimestamp())
}
func (it *instanceTracker) startStaleWatcher() {
defer it.wg.Done()
t := time.NewTicker(time.Minute)
defer t.Stop()
for {
select {
case <-it.stop:
return
case <-t.C:
it.cleanStale()
}
}
}
var entryTTLSeconds = uint64(time.Hour.Seconds())
func (it *instanceTracker) cleanStale() {
ct := fasttime.UnixTimestamp()
var toDelete map[string]*atomic.Uint64
it.mu.RLock()
for key, lastAccessTime := range it.lastAccessByKey {
accessedAt := lastAccessTime.Load()
if ct > accessedAt+entryTTLSeconds {
if toDelete == nil {
toDelete = make(map[string]*atomic.Uint64)
}
toDelete[key] = lastAccessTime
}
}
it.mu.RUnlock()
if len(toDelete) > 0 {
it.mu.Lock()
for key, lastAccessTime := range toDelete {
accessedAt := lastAccessTime.Load()
// concurrent goroutine may refresh lastAccessTime
if ct > accessedAt+entryTTLSeconds {
delete(it.lastAccessByKey, key)
}
}
it.mu.Unlock()
}
}
func (it *instanceTracker) mustStop() {
close(it.stop)
it.wg.Wait()
}

View File

@@ -1,104 +0,0 @@
//go:build synctest
package mdx
import (
"testing"
"testing/synctest"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
func TestMdxInstanceCleanup(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
filter := NewFilter()
defer filter.MustStop()
assertFilterLen := func(expectedLen int) {
t.Helper()
if filter.VMInstancesCount() != expectedLen {
t.Fatalf("unexpected instance map length; got %d; want %d", filter.VMInstancesCount(), expectedLen)
}
}
ctx := GetContext()
filter.Filter(ctx, []prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_up"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "go_gc_duration_seconds"},
{Name: "instance", Value: "node-exporter1"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "http_request_duration_seconds"},
{Name: "instance", Value: "service1"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "vmagent1:8429"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "scrape_targets_up"},
{Name: "instance", Value: "vmagent1:8429"},
{Name: "job", Value: "test"},
},
},
},
)
PutContext(ctx)
time.Sleep(1 * time.Minute)
// the entries should not be cleaned.
assertFilterLen(2)
time.Sleep(58 * time.Minute)
// receive samples from victoria-metrics1:8428 after 59 minutes.
// so the entry will be refreshed.
ctx = GetContext()
filter.Filter(ctx, []prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_up"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
},
)
PutContext(ctx)
assertFilterLen(2)
// entry for job:instance - test:vmagent1:8429 must be removed
time.Sleep(4 * time.Minute)
assertFilterLen(1)
// no samples from vmagent1:8429 in the last hour, so it should be removed from the mdx instance list.
time.Sleep(2 * time.Hour)
assertFilterLen(0)
})
}

View File

@@ -1,435 +0,0 @@
package mdx
import (
"fmt"
"sync"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
func TestMdxInstanceFilter(t *testing.T) {
originalVmLabel := *vmLabel
*vmLabel = "service=victoriametrics"
t.Cleanup(func() {
*vmLabel = originalVmLabel
})
f := func(input []prompb.TimeSeries, expectedOutput []prompb.TimeSeries) {
t.Helper()
filter := NewFilter()
defer filter.MustStop()
ctx := GetContext()
defer PutContext(ctx)
inputCopy := append([]prompb.TimeSeries{}, input...)
output := filter.Filter(ctx, inputCopy)
if diff := cmp.Diff(expectedOutput, output); len(diff) > 0 {
t.Fatalf("unexpected result (-want, +got):\n%s", diff)
}
// make sure that result is the same over multiple calls
inputCopy = append([]prompb.TimeSeries{}, input...)
output = filter.Filter(ctx, inputCopy)
if diff := cmp.Diff(expectedOutput, output); len(diff) > 0 {
t.Fatalf("unexpected result (-want, +got):\n%s", diff)
}
}
// metrics with vm_app_version and different order of labels.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics2:8428"},
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics3:8428"},
{Name: "__name__", Value: "vm_app_version"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics2:8428"},
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics3:8428"},
{Name: "__name__", Value: "vm_app_version"},
{Name: "victoriametrics_app", Value: "true"},
},
}},
)
// metrics without vm_app_version but with service=victoriametrics that is specified in `-mdx.label`.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "true"},
},
},
})
// metrics without vm_app_version but with service=victoriametrics that is specified in `-mdx.label`.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
},
}},
[]prompb.TimeSeries{
// 2.
// metrics without vm_app_version but with service=victoriametrics that is specified in `-mdx.label`.
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "true"},
},
}})
// metrics with vm_app_version and service=victoriametrics should be preserved.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "job", Value: "test"},
{Name: "service", Value: "victoriametrics"},
{Name: "__name__", Value: "vm_app_version"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "job", Value: "test"},
{Name: "service", Value: "victoriametrics"},
{Name: "__name__", Value: "vm_app_version"},
{Name: "victoriametrics_app", Value: "true"},
},
},
},
)
// metrics without vm_app_version and `service=victoriametrics` but with `victoriametrics_app=true`, which should be preserved.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics6:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "victoriametrics_app", Value: "true"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics6:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "vm_slow_queries_total"},
{Name: "victoriametrics_app", Value: "true"},
},
},
})
// metrics without vm_app_version and service=victoriametrics and `victoriametrics_app=true`, which should be filtered out.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "go_gc_duration_seconds"},
{Name: "instance", Value: "node-exporter1"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "http_request_duration_seconds"},
{Name: "instance", Value: "service1"},
{Name: "job", Value: "test"},
},
},
},
[]prompb.TimeSeries{},
)
// metrics with vm_app_version but job or instance is empty (or missing), they should be dropped.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: ""},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "vmagent2:8429"},
{Name: "job", Value: ""},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "vmagent2:8429"},
},
},
},
[]prompb.TimeSeries{})
// metrics without vm_app_version, but the instances were already registered with first timeseries
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_rows_inserted_total"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vminsert_request_duration_seconds_bucket"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_rows_inserted_total"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vminsert_request_duration_seconds_bucket"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
})
// metrics without vm_app_version, `service=victoriametrics` and `victoriametrics_app=true`, and the instance wasn't already registered in the previous call, so it will be dropped.
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vminsert_request_duration_seconds_bucket"},
{Name: "instance", Value: "victoria-metrics7:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "service", Value: "victoriametrics"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "job", Value: "test"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "service", Value: "victoriametrics"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
}},
)
// metrics with duplicate victoriametrics_app label
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "other_value"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "other_value"},
{Name: "victoriametrics_app", Value: "true"},
},
},
})
// metrics with duplicate job and instance labels
// last value wins
f([]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "job", Value: "test2"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "service", Value: "victoriametrics"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "http_requests_total"},
{Name: "job", Value: "test2"},
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "service", Value: "victoriametrics"},
},
},
},
[]prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "job", Value: "test2"},
{Name: "instance", Value: "victoria-metrics4:8428"},
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "__name__", Value: "http_requests_total"},
{Name: "job", Value: "test2"},
{Name: "instance", Value: "victoria-metrics5:8428"},
{Name: "service", Value: "victoriametrics"},
{Name: "victoriametrics_app", Value: "true"},
},
}})
}
func TestMdxInstanceFilterConcurrent(t *testing.T) {
originalVmLabel := *vmLabel
*vmLabel = "service=victoriametrics"
t.Cleanup(func() { *vmLabel = originalVmLabel })
filter := NewFilter()
defer filter.MustStop()
const concurrency = 8
const iterations = 200
generateSeries := func(g int) []prompb.TimeSeries {
return []prompb.TimeSeries{
{Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "instance", Value: fmt.Sprintf("vm-%d:8428", g)},
}},
// shared job:instance
{Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "job", Value: "test"},
{Name: "instance", Value: "vmagent:8428"},
}},
}
}
var wg sync.WaitGroup
for worker := range concurrency {
wg.Go(func() {
input := generateSeries(worker)
var expectedOutput []prompb.TimeSeries
for _, inputTs := range input {
labels := append([]prompb.Label{}, inputTs.Labels...)
labels = append(labels, prompb.Label{Name: vmAppLabelName, Value: vmAppLabelValue})
expectedOutput = append(expectedOutput, prompb.TimeSeries{Labels: labels})
}
for range iterations {
ctx := GetContext()
inputCopy := append([]prompb.TimeSeries{}, input...)
output := filter.Filter(ctx, inputCopy)
if diff := cmp.Diff(expectedOutput, output); len(diff) > 0 {
t.Errorf("unexpected result (-want, +got):\n%s", diff)
}
PutContext(ctx)
}
})
}
wg.Wait()
// goroutines + 1 shared
if got := filter.VMInstancesCount(); got != concurrency+1 {
t.Errorf("unexpected instance count: got %d, want %d", got, concurrency+1)
}
}

View File

@@ -1,85 +0,0 @@
package mdx
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
)
func BenchmarkFilter(b *testing.B) {
f := func(name string, input, want []prompb.TimeSeries) {
b.Helper()
b.Run(name, func(b *testing.B) {
filter := NewFilter()
defer filter.MustStop()
b.ReportAllocs()
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ctx := GetContext()
localInput := append([]prompb.TimeSeries{}, input...)
tss := filter.Filter(ctx, localInput)
if len(tss) != len(want) {
diff := cmp.Diff(want, tss)
b.Fatalf("unexpected result (-want, +got):\n%s", diff)
}
PutContext(ctx)
}
})
})
}
input := []prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "http_requests_total"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "http_requests_errors_total"},
},
},
}
expected := []prompb.TimeSeries{
{
Labels: []prompb.Label{
{Name: "__name__", Value: "vm_app_version"},
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "http_requests_total"},
{Name: "victoriametrics_app", Value: "true"},
},
},
{
Labels: []prompb.Label{
{Name: "instance", Value: "victoria-metrics1:8428"},
{Name: "job", Value: "test"},
{Name: "__name__", Value: "http_requests_errors_total"},
{Name: "victoriametrics_app", Value: "true"},
},
},
}
f("match vm_app_version", input, expected)
}

View File

@@ -112,19 +112,11 @@ func (m *TimeSeries) size() (n int) {
}
for _, e := range m.Labels {
l := e.size()
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
for _, e := range m.Samples {
l := e.size()
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
return n
}
@@ -134,18 +126,10 @@ func (m *Label) size() (n int) {
return 0
}
if l := len(m.Name); l > 0 {
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
if l := len(m.Value); l > 0 {
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
return n
}
@@ -176,11 +160,6 @@ func (m *WriteRequest) marshalToSizedBuffer(dst []byte) (int, error) {
}
func encodeVarint(dst []byte, offset int, v uint64) int {
if v < 1<<7 {
offset--
dst[offset] = byte(v)
return offset
}
offset -= sov(v)
base := offset
for v >= 1<<7 {
@@ -201,11 +180,7 @@ func (m *WriteRequest) size() (n int) {
}
for _, e := range m.Metadata {
l := e.size()
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
return n
}
@@ -263,25 +238,13 @@ func (m *MetricMetadata) size() (n int) {
n += 1 + sov(uint64(m.Type))
}
if l := len(m.MetricFamilyName); l > 0 {
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
if l := len(m.Help); l > 0 {
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
if l := len(m.Unit); l > 0 {
if l < 128 {
n += 2 + l
} else {
n += 1 + l + sov(uint64(l))
}
n += 1 + l + sov(uint64(l))
}
if m.AccountID != 0 {
n += 1 + sov(uint64(m.AccountID))

View File

@@ -9,47 +9,48 @@ import (
)
// WriteMetricRelabelDebug writes /metric-relabel-debug page to w with the corresponding args.
func WriteMetricRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, format string, err error) {
writeRelabelDebug(w, false, targetID, metric, relabelConfigs, format, err)
func WriteMetricRelabelDebug(w io.Writer, targetID, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, format string, err error) {
writeRelabelDebug(w, false, targetID, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, format, err)
}
// WriteTargetRelabelDebug writes /target-relabel-debug page to w with the corresponding args.
func WriteTargetRelabelDebug(w io.Writer, targetID, metric, relabelConfigs, format string, err error) {
writeRelabelDebug(w, true, targetID, metric, relabelConfigs, format, err)
writeRelabelDebug(w, true, targetID, metric, relabelConfigs, 0, 0, format, err)
}
func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, relabelConfigs, format string, err error) {
func writeRelabelDebug(w io.Writer, isTargetRelabel bool, targetID, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, format string, err error) {
if metric == "" {
metric = "{}"
}
targetURL := ""
if err != nil {
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
return
}
metric, err = normalizeInputLabels(metric)
if err != nil {
err = fmt.Errorf("cannot parse metric: %w", err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
return
}
labels, err := promutil.NewLabelsFromString(metric)
if err != nil {
err = fmt.Errorf("cannot parse metric: %w", err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
return
}
pcs, err := ParseRelabelConfigsData([]byte(relabelConfigs))
if err != nil {
err = fmt.Errorf("cannot parse relabel configs: %w", err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, err)
WriteRelabelDebugSteps(w, targetURL, targetID, format, nil, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err)
return
}
dss, targetURL := newDebugRelabelSteps(pcs, labels, isTargetRelabel)
WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, nil)
WriteRelabelDebugSteps(w, targetURL, targetID, format, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, nil)
}
func newDebugRelabelSteps(pcs *ParsedConfigs, labels *promutil.Labels, isTargetRelabel bool) ([]DebugStep, string) {

View File

@@ -6,15 +6,15 @@
{% stripspace %}
{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, err error) %}
{% func RelabelDebugSteps(targetURL, targetID, format string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) %}
{% if format == "json" %}
{%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, err) %}
{%= RelabelDebugStepsJSON(targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err) %}
{% else %}
{%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, err) %}
{%= RelabelDebugStepsHTML(targetURL, targetID, dss, metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, err) %}
{% endif %}
{% endfunc %}
{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
{% func RelabelDebugStepsHTML(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) %}
<!DOCTYPE html>
<html lang="en">
<head>
@@ -29,6 +29,28 @@ function submitRelabelDebugForm(e) {
}
form.method = method;
}
function initRelabelConfigsHighlight() {
var ta = document.getElementById('relabel-configs-input');
var bd = document.getElementById('relabel-configs-backdrop');
if (!ta || !bd) return;
function escapeHtml(s) {
return s.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
}
function highlight(text) {
return text.split('\n').map(function(line) {
var e = escapeHtml(line);
return /^\s*#/.test(line)
? '<span style="color:#999">'+e+'</span>'
: '<span style="color:#212529">'+e+'</span>';
}).join('\n');
}
function update() { bd.innerHTML = highlight(ta.value)+'\n'; }
ta.addEventListener('input', update);
ta.addEventListener('scroll', function() { bd.scrollTop = ta.scrollTop; });
update();
}
document.addEventListener('DOMContentLoaded', initRelabelConfigsHighlight);
</script>
</head>
<body>
@@ -51,7 +73,7 @@ function submitRelabelDebugForm(e) {
<div class="m-3">
<form method="POST" onsubmit="submitRelabelDebugForm(event)">
{%= relabelDebugFormInputs(metric, relabelConfigs) %}
{%= relabelDebugFormInputs(metric, relabelConfigs, urlRelabelIndexLength, urlRelabelIndexCurrent, isTargetRelabel, targetID) %}
{% if targetID != "" %}
<input type="hidden" name="id" value="{%s targetID %}" />
{% endif %}
@@ -72,12 +94,39 @@ function submitRelabelDebugForm(e) {
</html>
{% endfunc %}
{% func relabelDebugFormInputs(metric, relabelConfigs string) %}
{% func relabelDebugFormInputs(metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, targetID string) %}
<div>
Relabel configs:<br/>
<textarea name="relabel_configs" style="width: 100%; height: 15em; font-family: monospace" class="m-1">{%s relabelConfigs %}</textarea>
</div>
<!-- show remote write relabel reload only for scrape metric relabel debug. discovery debug and pure relabel debug should not display this section -->
{% if !isTargetRelabel && targetID != "" %}
<div>
<div class="m-1">
<div class="d-flex align-items-center gap-2 mt-1">
{% if urlRelabelIndexLength > 0 %}
<select name="url_relabel_configs_index" class="form-select form-select-sm w-auto">
{% for i := range urlRelabelIndexLength %}
{% if urlRelabelIndexCurrent == i %}
<option value="{%d i %}" selected="selected">remote-write-url-{%d i %}</option>
{% else %}
<option value="{%d i %}">remote-write-url-{%d i %}</option>
{% endif %}
{% endfor %}
</select>
<input type="submit" name="reload_url_relabel_configs" value="Reload" class="btn btn-secondary btn-sm" onclick="return confirm('Reload will discard all modifications to the current configuration. Continue?')" />
{% endif %}
</div>
</div>
</div>
{% endif %}
<!-- the following text area css was generated with the help of AI to display yaml comments (starting with #) in gray. it could be rewritten in the future -->
<div class="m-1" style="position:relative;height:15em;">
<div id="relabel-configs-backdrop" style="position:absolute;top:0;left:0;right:0;bottom:0;pointer-events:none;overflow:hidden;font-family:monospace;white-space:pre-wrap;padding:0.375rem 0.75rem;border:1px solid transparent;"></div>
<textarea id="relabel-configs-input" name="relabel_configs" class="form-control" style="position:absolute;top:0;left:0;height:100%;font-family:monospace;color:transparent;caret-color:#212529;background:transparent;resize:none;overflow-y:scroll;">
{%s relabelConfigs %}
</textarea>
</div>
</div>
<div>
Labels:<br/>
<textarea name="metric" style="width: 100%; height: 5em; font-family: monospace" class="m-1">{%s metric %}</textarea>
@@ -153,7 +202,7 @@ function submitRelabelDebugForm(e) {
{% endif %}
{% endfunc %}
{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, err error) %}
{% func RelabelDebugStepsJSON(targetURL, targetID string, dss []DebugStep, metric, relabelConfigs string, urlRelabelIndexLength, urlRelabelIndexCurrent int, isTargetRelabel bool, err error) %}
{
{% if err != nil %}
"status": "error",

File diff suppressed because it is too large Load Diff

View File

@@ -10,10 +10,10 @@ import (
// TestWriteRelabelDebugSupportFormats verifies the relabeling debug input, rules and output.
func TestWriteRelabelDebugSupportFormats(t *testing.T) {
f := func(input, rule, expect string) {
f := func(input, rules, expect string) {
// execute
outputWriter := bytes.NewBuffer(nil)
writeRelabelDebug(outputWriter, false, "", input, rule, "json", nil)
writeRelabelDebug(outputWriter, false, "", input, rules, 0, 0, "json", nil)
// the response is in JSON with HTML content, extract the `resultingLabels` in JSON and unescape it.
resultingLabels := fastjson.GetString(outputWriter.Bytes(), `resultingLabels`)
@@ -41,4 +41,20 @@ func TestWriteRelabelDebugSupportFormats(t *testing.T) {
f(`{_name__="metric_name"`, ruleTestParsing, ``)
f(`_name__="metric_name}"`, ruleTestParsing, ``)
f(`metrics_name}"`, ruleTestParsing, ``)
// test multiple rules including remote writes
// drop all labels and add one in URL relabeling
rule1 := `
- action: labeldrop
regex: "drop_me_metrics_relabel"
`
rule2 := `
- action: labeldrop
regex: "drop_me_remote_write_relabel"
`
rule3 := `
- target_label: add_me_url_relabel
replacement: added
`
f(`{__name__="metric_name", drop_me_metrics_relabel="1", drop_me_remote_write_relabel="2"}`, rule1+rule2+rule3, `metric_name{add_me_url_relabel="added"}`)
}

View File

@@ -127,7 +127,7 @@ func getCreds(cfg *apiConfig) (*apiCredentials, error) {
// See https://cloud.yandex.com/en-ru/docs/compute/operations/vm-connect/auth-inside-vm
func getInstanceCreds(cfg *apiConfig) (*apiCredentials, error) {
// Try obtaining GCE-like creds at first.
// See https://yandex.cloud/en/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm
// See https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm
creds, err := getGCEInstanceCreds(cfg)
if err == nil {
return creds, nil
@@ -150,7 +150,7 @@ func getInstanceCreds(cfg *apiConfig) (*apiCredentials, error) {
// getGCEInstanceCreds gets Yandex Cloud IAM token using GCE API
//
// See https://yandex.cloud/en/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm
// See https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm
func getGCEInstanceCreds(cfg *apiConfig) (*apiCredentials, error) {
endpoint := "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token"
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
@@ -183,7 +183,7 @@ func getGCEInstanceCreds(cfg *apiConfig) (*apiCredentials, error) {
}, nil
}
// See https://yandex.cloud/en/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm
// See https://yandex.cloud/en-ru/docs/compute/operations/vm-connect/auth-inside-vm#auth-inside-vm
type gceAPICredentials struct {
AccessToken string `json:"access_token"`
ExpiresIn int `json:"expires_in"`

View File

@@ -3,34 +3,92 @@ package promscrape
import (
"fmt"
"net/http"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
)
// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page
func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) {
// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page.
// remotewrite-related relabel configs could be empty as vmsingle doesn't provide remote write feature.
func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request, rwGlobalRelabelConfigs string, rwURLRelabelConfigss []string) {
targetID := r.FormValue("id")
metric := r.FormValue("metric")
relabelConfigs := r.FormValue("relabel_configs")
// if set, it means user want to load relabel config for another url so everything will be reloaded.
reloadRWURLRelabelConfigs := r.FormValue("reload_url_relabel_configs")
// only for per-URL configs and has to be set with reload_url_relabel_configs.
rwURLRelabelConfigsIdxStr := r.FormValue("url_relabel_configs_index")
format := r.FormValue("format")
var err error
if metric == "" && relabelConfigs == "" && targetID != "" {
// if all per-URL config is empty, it means no per-URL rule is configured.
// set it to 0 so the user do not see the options in debug page.
rwURLRelabelConfigsLength := 0
for _, urlRelabelConfig := range rwURLRelabelConfigss {
if urlRelabelConfig != "" {
rwURLRelabelConfigsLength = len(rwURLRelabelConfigss)
break
}
}
rwURLRelabelConfigsIdx, idxErr := strconv.Atoi(rwURLRelabelConfigsIdxStr)
if idxErr != nil {
rwURLRelabelConfigsIdx = -1
}
// load the initial data with specific remote write URL index (default 0) in 2 cases:
// - everything is not set.
// - `reload` is set.
init := metric == "" && relabelConfigs == "" && reloadRWURLRelabelConfigs == ""
reload := reloadRWURLRelabelConfigs != ""
if (init || reload) && targetID != "" {
pcs, labels, ok := getMetricRelabelContextByTargetID(targetID)
if !ok {
err = fmt.Errorf("cannot find target for id=%s", targetID)
targetID = ""
} else {
metric = labels.String()
relabelConfigs = pcs.String()
// set the per-URL remote write relabel according to index, any error will fall back the index to 0.
rwURLRelabelConfigs := ""
if len(rwURLRelabelConfigss) > 0 {
// ignore the error if the input is invalid or exceed the length, and fallback to 0.
if rwURLRelabelConfigsIdx < 0 || rwURLRelabelConfigsIdx >= len(rwURLRelabelConfigss) {
rwURLRelabelConfigsIdx = 0
}
rwURLRelabelConfigs = rwURLRelabelConfigss[rwURLRelabelConfigsIdx]
}
relabelConfigs = composeRelabelConfigs(pcs.String(), rwGlobalRelabelConfigs, rwURLRelabelConfigs, rwURLRelabelConfigsIdx)
}
}
if format == "json" {
httpserver.EnableCORS(w, r)
w.Header().Set("Content-Type", "application/json")
}
promrelabel.WriteMetricRelabelDebug(w, targetID, metric, relabelConfigs, format, err)
promrelabel.WriteMetricRelabelDebug(w, targetID, metric, relabelConfigs, rwURLRelabelConfigsLength, rwURLRelabelConfigsIdx, format, err)
}
func composeRelabelConfigs(relabelConfigs, rwGlobalRelabelConfigs, rwURLRelabelConfigs string, rwURLIdx int) string {
if relabelConfigs != "" {
relabelConfigs = "# -promscrape.config\n" + relabelConfigs
}
if rwGlobalRelabelConfigs != "" {
relabelConfigs += "\n# -remoteWrite.relabelConfig"
relabelConfigs += "\n" + rwGlobalRelabelConfigs
}
if rwURLRelabelConfigs != "" {
relabelConfigs += fmt.Sprintf("\n# -remoteWrite.urlRelabelConfig=remote-write-url-%d", rwURLIdx)
relabelConfigs += "\n" + rwURLRelabelConfigs
}
return relabelConfigs
}
// WriteTargetRelabelDebug generates response for /target-relabel-debug page
@@ -48,7 +106,7 @@ func WriteTargetRelabelDebug(w http.ResponseWriter, r *http.Request) {
targetID = ""
} else {
metric = labels.labelsString()
relabelConfigs = pcs.String()
relabelConfigs = "# -promscrape.config\n" + pcs.String()
}
}
if format == "json" {

View File

@@ -223,7 +223,7 @@ func (r *Row) unmarshal(s string, tagsPool []Tag, noEscapes bool) ([]Tag, error)
// This looks like OpenMetrics timestamp in Unix seconds.
// Convert it to milliseconds.
//
// See https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#timestamps
// See https://github.com/OpenObservability/OpenMetrics/blob/master/specification/OpenMetrics.md#timestamps
ts *= 1000
}
r.Timestamp = int64(ts)
@@ -886,7 +886,7 @@ func unmarshalMetadata(dst []Metadata, s string, errLogger func(s string)) []Met
// "untyped" is the Prometheus exposition format name; "unknown" is the OpenMetrics equivalent.
md.Type = prompb.MetricTypeUnknown
case "info":
// OpenMetrics info type - see https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md
// OpenMetrics info type - see https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md
md.Type = prompb.MetricTypeInfo
case "gaugehistogram":
// OpenMetrics GaugeHistogram type

View File

@@ -379,7 +379,7 @@ cassandra_token_ownership_ratio 78.9`, &Rows{
}},
})
// Exemplars - see https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
// Exemplars - see https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#exemplars-1
f(`foo_bucket{le="10",a="#b"} 17 # {trace_id="oHg5SJ#YRHA0"} 9.8 1520879607.789
abc 123 456 # foobar
foo 344#bar`, &Rows{
@@ -411,7 +411,7 @@ cassandra_token_ownership_ratio 78.9`, &Rows{
})
// "Infinity" word - this has been added in OpenMetrics.
// See https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md
// See https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md
// Checks for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924
inf := math.Inf(1)
f(`
@@ -933,7 +933,7 @@ cassandra_token_ownership_ratio 78.9
}},
}, &MetadataRows{})
// Exemplars - see https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
// Exemplars - see https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md#exemplars-1
f(`foo_bucket{le="10",a="#b"} 17 # {trace_id="oHg5SJ#YRHA0"} 9.8 1520879607.789
abc 123 456 # foobar
foo 344#bar`, &Rows{
@@ -965,7 +965,7 @@ cassandra_token_ownership_ratio 78.9
}, &MetadataRows{})
// "Infinity" word - this has been added in OpenMetrics.
// See https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md
// See https://github.com/OpenObservability/OpenMetrics/blob/master/OpenMetrics.md
// Checks for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/924
inf := math.Inf(1)
f(`

View File

@@ -3,7 +3,6 @@ package storage
import (
"bytes"
"fmt"
"math"
"runtime"
"slices"
"sort"
@@ -12,7 +11,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
)
@@ -496,6 +494,25 @@ func MarshalMetricNameRaw(dst []byte, labels []prompb.Label) []byte {
return dst
}
// marshalRaw marshals mn to dst and returns the result.
//
// The results may be unmarshaled with MetricName.UnmarshalRaw.
//
// This function is for testing purposes. MarshalMetricNameRaw must be used
// in prod instead.
func (mn *MetricName) marshalRaw(dst []byte) []byte {
dst = marshalBytesFast(dst, nil)
dst = marshalBytesFast(dst, mn.MetricGroup)
mn.sortTags()
for i := range mn.Tags {
tag := &mn.Tags[i]
dst = marshalBytesFast(dst, tag.Key)
dst = marshalBytesFast(dst, tag.Value)
}
return dst
}
// UnmarshalRaw unmarshals mn encoded with MarshalMetricNameRaw.
func (mn *MetricName) UnmarshalRaw(src []byte) error {
mn.Reset()
@@ -522,18 +539,12 @@ func (mn *MetricName) UnmarshalRaw(src []byte) error {
}
func marshalStringFast(dst []byte, s string) []byte {
if len(s) > math.MaxUint16 {
logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
}
dst = encoding.MarshalUint16(dst, uint16(len(s)))
dst = append(dst, s...)
return dst
}
func marshalBytesFast(dst []byte, s []byte) []byte {
if len(s) > math.MaxUint16 {
logger.Panicf("BUG: s len %d cannot exceed %d", len(s), math.MaxUint16)
}
dst = encoding.MarshalUint16(dst, uint16(len(s)))
dst = append(dst, s...)
return dst

View File

@@ -6,25 +6,6 @@ import (
"testing"
)
// marshalRaw marshals mn to dst and returns the result.
//
// The results may be unmarshaled with MetricName.UnmarshalRaw.
//
// This function is for testing purposes. MarshalMetricNameRaw must be used
// in prod instead.
func (mn *MetricName) marshalRaw(dst []byte) []byte {
dst = marshalBytesFast(dst, nil)
dst = marshalBytesFast(dst, mn.MetricGroup)
mn.sortTags()
for i := range mn.Tags {
tag := &mn.Tags[i]
dst = marshalBytesFast(dst, tag.Key)
dst = marshalBytesFast(dst, tag.Value)
}
return dst
}
func TestMetricNameString(t *testing.T) {
f := func(mn *MetricName, resultExpected string) {
t.Helper()

View File

@@ -2,7 +2,6 @@ package metricsmetadata
import (
"fmt"
"math"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/encoding"
@@ -25,7 +24,7 @@ type Row struct {
}
// MarshalTo serializes Row into provided buffer and returns result
func (mr *Row) MarshalTo(dst []byte) ([]byte, error) {
func (mr *Row) MarshalTo(dst []byte) []byte {
dstLen := len(dst)
// tenant information (accountID and projectID)
dstSize := dstLen + 8
@@ -38,20 +37,10 @@ func (mr *Row) MarshalTo(dst []byte) ([]byte, error) {
dst = encoding.MarshalUint32(dst, mr.AccountID)
dst = encoding.MarshalUint32(dst, mr.ProjectID)
dst = encoding.MarshalUint32(dst, uint32(mr.Type))
var err error
dst, err = marshalBytesFast(dst, mr.MetricFamilyName)
if err != nil {
return dst, fmt.Errorf("cannot marshal MetricFamilyName: %w", err)
}
dst, err = marshalBytesFast(dst, mr.Help)
if err != nil {
return dst, fmt.Errorf("cannot marshal Help: %w", err)
}
dst, err = marshalBytesFast(dst, mr.Unit)
if err != nil {
return dst, fmt.Errorf("cannot marshal Unit: %w", err)
}
return dst, nil
dst = marshalBytesFast(dst, mr.MetricFamilyName)
dst = marshalBytesFast(dst, mr.Help)
dst = marshalBytesFast(dst, mr.Unit)
return dst
}
// Unmarshal parses Row from provided buffer and returns tail buffer
@@ -137,11 +126,8 @@ func UnmarshalRows(dst []Row, src []byte, maxRows int) ([]Row, []byte, error) {
return dst, src, nil
}
func marshalBytesFast(dst []byte, s []byte) ([]byte, error) {
if len(s) > math.MaxUint16 {
return dst, fmt.Errorf("size of s: %d cannot exceed max uint16", len(s))
}
func marshalBytesFast(dst []byte, s []byte) []byte {
dst = encoding.MarshalUint16(dst, uint16(len(s)))
dst = append(dst, s...)
return dst, nil
return dst
}

View File

@@ -325,17 +325,6 @@ func (s *Search) NextMetricBlock() bool {
// SearchQuery is used for sending search queries from vmselect to vmstorage.
type SearchQuery struct {
AccountID uint32
ProjectID uint32
// TenantTokens and IsMultiTenant is artificial fields
// they're only exist at runtime and cannot be transferred
// via network calls for keeping communication protocol compatibility
// TODO:@f41gh7 introduce breaking change to the protocol later
// and use TenantTokens instead of AccountID and ProjectID
TenantTokens []TenantToken
IsMultiTenant bool
// The time range for searching time series
MinTimestamp int64
MaxTimestamp int64
@@ -484,16 +473,7 @@ func (sq *SearchQuery) String() string {
start := TimestampToHumanReadableFormat(sq.MinTimestamp)
end := TimestampToHumanReadableFormat(sq.MaxTimestamp)
a := sq.FiltersString()
if !sq.IsMultiTenant {
return fmt.Sprintf("accountID=%d, projectID=%d, filters=%s, timeRange=[%s..%s]", sq.AccountID, sq.ProjectID, a, start, end)
}
tts := make([]string, len(sq.TenantTokens))
for i, tt := range sq.TenantTokens {
tts[i] = tt.String()
}
return fmt.Sprintf("tenants=[%s], filters=%s, timeRange=[%s..%s]", strings.Join(tts, ","), a, start, end)
return fmt.Sprintf("filters=%s, timeRange=[%s..%s]", a, start, end)
}
// FiltersString returns string representation of the tag filters.
@@ -513,9 +493,8 @@ func tagFiltersToString(tfs []TagFilter) string {
return "{" + strings.Join(a, ",") + "}"
}
// MarshalWithoutTenant appends marshaled sq without AccountID/ProjectID to dst and returns the result.
// It is expected that TenantToken is already marshaled to dst.
func (sq *SearchQuery) MarshalWithoutTenant(dst []byte) []byte {
// Marshal appends marshaled sq to dst and returns the result.
func (sq *SearchQuery) Marshal(dst []byte) []byte {
dst = encoding.MarshalVarInt64(dst, sq.MinTimestamp)
dst = encoding.MarshalVarInt64(dst, sq.MaxTimestamp)
dst = encoding.MarshalVarUint64(dst, uint64(len(sq.TagFilterss)))
@@ -525,25 +504,11 @@ func (sq *SearchQuery) MarshalWithoutTenant(dst []byte) []byte {
dst = tagFilters[i].Marshal(dst)
}
}
dst = encoding.MarshalUint32(dst, uint32(sq.MaxMetrics))
return dst
}
// Unmarshal unmarshals sq from src and returns the tail.
func (sq *SearchQuery) Unmarshal(src []byte) ([]byte, error) {
if len(src) < 4 {
return src, fmt.Errorf("cannot unmarshal AccountID: too short src len: %d; must be at least %d bytes", len(src), 4)
}
sq.AccountID = encoding.UnmarshalUint32(src)
src = src[4:]
if len(src) < 4 {
return src, fmt.Errorf("cannot unmarshal ProjectID: too short src len: %d; must be at least %d bytes", len(src), 4)
}
sq.ProjectID = encoding.UnmarshalUint32(src)
src = src[4:]
sq.TenantTokens = []TenantToken{{AccountID: sq.AccountID, ProjectID: sq.ProjectID}}
minTs, nSize := encoding.UnmarshalVarInt64(src)
if nSize <= 0 {
return src, fmt.Errorf("cannot unmarshal MinTimestamp from varint")
@@ -584,12 +549,6 @@ func (sq *SearchQuery) Unmarshal(src []byte) ([]byte, error) {
sq.TagFilterss[i] = tagFilters
}
if len(src) < 4 {
return src, fmt.Errorf("cannot unmarshal MaxMetrics: too short src len: %d; must be at least %d bytes", len(src), 4)
}
sq.MaxMetrics = int(encoding.UnmarshalUint32(src))
src = src[4:]
return src, nil
}

View File

@@ -32,12 +32,7 @@ func TestSearchQueryMarshalUnmarshal(t *testing.T) {
// Skip nil sq1.
continue
}
tt := TenantToken{
AccountID: sq1.AccountID,
ProjectID: sq1.ProjectID,
}
buf = tt.Marshal(buf[:0])
buf = sq1.MarshalWithoutTenant(buf)
buf = sq1.Marshal(buf[:0])
tail, err := sq2.Unmarshal(buf)
if err != nil {
@@ -46,12 +41,6 @@ func TestSearchQueryMarshalUnmarshal(t *testing.T) {
if len(tail) > 0 {
t.Fatalf("unexpected tail left after SearchQuery unmarshaling; tail (len=%d): %q", len(tail), tail)
}
if sq2.AccountID != sq1.AccountID {
t.Fatalf("unexpected AccountID; got %d; want %d", sq2.AccountID, sq1.AccountID)
}
if sq2.ProjectID != sq1.ProjectID {
t.Fatalf("unexpected ProjectID; got %d; want %d", sq2.ProjectID, sq1.ProjectID)
}
if sq1.MinTimestamp != sq2.MinTimestamp {
t.Fatalf("unexpected MinTimestamp; got %d; want %d", sq2.MinTimestamp, sq1.MinTimestamp)
}

View File

@@ -1,7 +1,6 @@
package timeserieslimits
import (
"math"
"time"
"github.com/VictoriaMetrics/metrics"
@@ -9,7 +8,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/atomicutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
)
var (
@@ -46,9 +44,6 @@ func Init(inputMaxLabelsPerTimeseries, inputMaxLabelNameLen, inputMaxLabelValueL
_ = metrics.GetOrCreateGauge(`vm_rows_ignored_total{reason="too_long_label_value"}`, func() float64 {
return float64(ignoredSeriesWithTooLongLabelValue.Load())
})
_ = metrics.GetOrCreateGauge(`vm_rows_ignored_total{reason="too_long_metric_metadata_value"}`, func() float64 {
return float64(ignoredMetricsMetadataWithTooLongValue.Load())
})
}
var (
@@ -66,8 +61,6 @@ var (
// ignoredSeriesWithTooLongLabelValue is the number of ignored series which contain labels with too long values
ignoredSeriesWithTooLongLabelValue atomicutil.Uint64
ignoredMetricsMetadataWithTooLongValue atomicutil.Uint64
)
func trackIgnoredSeriesWithTooManyLabels(labels []prompb.Label) {
@@ -139,56 +132,3 @@ func IsExceeding(labels []prompb.Label) bool {
}
return false
}
func trackIgnoredMetricMetadataWithTooLongValue(fieldName, metricName string, fieldSize int) {
ignoredMetricsMetadataWithTooLongValue.Add(1)
select {
case <-ignoredSeriesWithTooLongLabelValueLogTicker.C:
// Do not call logger.WithThrottler() here, since this will result in increased CPU usage
logger.Warnf("ignoring metric metadata with metric name %q; field %q value length=%d exceeds %d limit; "+
"reduce the size of field at metric metadata source.",
metricName, fieldName, fieldSize, metricMetadataMaxFieldValueSize)
default:
}
}
// metricMetadataMaxFieldValueSize defines max size of string fields at MetricMetadata
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11128 for details
const metricMetadataMaxFieldValueSize = math.MaxUint16
// IsMetricMetadataExceeding returns true if prompb.MetricMetadata Help, MetricFamilyName, or Unit field value size exceed the 64KiB limit.
//
// Additionally, it increments the corresponding metrics and prints warning messages to the log.
func IsMetricMetadataExceeding(md *prompb.MetricMetadata) bool {
if len(md.Help) > metricMetadataMaxFieldValueSize {
trackIgnoredMetricMetadataWithTooLongValue("help", md.MetricFamilyName, len(md.Help))
return true
}
if len(md.MetricFamilyName) > metricMetadataMaxFieldValueSize {
trackIgnoredMetricMetadataWithTooLongValue("metricFamilyName", md.MetricFamilyName, len(md.MetricFamilyName))
return true
}
if len(md.Unit) > metricMetadataMaxFieldValueSize {
trackIgnoredMetricMetadataWithTooLongValue("unit", md.MetricFamilyName, len(md.Unit))
return true
}
return false
}
// IsPrometheusMetadataExceeding returns true if prometheus.Metadata Help or Metric field value size exceed the 64KiB limit.
//
// Additionally, it increments the corresponding metrics and prints warning messages to the log.
func IsPrometheusMetadataExceeding(md *prometheus.Metadata) bool {
if len(md.Help) > metricMetadataMaxFieldValueSize {
trackIgnoredMetricMetadataWithTooLongValue("help", md.Metric, len(md.Help))
return true
}
if len(md.Metric) > metricMetadataMaxFieldValueSize {
trackIgnoredMetricMetadataWithTooLongValue("metric", md.Metric, len(md.Metric))
return true
}
return false
}

File diff suppressed because it is too large Load Diff