Compare commits

..

5 Commits

Author SHA1 Message Date
Pablo Fernandez
b08d7943e6 Fix vmagent.yaml example urls 2026-06-03 16:48:35 +01:00
Pablo Fernandez
c731b20e01 Grammar pass 2026-06-03 15:30:54 +01:00
Pablo Fernandez
627c72b84c Fix some config issues and added verification step 2026-06-03 15:27:24 +01:00
Pablo Fernandez
2f162f7e3f WIP draft multi-retention guide update 2026-06-02 22:40:39 +01:00
Pablo Fernandez
2901b3df2a Add initial draft for the multi-retention guide 2026-06-02 17:54:21 +01:00
65 changed files with 537 additions and 2137 deletions

View File

@@ -52,14 +52,14 @@ jobs:
restore-keys: go-artifacts-${{ runner.os }}-codeql-analyze-${{ steps.go.outputs.go-version }}-
- name: Initialize CodeQL
uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5
uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
languages: go
- name: Autobuild
uses: github/codeql-action/autobuild@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5
uses: github/codeql-action/autobuild@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5
uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4.35.3
with:
category: 'language:go'

View File

@@ -79,8 +79,7 @@ var (
"writing them to remote storage. "+
"Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. "+
"By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. "+
"This option may be used for improving data compression for the stored metrics. "+
"See also -remoteWrite.significantFigures")
"This option may be used for improving data compression for the stored metrics")
sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. `+
`This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. `+
`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}`+

View File

@@ -95,7 +95,6 @@ type groupMetrics struct {
iterationTotal *metrics.Counter
iterationDuration *metrics.Summary
iterationMissed *metrics.Counter
iterationReset *metrics.Counter
iterationInterval *metrics.Gauge
}
@@ -331,7 +330,6 @@ func (g *Group) Init() {
g.metrics.iterationTotal = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_total{%s}`, labels))
g.metrics.iterationDuration = g.metrics.set.NewSummary(fmt.Sprintf(`vmalert_iteration_duration_seconds{%s}`, labels))
g.metrics.iterationMissed = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_missed_total{%s}`, labels))
g.metrics.iterationReset = g.metrics.set.NewCounter(fmt.Sprintf(`vmalert_iteration_reset_total{%s}`, labels))
g.metrics.iterationInterval = g.metrics.set.NewGauge(fmt.Sprintf(`vmalert_iteration_interval_seconds{%s}`, labels), func() float64 {
i := g.Interval.Seconds()
return i
@@ -476,16 +474,14 @@ func (g *Group) Start(ctx context.Context, rw remotewrite.RWClient, rr datasourc
if missed < 0 {
// missed can become < 0 due to irregular delays during evaluation
// which can result in time.Since(evalTS) < g.Interval;
// or the system wall clock was changed backward,
// Reset the evalTS to the current time.
// or the system wall clock was changed backward
missed = 0
evalTS = time.Now()
g.metrics.iterationReset.Inc()
} else {
evalTS = evalTS.Add((missed + 1) * g.Interval)
}
if missed > 0 {
g.metrics.iterationMissed.Inc()
}
evalTS = evalTS.Add((missed + 1) * g.Interval)
eval(evalCtx, evalTS)
}

View File

@@ -11,8 +11,6 @@ import (
"strconv"
"strings"
"github.com/VictoriaMetrics/metricsql"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/rule"
@@ -162,12 +160,12 @@ func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
case "/vmalert/api/v1/alerts", "/api/v1/alerts":
// path used by Grafana for ng alerting
af, err := newAlertsFilter(r)
gf, err := newGroupsFilter(r)
if err != nil {
errJson(w, r, err)
return true
}
data, err := rh.listAlerts(af)
data, err := rh.listAlerts(gf)
if err != nil {
errJson(w, r, err)
return true
@@ -327,48 +325,6 @@ func (gf *groupsFilter) matches(group *rule.Group) bool {
return true
}
type alertsFilter struct {
gf *groupsFilter
match [][]metricsql.LabelFilter
}
func getMatchFilters(matches []string) ([][]metricsql.LabelFilter, *httpserver.ErrorWithStatusCode) {
if len(matches) == 0 {
return nil, nil
}
tfss := make([][]metricsql.LabelFilter, 0, len(matches))
for _, s := range matches {
expr, err := metricsql.Parse(s)
if err != nil {
return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": failed to parse %q: %w`, s, err), http.StatusBadRequest)
}
me, ok := expr.(*metricsql.MetricExpr)
if !ok {
return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": expecting metricSelector; got %q`, expr.AppendString(nil)), http.StatusBadRequest)
}
if len(me.LabelFilterss) == 0 {
return nil, errResponse(fmt.Errorf(`invalid parameter "match[]": labelFilterss cannot be empty`), http.StatusBadRequest)
}
tfss = append(tfss, me.LabelFilterss...)
}
return tfss, nil
}
func newAlertsFilter(r *http.Request) (*alertsFilter, *httpserver.ErrorWithStatusCode) {
gf, err := newGroupsFilter(r)
if err != nil {
return nil, err
}
var af alertsFilter
af.gf = gf
af.match, err = getMatchFilters(r.Form["match[]"])
if err != nil {
return nil, err
}
return &af, nil
}
// see https://prometheus.io/docs/prometheus/latest/querying/api/#rules
type rulesFilter struct {
gf *groupsFilter
@@ -379,7 +335,6 @@ type rulesFilter struct {
maxGroups int
pageNum int
search string
match [][]metricsql.LabelFilter
extendedStates bool
}
@@ -400,10 +355,7 @@ func newRulesFilter(r *http.Request) (*rulesFilter, *httpserver.ErrorWithStatusC
return nil, errResponse(fmt.Errorf(`invalid parameter "type": not supported value %q`, ruleTypeParam), http.StatusBadRequest)
}
}
rf.match, err = getMatchFilters(r.Form["match[]"])
if err != nil {
return nil, err
}
states := vs["state"]
if len(states) == 0 {
states = vs["filter"]
@@ -464,47 +416,12 @@ func (rf *rulesFilter) matchesRule(r *rule.ApiRule) bool {
if len(rf.ruleNames) > 0 && !slices.Contains(rf.ruleNames, r.Name) {
return false
}
if !areLabelsMatch(r.Labels, rf.match) {
return false
}
if len(rf.states) == 0 {
return true
}
return slices.Contains(rf.states, r.State)
}
func areLabelsMatch(labels map[string]string, matches [][]metricsql.LabelFilter) bool {
if len(matches) == 0 {
return true
}
// labels need to match at least one of the provided match[] arg
return slices.ContainsFunc(matches, func(filters []metricsql.LabelFilter) bool {
for _, mf := range filters {
if !isLabelFilterMatch(labels[mf.Label], mf) {
return false
}
}
return true
})
}
func isLabelFilterMatch(s string, match metricsql.LabelFilter) bool {
if !match.IsRegexp {
if match.IsNegative {
return s != match.Value
}
return s == match.Value
}
re, err := metricsql.CompileRegexpAnchored(match.Value)
if err != nil {
return false
}
if match.IsNegative {
return !re.MatchString(s)
}
return re.MatchString(s)
}
func (rh *requestHandler) groups(rf *rulesFilter) *listGroupsResponse {
rh.m.groupsMu.RLock()
defer rh.m.groupsMu.RUnlock()
@@ -626,14 +543,14 @@ func (rh *requestHandler) groupAlerts() []rule.GroupAlerts {
return gAlerts
}
func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
func (rh *requestHandler) listAlerts(gf *groupsFilter) ([]byte, *httpserver.ErrorWithStatusCode) {
rh.m.groupsMu.RLock()
defer rh.m.groupsMu.RUnlock()
lr := listAlertsResponse{Status: "success"}
lr.Data.Alerts = make([]*rule.ApiAlert, 0)
for _, group := range rh.m.groups {
if !af.gf.matches(group) {
if !gf.matches(group) {
continue
}
g := group.ToAPI()
@@ -641,11 +558,7 @@ func (rh *requestHandler) listAlerts(af *alertsFilter) ([]byte, *httpserver.Erro
if r.Type != rule.TypeAlerting {
continue
}
for _, alert := range r.Alerts {
if areLabelsMatch(alert.Labels, af.match) {
lr.Data.Alerts = append(lr.Data.Alerts, alert)
}
}
lr.Data.Alerts = append(lr.Data.Alerts, r.Alerts...)
}
}

View File

@@ -348,7 +348,7 @@
typeK, ns := keys[i], targets[notifier.TargetType(keys[i])]
count := len(ns)
%}
<div class="w-100 flex-column">
<div class="w-100 flex-column vm-group">
<span class="d-flex justify-content-between" id="group-{%s typeK %}">
<a href="#group-{%s typeK %}">{%s typeK %} ({%d count %})</a>
<span
@@ -361,7 +361,7 @@
<div id="item-{%s typeK %}" class="collapse show">
<table class="table table-striped table-hover table-sm">
<thead>
<tr>
<tr class="vm-item">
<th scope="col">Labels</th>
<th scope="col">Address</th>
</tr>

View File

@@ -1115,7 +1115,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
//line app/vmalert/web.qtpl:350
qw422016.N().S(`
<div class="w-100 flex-column">
<div class="w-100 flex-column vm-group">
<span class="d-flex justify-content-between" id="group-`)
//line app/vmalert/web.qtpl:352
qw422016.E().S(typeK)
@@ -1152,7 +1152,7 @@ func StreamListTargets(qw422016 *qt422016.Writer, r *http.Request, targets map[n
qw422016.N().S(`" class="collapse show">
<table class="table table-striped table-hover table-sm">
<thead>
<tr>
<tr class="vm-item">
<th scope="col">Labels</th>
<th scope="col">Address</th>
</tr>

View File

@@ -10,8 +10,6 @@ import (
"testing"
"time"
"github.com/VictoriaMetrics/metricsql"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
@@ -39,14 +37,12 @@ func TestHandler(t *testing.T) {
Concurrency: 1,
Rules: []config.Rule{
{
ID: 0,
Alert: "alert",
Labels: map[string]string{"job": "foo"},
ID: 0,
Alert: "alert",
},
{
ID: 1,
Record: "record",
Labels: map[string]string{"job": "bar"},
},
},
}, fq, 1*time.Minute, nil)
@@ -132,18 +128,6 @@ func TestHandler(t *testing.T) {
if length := len(lr.Data.Alerts); length != 2 {
t.Fatalf("expected 2 alert got %d", length)
}
lr = listAlertsResponse{}
getResp(t, ts.URL+`/api/v1/alerts?match[]={job="foo"}`, &lr, 200)
if length := len(lr.Data.Alerts); length != 3 {
t.Fatalf("expected 3 alerts got %d", length)
}
lr = listAlertsResponse{}
getResp(t, ts.URL+`/api/v1/alerts?match[]={job="bar"}`, &lr, 200)
if length := len(lr.Data.Alerts); length != 0 {
t.Fatalf("expected 0 alerts got %d", length)
}
})
t.Run("/api/v1/alert?alertID&groupID", func(t *testing.T) {
expAlert := rule.NewAlertAPI(ar, ar.GetAlerts()[0])
@@ -258,13 +242,6 @@ func TestHandler(t *testing.T) {
check("/vmalert/api/v1/rules?datasource_type=graphite", 200, 1, 2)
check("/vmalert/api/v1/rules?datasource_type=graphiti", 400, 0, 0)
// invalid match[] params
check(`/vmalert/api/v1/rules?match[]={job=!"foo"}`, 400, 0, 0)
check(`/vmalert/api/v1/rules?match[]={job="foo"}`, 200, 3, 3)
check(`/vmalert/api/v1/rules?match[]={job="bar"}`, 200, 3, 3)
check(`/vmalert/api/v1/rules?match[]={job="bar"}&match[]={job="foo"}`, 200, 3, 6)
check(`/vmalert/api/v1/rules?match[]={job="barzz"}`, 200, 0, 0)
// no filtering expected due to bad params
check("/api/v1/rules?type=badParam", 400, 0, 0)
check("/api/v1/rules?foo=bar", 200, 3, 6)
@@ -390,116 +367,3 @@ func TestEmptyResponse(t *testing.T) {
}
})
}
func TestMatchesRule(t *testing.T) {
parseMatch := func(t *testing.T, selectors []string) [][]metricsql.LabelFilter {
t.Helper()
var match [][]metricsql.LabelFilter
for _, s := range selectors {
expr, err := metricsql.Parse(s)
if err != nil {
t.Fatalf("failed to parse selector %q: %v", s, err)
}
me, ok := expr.(*metricsql.MetricExpr)
if !ok {
t.Fatalf("expected MetricExpr for %q, got %T", s, expr)
}
match = append(match, me.LabelFilterss...)
}
return match
}
f := func(t *testing.T, selectors []string, labels map[string]string, wantMatch bool) {
t.Helper()
rf := &rulesFilter{
gf: &groupsFilter{},
match: parseMatch(t, selectors),
}
r := &rule.ApiRule{Labels: labels}
got := rf.matchesRule(r)
if got != wantMatch {
t.Fatalf("matchesRule(%v) with selectors %v: got %v, want %v",
labels, selectors, got, wantMatch)
}
}
f(t, nil, map[string]string{"foo": "bar"}, true)
f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "bar"}, true)
f(t, []string{`{foo="bar"}`}, map[string]string{"foo": "baz"}, false)
f(t, []string{`{foo="bar"}`}, map[string]string{"bar": "baz"}, false)
f(t, []string{`{foo=""}`}, map[string]string{"bar": "baz"}, true)
f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "baz"}, true)
f(t, []string{`{foo!="bar"}`}, map[string]string{"foo": "bar"}, false)
f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "bar"}, true)
f(t, []string{`{foo=~"bar.*"}`}, map[string]string{"foo": "baz"}, false)
f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "baz"}, true)
f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "bar"}, true)
f(t, []string{`{bar=~"baz|bar"}`}, map[string]string{"bar": "foo"}, false)
f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "baz"}, true)
f(t, []string{`{foo!~"bar.*"}`}, map[string]string{"foo": "bar"}, false)
// single match[] with multiple filters
f(t,
[]string{`{job="foo",instance="bar"}`},
map[string]string{"job": "foo", "instance": "bar"},
true,
)
f(t,
[]string{`{job="foo",instance="bar"}`},
map[string]string{"job": "other", "instance": "bar"},
false,
)
f(t,
[]string{`{foo="bar",baz=~"b.*"}`},
map[string]string{"foo": "bar", "baz": "bazinga"},
true,
)
f(t,
[]string{`{foo="bar",baz=~"b.*"}`},
map[string]string{"foo": "other", "baz": "bazinga"},
false,
)
// multiple matches[]
f(t,
[]string{`{foo="bar"}`, `{foo="baz"}`},
map[string]string{"foo": "baz"},
true,
)
f(t,
[]string{`{foo="bar"}`, `{foo="baz"}`},
map[string]string{"foo": "unknown"},
false,
)
f(t,
[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
map[string]string{"bar": "bazinga"},
true,
)
f(t,
[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
map[string]string{"foo": "bartender"},
true,
)
f(t,
[]string{`{foo=~"bar.*"}`, `{bar=~"baz.*"}`},
map[string]string{"foo": "other", "bar": "other"},
false,
)
f(t,
[]string{`{job="foo",instance="bar"}`, `{foo="bar"}`},
map[string]string{"foo": "bar"},
true,
)
f(t,
[]string{`{job="foo", instance="bar"}`, `{foo="bar"}`},
map[string]string{"instance": "barr", "job": "foo"},
false,
)
}

View File

@@ -889,8 +889,7 @@ func reloadAuthConfig() (bool, error) {
}
mp := authUsers.Load()
jwtc := jwtAuthCache.Load()
logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp)+len(jwtc.users), *authConfigPath)
logger.Infof("loaded information about %d users from -auth.config=%q", len(*mp), *authConfigPath)
return true, nil
}

View File

@@ -317,7 +317,7 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tk
defer ui.endConcurrencyLimit()
// Process the request.
processRequest(w, r, ui, tkn, userName)
processRequest(w, r, ui, tkn)
}
func beginConcurrencyLimit(ctx context.Context) error {
@@ -391,7 +391,7 @@ func bufferRequestBody(ctx context.Context, r io.ReadCloser, userName string) (i
return bb, nil
}
func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token, userName string) {
func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *jwt.Token) {
u := normalizeURL(r.URL)
up, hc := ui.getURLPrefixAndHeaders(u, r.Host, r.Header)
isDefault := false
@@ -409,7 +409,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
if ui.DumpRequestOnErrors {
di = debugInfo(u, r)
}
httpserver.Errorf(w, r, "user %s missing route for %q%s", userName, u.String(), di)
httpserver.Errorf(w, r, "missing route for %q%s", u.String(), di)
return
}
up, hc = ui.DefaultURL, ui.HeadersConf
@@ -455,7 +455,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo, tkn *j
ui.backendErrors.Inc()
}
err := &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), userName),
Err: fmt.Errorf("all the %d backends for the user %q are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend", up.getBackendsCount(), ui.name()),
StatusCode: http.StatusBadGateway,
}
httpserver.Errorf(w, r, "%s", err)

View File

@@ -307,24 +307,6 @@ statusCode=200
requested_url={BACKEND}/bar/a/b`
f(cfgStr, requestURL, backendHandler, responseExpected)
// correct authorization but unexisted path, hence missing route error.
cfgStr = `
users:
- username: foo
password: secret
url_map:
- src_paths:
- "/api/v1/write"
url_prefix: "{BACKEND}/bar"`
requestURL = "http://foo:secret@some-host.com/a/b"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "requested_url=http://%s%s", r.Host, r.URL)
}
responseExpected = `
statusCode=400
user foo missing route for "http://foo:secret@some-host.com/a/b"`
f(cfgStr, requestURL, backendHandler, responseExpected)
// verify how path cleanup works
cfgStr = `
unauthorized_user:
@@ -421,7 +403,7 @@ unauthorized_user:
}
responseExpected = `
statusCode=400
user unauthorized missing route for "http://some-host.com/abc?de=fg"`
missing route for "http://some-host.com/abc?de=fg"`
f(cfgStr, requestURL, backendHandler, responseExpected)
// missing default_url and default url_prefix for unauthorized user with dump_request_on_errors enabled
@@ -437,7 +419,7 @@ unauthorized_user:
}
responseExpected = `
statusCode=400
user unauthorized missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
missing route for "http://some-host.com/abc?de=fg" (host: "some-host.com"; path: "/abc"; args: "de=fg"; headers:Connection: Some-Header,Other-Header
Pass-Header: abc
Some-Header: foobar
X-Forwarded-For: 12.34.56.78
@@ -479,7 +461,7 @@ unauthorized_user:
}
responseExpected = `
statusCode=502
all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
f(cfgStr, requestURL, backendHandler, responseExpected)
// all the backend_urls are unavailable for authorized user
@@ -519,7 +501,7 @@ unauthorized_user:
}
responseExpected = `
statusCode=502
all the 0 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
all the 0 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
f(cfgStr, requestURL, backendHandler, responseExpected)
netutil.Resolver = origResolver
@@ -536,7 +518,7 @@ unauthorized_user:
}
responseExpected = `
statusCode=502
all the 2 backends for the user "unauthorized" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
all the 2 backends for the user "" are unavailable for proxying the request - check previous WARN logs to see the exact error for each failed backend`
f(cfgStr, requestURL, backendHandler, responseExpected)
if n := retries.Load(); n != 2 {
t.Fatalf("unexpected number of retries; got %d; want 2", n)
@@ -563,31 +545,6 @@ requested_url={BACKEND}/path2/foo/?de=fg`
if n := retries.Load(); n != 2 {
t.Fatalf("unexpected number of retries; got %d; want 2", n)
}
// make sure that empty config value erases client extra filters and extra labels
cfgStr = `
unauthorized_user:
url_prefix: {BACKEND}/foo?bar=baz&extra_filters[]=&extra_label=&extra_filters=`
requestURL = "http://some-host.com/abc/def?some_arg=some_value&extra_filters[]=baz&extra_label=tenant=admin&extra_filters=bar"
backendHandler = func(w http.ResponseWriter, r *http.Request) {
h := w.Header()
h.Set("Connection", "close")
h.Set("Foo", "bar")
var bb bytes.Buffer
if err := r.Header.Write(&bb); err != nil {
panic(fmt.Errorf("unexpected error when marshaling headers: %w", err))
}
fmt.Fprintf(w, "requested_url=http://%s%s\n%s", r.Host, r.URL, bb.String())
}
responseExpected = `
statusCode=200
Foo: bar
requested_url={BACKEND}/foo/abc/def?bar=baz&extra_filters=&extra_filters%5B%5D=&extra_label=&some_arg=some_value
Pass-Header: abc
User-Agent: vmauth
X-Forwarded-For: 12.34.56.78, 42.2.3.84`
f(cfgStr, requestURL, backendHandler, responseExpected)
}
func TestJWTRequestHandler(t *testing.T) {

View File

@@ -146,8 +146,7 @@ var (
Name: vmRoundDigits,
Value: 100,
Usage: "Round metric values to the given number of decimal digits after the point. " +
"This option may be used for increasing on-disk compression level for the stored metrics. " +
"See also --vm-significant-figures option",
"This option may be used for increasing on-disk compression level for the stored metrics",
},
&cli.StringSliceFlag{
Name: vmExtraLabel,
@@ -501,96 +500,6 @@ var (
}
)
const (
mimirPath = "mimir-path"
mimirTenantID = "mimir-tenant-id"
mimirConcurrency = "mimir-concurrency"
mimirFilterTimeStart = "mimir-filter-time-start"
mimirFilterTimeEnd = "mimir-filter-time-end"
mimirFilterLabel = "mimir-filter-label"
mimirFilterLabelValue = "mimir-filter-label-value"
mimirCredsFilePath = "mimir-creds-file-path"
mimirConfigFilePath = "mimir-config-file-path"
mimirConfigProfile = "mimir-config-profile"
mimirCustomS3Endpoint = "mimir-custom-s3-endpoint"
mimirS3ForcePathStyle = "mimir-s3-force-path-style"
mimirS3TLSInsecureSkipVerify = "mimir-s3-tls-insecure-skip-verify"
mimirSSEKMSKeyID = "mimir-s3-sse-kms-key-id"
mimirSSEAlgorithm = "mimir-s3-sse-algorithm"
)
var (
mimirFlags = []cli.Flag{
&cli.StringFlag{
Name: mimirPath,
Usage: "Path to Mimir storage bucket or local folder.",
Required: true,
},
&cli.StringFlag{
Name: mimirTenantID,
Usage: "Tenant ID for Mimir storage",
},
&cli.IntFlag{
Name: mimirConcurrency,
Usage: "Number of concurrently running block readers",
Value: 1,
},
&cli.StringFlag{
Name: mimirFilterTimeStart,
Usage: "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
Required: true,
},
&cli.StringFlag{
Name: mimirFilterTimeEnd,
Usage: "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
Required: true,
},
&cli.StringFlag{
Name: mimirFilterLabel,
Usage: "Mimir label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
},
&cli.StringFlag{
Name: mimirFilterLabelValue,
Usage: fmt.Sprintf("Regular expression to filter label from %q flag.", mimirFilterLabel),
Value: ".*",
},
&cli.StringFlag{
Name: mimirCredsFilePath,
Usage: "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
},
&cli.StringFlag{
Name: mimirConfigFilePath,
Usage: "Path to file with S3 configs. Configs are loaded from default location if not set. See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
},
&cli.StringFlag{
Name: mimirConfigProfile,
Usage: "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used",
},
&cli.StringFlag{
Name: mimirCustomS3Endpoint,
Usage: "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set",
},
&cli.BoolFlag{
Name: mimirS3ForcePathStyle,
Usage: "Prefixing endpoint with bucket name when set false, true by default.",
Value: true,
},
&cli.BoolFlag{
Name: mimirS3TLSInsecureSkipVerify,
Usage: "Whether to skip TLS verification when connecting to the S3 endpoint.",
},
&cli.StringFlag{
Name: mimirSSEKMSKeyID,
Usage: "SSE KMS Key ID for use with S3-compatible storages.",
},
&cli.StringFlag{
Name: mimirSSEAlgorithm,
Usage: "SSE algorithm for use with S3-compatible storages.",
},
}
)
const (
vmNativeFilterMatch = "vm-native-filter-match"
vmNativeFilterTimeStart = "vm-native-filter-time-start"

View File

@@ -18,7 +18,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/mimir"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -297,56 +296,6 @@ func main() {
return pp.run(ctx)
},
},
{
Name: "mimir",
Usage: "Migrate time series from Mimir object storage or local filesystem",
Flags: mergeFlags(globalFlags, mimirFlags, vmFlags),
Before: beforeFn,
Action: func(c *cli.Context) error {
fmt.Println("Mimir import mode")
vmCfg, err := initConfigVM(c)
if err != nil {
return fmt.Errorf("failed to init VM configuration: %s", err)
}
importer, err = vm.NewImporter(ctx, vmCfg)
if err != nil {
return fmt.Errorf("failed to create VM importer: %s", err)
}
mCfg := mimir.Config{
Filter: mimir.Filter{
TimeMin: c.String(mimirFilterTimeStart),
TimeMax: c.String(mimirFilterTimeEnd),
Label: c.String(mimirFilterLabel),
LabelValue: c.String(mimirFilterLabelValue),
},
Path: c.String(mimirPath),
TenantID: c.String(mimirTenantID),
CredsFilePath: c.String(mimirCredsFilePath),
ConfigFilePath: c.String(mimirConfigFilePath),
ConfigProfile: c.String(mimirConfigProfile),
CustomS3Endpoint: c.String(mimirCustomS3Endpoint),
S3ForcePathStyle: c.Bool(mimirS3ForcePathStyle),
S3TLSInsecureSkipVerify: c.Bool(mimirS3TLSInsecureSkipVerify),
SSEKMSKeyID: c.String(mimirSSEKMSKeyID),
SSEAlgorithm: c.String(mimirSSEAlgorithm),
}
cl, err := mimir.NewClient(ctx, mCfg)
if err != nil {
return fmt.Errorf("failed to create mimir client: %s", err)
}
pp := prometheusProcessor{
cl: cl,
im: importer,
cc: c.Int(mimirConcurrency),
isVerbose: c.Bool(globalVerbose),
}
return pp.run(ctx)
},
},
{
Name: "thanos",
Usage: "Migrate time series from Thanos blocks (supports raw and downsampled data)",
@@ -354,6 +303,7 @@ func main() {
Before: beforeFn,
Action: func(c *cli.Context) error {
fmt.Println("Thanos import mode")
vmCfg, err := initConfigVM(c)
if err != nil {
return fmt.Errorf("failed to init VM configuration: %s", err)
@@ -363,6 +313,7 @@ func main() {
if err != nil {
return fmt.Errorf("failed to create VM importer: %s", err)
}
thanosCfg := thanos.Config{
Snapshot: c.String(thanosSnapshot),
Filter: thanos.Filter{

View File

@@ -1,195 +0,0 @@
package mimir
import (
"fmt"
"log"
"os"
"path/filepath"
"sync"
"github.com/oklog/ulid/v2"
"github.com/prometheus/prometheus/tsdb"
"github.com/prometheus/prometheus/tsdb/tombstones"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
)
var _ tsdb.BlockReader = (*lazyBlockReader)(nil)
// lazyBlockReader is stores block id and segment num information.
// It is used to lazily fetch and parse block data.
// It implements tsdb.BlockReader interface.
type lazyBlockReader struct {
// Block ID.
ID ulid.ULID
// SegmentsNum stores the number of chunks segments in the block.
SegmentsNum int
mu sync.Mutex
reader *tsdb.Block
tempDirPath string
fs common.RemoteFS
err error
}
// newLazyBlockReader returns a new LazyBlockReader for the given block.
func newLazyBlockReader(block *Block, fs common.RemoteFS) (*lazyBlockReader, error) {
if block.SegmentsFormat != "1b6d" {
return nil, fmt.Errorf("unsupported segments format: %s", block.SegmentsFormat)
}
return &lazyBlockReader{
ID: block.ID,
SegmentsNum: block.SegmentsNum,
fs: fs,
}, nil
}
func (lbr *lazyBlockReader) initialize() error {
lbr.mu.Lock()
defer lbr.mu.Unlock()
if lbr.reader != nil {
return nil
}
// fetching block and parse it and store it in lbr.reader
temp, err := lbr.mkTempDir()
if err != nil {
return fmt.Errorf("failed to create temp dir: %s", err)
}
lbr.tempDirPath = temp
// TODO: replace fetchFile and writeFile with buffered IO if needed
meta, err := lbr.fetchFile(metaFilename)
if err != nil {
return err
}
if err := lbr.writeFile(temp, metaFilename, meta); err != nil {
return fmt.Errorf("failed to write meta file: %w", err)
}
idx, err := lbr.fetchFile(indexFilename)
if err != nil {
return fmt.Errorf("failed to fetch index file %q: %w", indexFilename, err)
}
if err := lbr.writeFile(temp, indexFilename, idx); err != nil {
return err
}
for i := 1; i <= lbr.SegmentsNum; i++ {
// segments formats has format 1b06d
// https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L32
chunkName := fmt.Sprintf("%06d", i)
blockChunkPath := filepath.Join("chunks", chunkName)
chunk, err := lbr.fetchFile(blockChunkPath)
if err != nil {
return fmt.Errorf("failed to fetch chunk file: %q: %w", chunkName, err)
}
if err := lbr.writeFile(temp, blockChunkPath, chunk); err != nil {
return fmt.Errorf("failed to write chunk file: %q: %s", chunkName, err)
}
}
// Set postingDecoder to nil because
// If it is nil then a default decoder is used, compatible with Prometheus v2.
pb, err := tsdb.OpenBlock(nil, temp, nil, nil)
if err != nil {
return fmt.Errorf("failed to open block %q: %w", lbr.ID, err)
}
lbr.reader = pb
return nil
}
// Index returns an IndexReader over the block's data.
func (lbr *lazyBlockReader) Index() (tsdb.IndexReader, error) {
if err := lbr.initialize(); err != nil {
return nil, err
}
return lbr.reader.Index()
}
// Chunks returns a ChunkReader over the block's data.
func (lbr *lazyBlockReader) Chunks() (tsdb.ChunkReader, error) {
if err := lbr.initialize(); err != nil {
return nil, err
}
return lbr.reader.Chunks()
}
// Tombstones returns a tombstones.Reader over the block's deleted data.
func (lbr *lazyBlockReader) Tombstones() (tombstones.Reader, error) {
if err := lbr.initialize(); err != nil {
return nil, err
}
return lbr.reader.Tombstones()
}
// Meta provides meta information about the block reader.
func (lbr *lazyBlockReader) Meta() tsdb.BlockMeta {
if err := lbr.initialize(); err != nil {
lbr.err = fmt.Errorf("cannot get BlockMeta: %w", err)
return tsdb.BlockMeta{}
}
return lbr.reader.Meta()
}
// Size returns the number of bytes that the block takes up on disk.
func (lbr *lazyBlockReader) Size() int64 {
if err := lbr.initialize(); err != nil {
lbr.err = fmt.Errorf("error get Size of the block: %s, return zero size", err)
return 0
}
return lbr.reader.Size()
}
// Err returns the last error that occurred on the block reader.
func (lbr *lazyBlockReader) Err() error {
return lbr.err
}
// Close closes block and releases all resources
func (lbr *lazyBlockReader) Close() error {
lbr.mu.Lock()
defer lbr.mu.Unlock()
if lbr.reader == nil {
return nil
}
err := lbr.reader.Close()
if err := os.RemoveAll(lbr.tempDirPath); err != nil {
log.Printf("failed to remove temp dir: %s", err)
}
lbr.reader = nil
lbr.tempDirPath = ""
return err
}
func (lbr *lazyBlockReader) mkTempDir() (string, error) {
temp, err := os.MkdirTemp("", lbr.ID.String())
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %s", err)
}
err = os.Mkdir(filepath.Join(temp, "chunks"), os.ModePerm)
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %s", err)
}
return temp, nil
}
func (lbr *lazyBlockReader) fetchFile(filePath string) ([]byte, error) {
blockID := lbr.ID.String()
blockPath := filepath.Join(blockID, filePath)
has, err := lbr.fs.HasFile(blockPath)
if err != nil {
return nil, err
}
if !has {
return nil, fmt.Errorf("block meta %s not found", blockID)
}
return lbr.fs.ReadFile(blockPath)
}
func (lbr *lazyBlockReader) writeFile(folder string, filename string, file []byte) error {
fileName := filepath.Join(folder, filename)
return os.WriteFile(fileName, file, os.ModePerm)
}

View File

@@ -1,238 +0,0 @@
package mimir
import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"log"
"github.com/oklog/ulid/v2"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/tsdb"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
utils "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
)
const (
bucketIndex = "bucket-index.json"
bucketIndexCompressedFilename = bucketIndex + ".gz"
metaFilename = "meta.json"
indexFilename = "index"
)
// BlockDeletionMark holds the information about a block's deletion mark in the index.
// This type was copied from the mimir repository https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L234.
type BlockDeletionMark struct {
// Block ID.
ID ulid.ULID `json:"block_id"`
// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
DeletionTime int64 `json:"deletion_time"`
}
// Block holds the information about a block in the index.
// This is a partial implementation of the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L73
type Block struct {
// Block ID.
ID ulid.ULID `json:"block_id"`
// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
MinTime int64 `json:"min_time"`
MaxTime int64 `json:"max_time"`
// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
// in the block.
SegmentsFormat string `json:"segments_format,omitempty"`
SegmentsNum int `json:"segments_num,omitempty"`
}
// Index contains all known blocks and markers of a tenant.
// This is a partial implementation pof the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L36
type Index struct {
// Version of the index format.
Version int `json:"version"`
// List of complete blocks (partial blocks are excluded from the index).
Blocks []*Block `json:"blocks"`
}
// Config contains a list of params needed
// for reading mimir snapshots
type Config struct {
// Path to remote storage bucket
Path string
// TenantID is the tenant id for the storage
TenantID string
Filter Filter
CredsFilePath string
ConfigFilePath string
ConfigProfile string
CustomS3Endpoint string
S3ForcePathStyle bool
S3TLSInsecureSkipVerify bool
SSEKMSKeyID string
SSEAlgorithm string
}
// Filter contains configuration for filtering
// the timeseries
type Filter struct {
TimeMin string
TimeMax string
Label string
LabelValue string
}
// Client is a wrapper over Prometheus tsdb.DBReader
type Client struct {
common.RemoteFS
filter filter
}
type filter struct {
min, max int64
label string
labelValue string
}
func (f filter) inRange(minTime, maxTime int64) bool {
fmin, fmax := f.min, f.max
if minTime == 0 {
fmin = minTime
}
if fmax == 0 {
fmax = maxTime
}
return minTime <= fmax && fmin <= maxTime
}
// NewClient creates and validates new Client
// with given Config
func NewClient(ctx context.Context, cfg Config) (*Client, error) {
if cfg.Path == "" {
return nil, fmt.Errorf("path cannot be empty")
}
if cfg.TenantID != "" {
cfg.Path = fmt.Sprintf("%s/%s", cfg.Path, cfg.TenantID)
}
var c Client
rfs, err := newRemoteFS(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("cannot parse `-src`=%q: %w", cfg.Path, err)
}
c.RemoteFS = rfs
timeMin, err := utils.ParseTime(cfg.Filter.TimeMin)
if err != nil {
return nil, fmt.Errorf("failed to parse min time in filter: %s", err)
}
timeMax, err := utils.ParseTime(cfg.Filter.TimeMax)
if err != nil {
return nil, fmt.Errorf("failed to parse max time in filter: %s", err)
}
c.filter = filter{
min: timeMin.UnixMilli(),
max: timeMax.UnixMilli(),
label: cfg.Filter.Label,
labelValue: cfg.Filter.LabelValue,
}
return &c, nil
}
// Explore a fetches bucket-index.json file from a remote storage or local filesystem
// and filter blocks via the defined time range, but does not take into account label filters.
func (c *Client) Explore() ([]tsdb.BlockReader, error) {
log.Printf("Fetching blocks from remote storage")
indexFile, err := c.fetchIndexFile()
if err != nil {
return nil, fmt.Errorf("failed to fetch index file: %s", err)
}
var blocksToImport []tsdb.BlockReader
for _, block := range indexFile.Blocks {
if !c.filter.inRange(block.MinTime, block.MaxTime) {
// Skipping block outside of time range
continue
}
if block.ID.String() == "" {
continue
}
lazyBlockReader, err := newLazyBlockReader(block, c.RemoteFS)
if err != nil {
return nil, fmt.Errorf("failed to create lazy block reader: %s", err)
}
blocksToImport = append(blocksToImport, lazyBlockReader)
}
return blocksToImport, nil
}
// Read reads the given BlockReader according to configured
// time and label filters.
func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error) {
meta := block.Meta()
if b, ok := block.(*lazyBlockReader); ok && b.Err() != nil {
return nil, fmt.Errorf("failed to read block: %s", b.Err())
}
if meta.ULID.String() == "" {
return nil, fmt.Errorf("unexpected block without id")
}
minTime, maxTime := meta.MinTime, meta.MaxTime
if c.filter.min != 0 {
minTime = c.filter.min
}
if c.filter.max != 0 {
maxTime = c.filter.max
}
q, err := tsdb.NewBlockQuerier(block, minTime, maxTime)
if err != nil {
return nil, err
}
ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
return &prometheus.CloseableSeriesSet{SeriesSet: ss, Close: q.Close}, nil
}
func (c *Client) fetchIndexFile() (*Index, error) {
has, err := c.HasFile(bucketIndexCompressedFilename)
if err != nil {
return nil, err
}
if !has {
return nil, fmt.Errorf("bucket-index.json.gz not found")
}
file, err := c.ReadFile(bucketIndexCompressedFilename)
if err != nil {
return nil, fmt.Errorf("failed to read bucket index: %s", err)
}
r := bytes.NewReader(file)
// Read all the content.
gzipReader, err := gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("failed to create gzip reader: %s", err)
}
var indexFile Index
err = json.NewDecoder(gzipReader).Decode(&indexFile)
if err != nil {
return nil, fmt.Errorf("failed to decode bucket index: %s", err)
}
return &indexFile, nil
}

View File

@@ -1,93 +0,0 @@
package mimir
import (
"context"
"fmt"
"path/filepath"
"strings"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/azremote"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
)
// newRemoteFS returns new remote fs from the given Config.
func newRemoteFS(ctx context.Context, cfg Config) (common.RemoteFS, error) {
if len(cfg.Path) == 0 {
return nil, fmt.Errorf("path cannot be empty")
}
n := strings.Index(cfg.Path, "://")
if n < 0 {
return nil, fmt.Errorf("missing scheme in path %q. Supported schemes: `gs://`, `s3://`, `azblob://`, `fs://`", cfg.Path)
}
scheme := cfg.Path[:n]
dir := cfg.Path[n+len("://"):]
switch scheme {
case "fs":
if !filepath.IsAbs(dir) {
return nil, fmt.Errorf("dir must be absolute; got %q", dir)
}
fsr := &fsremote.FS{
Dir: filepath.Clean(dir),
}
return fsr, nil
case "gcs", "gs":
n := strings.Index(dir, "/")
if n < 0 {
return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
}
bucket := dir[:n]
dir = dir[n:]
fsr := &gcsremote.FS{
CredsFilePath: cfg.CredsFilePath,
Bucket: bucket,
Dir: dir,
}
if err := fsr.Init(ctx); err != nil {
return nil, fmt.Errorf("cannot initialize connection to gcs: %w", err)
}
return fsr, nil
case "azblob":
n := strings.Index(dir, "/")
if n < 0 {
return nil, fmt.Errorf("missing directory on the AZBlob container %q", dir)
}
bucket := dir[:n]
dir = dir[n:]
fsr := &azremote.FS{
Container: bucket,
Dir: dir,
}
if err := fsr.Init(ctx); err != nil {
return nil, fmt.Errorf("cannot initialize connection to AZBlob: %w", err)
}
return fsr, nil
case "s3":
n := strings.Index(dir, "/")
if n < 0 {
return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
}
bucket := dir[:n]
dir = dir[n:]
fsr := &s3remote.FS{
CredsFilePath: cfg.CredsFilePath,
ConfigFilePath: cfg.ConfigFilePath,
CustomEndpoint: cfg.CustomS3Endpoint,
TLSInsecureSkipVerify: cfg.S3TLSInsecureSkipVerify,
S3ForcePathStyle: cfg.S3ForcePathStyle,
ProfileName: cfg.ConfigProfile,
Bucket: bucket,
Dir: dir,
SSEKMSKeyId: cfg.SSEKMSKeyID,
SSEAlgorithm: s3remote.StringToEncryptionAlgorithm(cfg.SSEAlgorithm),
}
if err := fsr.Init(ctx); err != nil {
return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)
}
return fsr, nil
default:
return nil, fmt.Errorf("unsupported scheme %q", scheme)
}
}

View File

@@ -3,7 +3,6 @@ package main
import (
"context"
"fmt"
"io"
"log"
"strings"
"sync"
@@ -19,17 +18,10 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
)
// Runner is an interface for fetching and reading
// snapshot blocks
type Runner interface {
Explore() ([]tsdb.BlockReader, error)
Read(context.Context, tsdb.BlockReader) (*prometheus.CloseableSeriesSet, error)
}
type prometheusProcessor struct {
// Runner fetches and reads
// prometheus client fetches and reads
// snapshot blocks
cl Runner
cl *prometheus.Client
// importer performs import requests
// for timeseries data returned from
// snapshot blocks
@@ -56,7 +48,7 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
return nil
}
if err := pp.processBlocks(ctx, blocks); err != nil {
if err := pp.processBlocks(blocks); err != nil {
return fmt.Errorf("migration failed: %s", err)
}
@@ -65,17 +57,11 @@ func (pp *prometheusProcessor) run(ctx context.Context) error {
return nil
}
func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error {
css, err := pp.cl.Read(ctx, b)
func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
ss, err := pp.cl.Read(b)
if err != nil {
return fmt.Errorf("failed to read block: %s", err)
}
defer func() {
if err := css.Close(); err != nil {
log.Printf("cannot close SeriesSet for block: %q : %s\n", b.Meta().ULID, err)
}
}()
ss := css.SeriesSet
var it chunkenc.Iterator
for ss.Next() {
var name string
@@ -128,7 +114,7 @@ func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error
return ss.Err()
}
func (pp *prometheusProcessor) processBlocks(ctx context.Context, blocks []tsdb.BlockReader) error {
func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
promBlocksTotal.Add(len(blocks))
bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
if err := barpool.Start(); err != nil {
@@ -144,16 +130,11 @@ func (pp *prometheusProcessor) processBlocks(ctx context.Context, blocks []tsdb.
for range pp.cc {
wg.Go(func() {
for br := range blockReadersCh {
if err := pp.do(ctx, br); err != nil {
if err := pp.do(br); err != nil {
promErrorsTotal.Inc()
errCh <- fmt.Errorf("cannot read block %q: %s", br.Meta().ULID, err)
errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
return
}
if cb, ok := br.(io.Closer); ok {
if err := cb.Close(); err != nil {
errCh <- fmt.Errorf("cannot close block: %q: %w", br.Meta().ULID, err)
}
}
promBlocksProcessed.Inc()
bar.Increment()
}

View File

@@ -8,8 +8,6 @@ import (
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
)
// Config contains a list of params needed
@@ -62,13 +60,13 @@ func NewClient(cfg Config) (*Client, error) {
return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
}
c := &Client{DBReadOnly: db}
timeMin, timeMax, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
if err != nil {
return nil, fmt.Errorf("failed to parse time in filter: %s", err)
}
c.filter = filter{
min: timeMin,
max: timeMax,
min: minTime,
max: maxTime,
label: cfg.Filter.Label,
labelValue: cfg.Filter.LabelValue,
}
@@ -85,7 +83,7 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
if err != nil {
return nil, fmt.Errorf("failed to fetch blocks: %s", err)
}
s := &vmctlutil.Stats{
s := &Stats{
Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
Blocks: len(blocks),
}
@@ -110,15 +108,9 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
return blocksToImport, nil
}
// CloseableSeriesSet defines a SeriesSet with Close method
type CloseableSeriesSet struct {
SeriesSet storage.SeriesSet
Close func() error
}
// Read reads the given BlockReader according to configured
// time and label filters.
func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*CloseableSeriesSet, error) {
func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
minTime, maxTime := block.Meta().MinTime, block.Meta().MaxTime
if c.filter.min != 0 {
minTime = c.filter.min
@@ -130,8 +122,8 @@ func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (*CloseableSe
if err != nil {
return nil, err
}
ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
return &CloseableSeriesSet{ss, q.Close}, nil
ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
return ss, nil
}
func parseTime(start, end string) (int64, int64, error) {

View File

@@ -1,4 +1,4 @@
package vmctlutil
package prometheus
import (
"fmt"
@@ -18,7 +18,7 @@ type Stats struct {
// String returns string representation for s.
func (s Stats) String() string {
str := fmt.Sprintf("Snapshot stats:\n"+
str := fmt.Sprintf("Prometheus snapshot stats:\n"+
" blocks found: %d;\n"+
" blocks skipped by time filter: %d;\n"+
" min time: %d (%v);\n"+

View File

@@ -2,16 +2,13 @@
"math"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
) %}
{% stripspace %}
// Federate writes rs in /federate format.
// See https://prometheus.io/docs/prometheus/latest/federation/
{% func Federate(rs *netstorage.Result, escapeScheme string) %}
{% func Federate(rs *netstorage.Result) %}
{% code
values := rs.Values
timestamps := rs.Timestamps
@@ -27,54 +24,10 @@
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185
{% endcomment %}
{% return %}
{% endif %}
{% switch escapeScheme %}
{% case federateEscapeSchemeUTF8 %}
{%= prometheusFederateMetricNameUTF8(&rs.MetricName) %}{% space %}
{% case federateEscapeSchemeUnderscore %}
{%= prometheusFederateMetricNameEscapeUnderscore(&rs.MetricName) %}{% space %}
{% case "" %}
{%= prometheusMetricName(&rs.MetricName) %}{% space %}
{% endswitch %}
{% endif %}
{%= prometheusMetricName(&rs.MetricName) %}{% space %}
{%f= lastValue %}{% space %}
{%dl= timestamps[len(timestamps)-1] %}{% newline %}
{% endfunc %}
{% func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) %}
{%s= promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)) %}
{% if len(mn.Tags) > 0 %}
{
{% code tags := mn.Tags %}
{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)) %}={%= escapePrometheusLabel(tags[0].Value) %}
{% code tags = tags[1:] %}
{% for i := range tags %}
{% code tag := &tags[i] %}
,{%s= promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)) %}={%= escapePrometheusLabel(tag.Value) %}
{% endfor %}
}
{% endif %}
{% endfunc %}
{% func prometheusFederateMetricNameUTF8(mn *storage.MetricName) %}
{
{%= escapePrometheusLabel(mn.MetricGroup) %}
{% if len(mn.Tags) > 0 %}
,
{% code tags := mn.Tags %}
{%= escapePrometheusLabel(tags[0].Key) %}={%= escapePrometheusLabel(tags[0].Value) %}
{% code tags = tags[1:] %}
{% for i := range tags %}
{% code tag := &tags[i] %}
,{%= escapePrometheusLabel(tag.Key) %}={%= escapePrometheusLabel(tag.Value) %}
{% endfor %}
{% endif %}
}
{% endfunc %}
{% endstripspace %}

View File

@@ -9,241 +9,82 @@ import (
"math"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
)
// Federate writes rs in /federate format.// See https://prometheus.io/docs/prometheus/latest/federation/
//line app/vmselect/prometheus/federate.qtpl:14
//line app/vmselect/prometheus/federate.qtpl:11
import (
qtio422016 "io"
qt422016 "github.com/valyala/quicktemplate"
)
//line app/vmselect/prometheus/federate.qtpl:14
//line app/vmselect/prometheus/federate.qtpl:11
var (
_ = qtio422016.Copy
_ = qt422016.AcquireByteBuffer
)
//line app/vmselect/prometheus/federate.qtpl:14
func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result, escapeScheme string) {
//line app/vmselect/prometheus/federate.qtpl:16
//line app/vmselect/prometheus/federate.qtpl:11
func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) {
//line app/vmselect/prometheus/federate.qtpl:13
values := rs.Values
timestamps := rs.Timestamps
//line app/vmselect/prometheus/federate.qtpl:19
//line app/vmselect/prometheus/federate.qtpl:16
if len(timestamps) == 0 || len(values) == 0 {
//line app/vmselect/prometheus/federate.qtpl:19
//line app/vmselect/prometheus/federate.qtpl:16
return
//line app/vmselect/prometheus/federate.qtpl:19
//line app/vmselect/prometheus/federate.qtpl:16
}
//line app/vmselect/prometheus/federate.qtpl:21
//line app/vmselect/prometheus/federate.qtpl:18
lastValue := values[len(values)-1]
//line app/vmselect/prometheus/federate.qtpl:23
//line app/vmselect/prometheus/federate.qtpl:20
if math.IsNaN(lastValue) {
//line app/vmselect/prometheus/federate.qtpl:29
//line app/vmselect/prometheus/federate.qtpl:26
return
//line app/vmselect/prometheus/federate.qtpl:30
//line app/vmselect/prometheus/federate.qtpl:27
}
//line app/vmselect/prometheus/federate.qtpl:32
switch escapeScheme {
//line app/vmselect/prometheus/federate.qtpl:33
case federateEscapeSchemeUTF8:
//line app/vmselect/prometheus/federate.qtpl:34
streamprometheusFederateMetricNameUTF8(qw422016, &rs.MetricName)
//line app/vmselect/prometheus/federate.qtpl:34
qw422016.N().S(` `)
//line app/vmselect/prometheus/federate.qtpl:36
case federateEscapeSchemeUnderscore:
//line app/vmselect/prometheus/federate.qtpl:37
streamprometheusFederateMetricNameEscapeUnderscore(qw422016, &rs.MetricName)
//line app/vmselect/prometheus/federate.qtpl:37
qw422016.N().S(` `)
//line app/vmselect/prometheus/federate.qtpl:39
case "":
//line app/vmselect/prometheus/federate.qtpl:40
streamprometheusMetricName(qw422016, &rs.MetricName)
//line app/vmselect/prometheus/federate.qtpl:40
qw422016.N().S(` `)
//line app/vmselect/prometheus/federate.qtpl:41
}
//line app/vmselect/prometheus/federate.qtpl:43
qw422016.N().F(lastValue)
//line app/vmselect/prometheus/federate.qtpl:43
//line app/vmselect/prometheus/federate.qtpl:28
streamprometheusMetricName(qw422016, &rs.MetricName)
//line app/vmselect/prometheus/federate.qtpl:28
qw422016.N().S(` `)
//line app/vmselect/prometheus/federate.qtpl:44
//line app/vmselect/prometheus/federate.qtpl:29
qw422016.N().F(lastValue)
//line app/vmselect/prometheus/federate.qtpl:29
qw422016.N().S(` `)
//line app/vmselect/prometheus/federate.qtpl:30
qw422016.N().DL(timestamps[len(timestamps)-1])
//line app/vmselect/prometheus/federate.qtpl:44
//line app/vmselect/prometheus/federate.qtpl:30
qw422016.N().S(`
`)
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
}
//line app/vmselect/prometheus/federate.qtpl:45
func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result, escapeScheme string) {
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result) {
//line app/vmselect/prometheus/federate.qtpl:31
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/federate.qtpl:45
StreamFederate(qw422016, rs, escapeScheme)
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
StreamFederate(qw422016, rs)
//line app/vmselect/prometheus/federate.qtpl:31
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
}
//line app/vmselect/prometheus/federate.qtpl:45
func Federate(rs *netstorage.Result, escapeScheme string) string {
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
func Federate(rs *netstorage.Result) string {
//line app/vmselect/prometheus/federate.qtpl:31
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/federate.qtpl:45
WriteFederate(qb422016, rs, escapeScheme)
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
WriteFederate(qb422016, rs)
//line app/vmselect/prometheus/federate.qtpl:31
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/federate.qtpl:45
//line app/vmselect/prometheus/federate.qtpl:31
return qs422016
//line app/vmselect/prometheus/federate.qtpl:45
}
//line app/vmselect/prometheus/federate.qtpl:47
func streamprometheusFederateMetricNameEscapeUnderscore(qw422016 *qt422016.Writer, mn *storage.MetricName) {
//line app/vmselect/prometheus/federate.qtpl:48
qw422016.N().S(promrelabel.SanitizeMetricName(bytesutil.ToUnsafeString(mn.MetricGroup)))
//line app/vmselect/prometheus/federate.qtpl:49
if len(mn.Tags) > 0 {
//line app/vmselect/prometheus/federate.qtpl:49
qw422016.N().S(`{`)
//line app/vmselect/prometheus/federate.qtpl:51
tags := mn.Tags
//line app/vmselect/prometheus/federate.qtpl:52
qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tags[0].Key)))
//line app/vmselect/prometheus/federate.qtpl:52
qw422016.N().S(`=`)
//line app/vmselect/prometheus/federate.qtpl:52
streamescapePrometheusLabel(qw422016, tags[0].Value)
//line app/vmselect/prometheus/federate.qtpl:53
tags = tags[1:]
//line app/vmselect/prometheus/federate.qtpl:54
for i := range tags {
//line app/vmselect/prometheus/federate.qtpl:55
tag := &tags[i]
//line app/vmselect/prometheus/federate.qtpl:55
qw422016.N().S(`,`)
//line app/vmselect/prometheus/federate.qtpl:56
qw422016.N().S(promrelabel.SanitizeLabelName(bytesutil.ToUnsafeString(tag.Key)))
//line app/vmselect/prometheus/federate.qtpl:56
qw422016.N().S(`=`)
//line app/vmselect/prometheus/federate.qtpl:56
streamescapePrometheusLabel(qw422016, tag.Value)
//line app/vmselect/prometheus/federate.qtpl:57
}
//line app/vmselect/prometheus/federate.qtpl:57
qw422016.N().S(`}`)
//line app/vmselect/prometheus/federate.qtpl:59
}
//line app/vmselect/prometheus/federate.qtpl:60
}
//line app/vmselect/prometheus/federate.qtpl:60
func writeprometheusFederateMetricNameEscapeUnderscore(qq422016 qtio422016.Writer, mn *storage.MetricName) {
//line app/vmselect/prometheus/federate.qtpl:60
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/federate.qtpl:60
streamprometheusFederateMetricNameEscapeUnderscore(qw422016, mn)
//line app/vmselect/prometheus/federate.qtpl:60
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/federate.qtpl:60
}
//line app/vmselect/prometheus/federate.qtpl:60
func prometheusFederateMetricNameEscapeUnderscore(mn *storage.MetricName) string {
//line app/vmselect/prometheus/federate.qtpl:60
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/federate.qtpl:60
writeprometheusFederateMetricNameEscapeUnderscore(qb422016, mn)
//line app/vmselect/prometheus/federate.qtpl:60
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/federate.qtpl:60
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/federate.qtpl:60
return qs422016
//line app/vmselect/prometheus/federate.qtpl:60
}
//line app/vmselect/prometheus/federate.qtpl:62
func streamprometheusFederateMetricNameUTF8(qw422016 *qt422016.Writer, mn *storage.MetricName) {
//line app/vmselect/prometheus/federate.qtpl:62
qw422016.N().S(`{`)
//line app/vmselect/prometheus/federate.qtpl:64
streamescapePrometheusLabel(qw422016, mn.MetricGroup)
//line app/vmselect/prometheus/federate.qtpl:65
if len(mn.Tags) > 0 {
//line app/vmselect/prometheus/federate.qtpl:65
qw422016.N().S(`,`)
//line app/vmselect/prometheus/federate.qtpl:67
tags := mn.Tags
//line app/vmselect/prometheus/federate.qtpl:68
streamescapePrometheusLabel(qw422016, tags[0].Key)
//line app/vmselect/prometheus/federate.qtpl:68
qw422016.N().S(`=`)
//line app/vmselect/prometheus/federate.qtpl:68
streamescapePrometheusLabel(qw422016, tags[0].Value)
//line app/vmselect/prometheus/federate.qtpl:69
tags = tags[1:]
//line app/vmselect/prometheus/federate.qtpl:70
for i := range tags {
//line app/vmselect/prometheus/federate.qtpl:71
tag := &tags[i]
//line app/vmselect/prometheus/federate.qtpl:71
qw422016.N().S(`,`)
//line app/vmselect/prometheus/federate.qtpl:72
streamescapePrometheusLabel(qw422016, tag.Key)
//line app/vmselect/prometheus/federate.qtpl:72
qw422016.N().S(`=`)
//line app/vmselect/prometheus/federate.qtpl:72
streamescapePrometheusLabel(qw422016, tag.Value)
//line app/vmselect/prometheus/federate.qtpl:73
}
//line app/vmselect/prometheus/federate.qtpl:74
}
//line app/vmselect/prometheus/federate.qtpl:74
qw422016.N().S(`}`)
//line app/vmselect/prometheus/federate.qtpl:76
}
//line app/vmselect/prometheus/federate.qtpl:76
func writeprometheusFederateMetricNameUTF8(qq422016 qtio422016.Writer, mn *storage.MetricName) {
//line app/vmselect/prometheus/federate.qtpl:76
qw422016 := qt422016.AcquireWriter(qq422016)
//line app/vmselect/prometheus/federate.qtpl:76
streamprometheusFederateMetricNameUTF8(qw422016, mn)
//line app/vmselect/prometheus/federate.qtpl:76
qt422016.ReleaseWriter(qw422016)
//line app/vmselect/prometheus/federate.qtpl:76
}
//line app/vmselect/prometheus/federate.qtpl:76
func prometheusFederateMetricNameUTF8(mn *storage.MetricName) string {
//line app/vmselect/prometheus/federate.qtpl:76
qb422016 := qt422016.AcquireByteBuffer()
//line app/vmselect/prometheus/federate.qtpl:76
writeprometheusFederateMetricNameUTF8(qb422016, mn)
//line app/vmselect/prometheus/federate.qtpl:76
qs422016 := string(qb422016.B)
//line app/vmselect/prometheus/federate.qtpl:76
qt422016.ReleaseByteBuffer(qb422016)
//line app/vmselect/prometheus/federate.qtpl:76
return qs422016
//line app/vmselect/prometheus/federate.qtpl:76
//line app/vmselect/prometheus/federate.qtpl:31
}

View File

@@ -8,15 +8,15 @@ import (
)
func TestFederate(t *testing.T) {
f := func(rs *netstorage.Result, escapeScheme string, expectedResult string) {
f := func(rs *netstorage.Result, expectedResult string) {
t.Helper()
result := Federate(rs, escapeScheme)
result := Federate(rs)
if result != expectedResult {
t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, expectedResult)
}
}
f(&netstorage.Result{}, ``, ``)
f(&netstorage.Result{}, ``)
f(&netstorage.Result{
MetricName: storage.MetricName{
@@ -39,60 +39,5 @@ func TestFederate(t *testing.T) {
},
Values: []float64{1.23},
Timestamps: []int64{123},
}, ``, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
f(&netstorage.Result{
MetricName: storage.MetricName{
MetricGroup: []byte("foo.bar"),
Tags: []storage.Tag{
{
Key: []byte("some.!other"),
Value: []byte("value.unchanged!."),
},
{
Key: []byte("qqq"),
Value: []byte("\\"),
},
{
Key: []byte("!key"),
Value: []byte("value"),
},
{
Key: []byte("abc"),
// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
Value: []byte("a<b\"\\c"),
},
},
},
Values: []float64{1.23},
Timestamps: []int64{123},
}, federateEscapeSchemeUnderscore, `foo_bar{some__other="value.unchanged!.",qqq="\\",_key="value",abc="a<b\"\\c"} 1.23 123`+"\n")
f(&netstorage.Result{
MetricName: storage.MetricName{
MetricGroup: []byte("foo.bar"),
Tags: []storage.Tag{
{
Key: []byte("some.!other"),
Value: []byte("value.unchanged!."),
},
{
Key: []byte("qqq"),
Value: []byte("\\"),
},
{
Key: []byte("!key"),
Value: []byte("value"),
},
{
Key: []byte(`ab"c`),
// Verify that < isn't encoded. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5431
Value: []byte("a<b\"\\c"),
},
},
},
Values: []float64{1.23},
Timestamps: []int64{123},
}, federateEscapeSchemeUTF8, `{"foo.bar","some.!other"="value.unchanged!.","qqq"="\\","!key"="value","ab\"c"="a<b\"\\c"} 1.23 123`+"\n")
}, `foo{a="b",qqq="\\",abc="a<b\"\\c"} 1.23 123`+"\n")
}

View File

@@ -9,17 +9,16 @@ import (
)
func BenchmarkFederate(b *testing.B) {
rs := &netstorage.Result{
MetricName: storage.MetricName{
MetricGroup: []byte("foo_bar_?_._bazaaaa_total"),
MetricGroup: []byte("foo_bar_bazaaaa_total"),
Tags: []storage.Tag{
{
Key: []byte("instance:job"),
Key: []byte("instance"),
Value: []byte("foobarbaz:2344"),
},
{
Key: []byte("job.name"),
Key: []byte("job"),
Value: []byte("aaabbbccc"),
},
},
@@ -28,22 +27,12 @@ func BenchmarkFederate(b *testing.B) {
Timestamps: []int64{1234567890},
}
f := func(name, escapeScheme string) {
b.Helper()
b.Run(name, func(b *testing.B) {
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
var bb bytes.Buffer
for pb.Next() {
bb.Reset()
WriteFederate(&bb, rs, escapeScheme)
}
})
})
}
f("without escape", "")
f("allow-utf-8", federateEscapeSchemeUTF8)
f("legacy-underscore", federateEscapeSchemeUnderscore)
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
var bb bytes.Buffer
for pb.Next() {
bb.Reset()
WriteFederate(&bb, rs)
}
})
}

View File

@@ -108,11 +108,6 @@ func PrettifyQuery(w http.ResponseWriter, r *http.Request) {
_ = bw.Flush()
}
const (
federateEscapeSchemeUnderscore = "underscore"
federateEscapeSchemeUTF8 = "utf-8"
)
// FederateHandler implements /federate . See https://prometheus.io/docs/prometheus/latest/federation/
func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error {
defer federateDuration.UpdateDuration(startTime)
@@ -137,21 +132,6 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
return fmt.Errorf("cannot fetch data for %q: %w", sq, err)
}
// add best-effort format negotiation
// modern version of Prometheus always set allow-utf-8 in order to properly parse utf-8 names and labels
// prometheus below v3 uses underscore escaping by default and it's the most common standard
var escapeScheme string
accept := r.Header.Get("Accept")
if len(accept) > 0 && strings.Contains(accept, "allow-utf-8") {
escapeScheme = federateEscapeSchemeUTF8
}
// try fallback to legacy underscore escaping if needed for Prometheus only,
// it's not widely used after Prometheus v3.0 release
// most of the Prometheus scrapers already use allow-utf-8 header
isPrometheus := strings.HasPrefix(r.UserAgent(), "Prometheus")
if len(escapeScheme) == 0 && isPrometheus {
escapeScheme = federateEscapeSchemeUnderscore
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
@@ -161,7 +141,7 @@ func FederateHandler(startTime time.Time, w http.ResponseWriter, r *http.Request
return err
}
bb := sw.getBuffer(workerID)
WriteFederate(bb, rs, escapeScheme)
WriteFederate(bb, rs)
return sw.maybeFlushBuffer(bb)
})
if err == nil {

View File

@@ -37,7 +37,7 @@
<meta property="og:title" content="UI for VictoriaMetrics">
<meta property="og:url" content="https://victoriametrics.com/">
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
<script type="module" crossorigin src="./assets/index-U3iNn2Tx.js"></script>
<script type="module" crossorigin src="./assets/index-BjJ7fDL7.js"></script>
<link rel="modulepreload" crossorigin href="./assets/rolldown-runtime-COnpUsM8.js">
<link rel="modulepreload" crossorigin href="./assets/vendor-C8Kwp93_.js">
<link rel="stylesheet" crossorigin href="./assets/vendor-CnsZ1jie.css">

View File

@@ -1,4 +1,4 @@
FROM golang:1.26.4 AS build-web-stage
FROM golang:1.26.3 AS build-web-stage
COPY build /build
WORKDIR /build

View File

@@ -1,7 +1,7 @@
export const seriesFetchedWarning = `No match!
export const seriesFetchedWarning = `No match!
This query hasn't selected any time series from database.
Either the requested metrics are missing in the database,
or there is a typo in series selector.`;
export const partialWarning = `The shown results are marked as PARTIAL.
The result is marked as partial if one or more storage nodes failed to respond to the query.`;
The result is marked as partial if one or more vmstorage nodes failed to respond to the query.`;

View File

@@ -71,7 +71,7 @@ const RulesHeader = ({
<TextField
label="Search"
value={search}
placeholder="Filter by group or rule name"
placeholder="Filter by rule, name or labels"
startIcon={<SearchIcon />}
onChange={onChangeSearch}
/>

View File

@@ -79,25 +79,24 @@ type PrometheusWriteQuerier interface {
// QueryOpts contains various params used for querying or ingesting data
type QueryOpts struct {
Tenant string
Timeout string
Start string
End string
Time string
Step string
ExtraFilters []string
ExtraLabels []string
Trace string
ReduceMemUsage string
MaxLookback string
LatencyOffset string
Format string
NoCache string
Headers http.Header
From string
Until string
StorageStep string
DenyPartialResponse string
Tenant string
Timeout string
Start string
End string
Time string
Step string
ExtraFilters []string
ExtraLabels []string
Trace string
ReduceMemUsage string
MaxLookback string
LatencyOffset string
Format string
NoCache string
Headers http.Header
From string
Until string
StorageStep string
}
func (qos *QueryOpts) getHeaders() http.Header {
@@ -133,7 +132,6 @@ func (qos *QueryOpts) asURLValues() url.Values {
addNonEmpty("from", qos.From)
addNonEmpty("until", qos.Until)
addNonEmpty("storage_step", qos.StorageStep)
addNonEmpty("deny_partial_response", qos.DenyPartialResponse)
return uv
}

View File

@@ -1015,42 +1015,35 @@ func testGroupSkipSlowReplicas(tc *apptest.TestCase, opts *testGroupReplicationO
func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOpts) {
t := tc.T()
assertSeries := func(app *apptest.Vmselect, denyPartialResponse string, want *apptest.PrometheusAPIV1SeriesResponse) {
assertSeries := func(app *apptest.Vmselect, wantPartial bool) {
t.Helper()
tc.Assert(&apptest.AssertOptions{
Msg: "unexpected /api/v1/series response",
Got: func() any {
return app.PrometheusAPIV1Series(t, `{__name__=~".*"}`, apptest.QueryOpts{
Start: "2024-01-01T00:00:00Z",
End: "2024-01-31T00:00:00Z",
DenyPartialResponse: denyPartialResponse,
Start: "2024-01-01T00:00:00Z",
End: "2024-01-31T00:00:00Z",
}).Sort()
},
Want: want,
Want: &apptest.PrometheusAPIV1SeriesResponse{
Status: "success",
IsPartial: wantPartial,
},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data", "Error"),
cmpopts.IgnoreFields(apptest.PrometheusAPIV1SeriesResponse{}, "Data"),
},
})
}
allowPartialResponse := ""
denyPartialResponse := "1"
mustReturnPartialResponse := &apptest.PrometheusAPIV1SeriesResponse{
Status: "success",
IsPartial: true,
}
mustReturnFullResponse := &apptest.PrometheusAPIV1SeriesResponse{
Status: "success",
IsPartial: false,
}
mustReturnPartialResponse := true
mustReturnFullResponse := false
// All vmstorage replicas are available so both vmselects must return full
// response.
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselect, mustReturnFullResponse)
assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
// Stop groupRF-1 vmstorage nodes in first group.
//
@@ -1060,10 +1053,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
// about the replication factor and therefore they must still be able to
// return full dataset.
opts.c.storageGroups[0].stopNodes(tc, opts.groupRF-1)
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselect, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
assertSeries(opts.c.vmselectGlobalRF, mustReturnFullResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
// Stop groupRF-1 vmstorages in the remaining groups.
//
@@ -1073,10 +1066,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
for g := 1; g < len(opts.c.storageGroups); g++ {
opts.c.storageGroups[g].stopNodes(tc, opts.groupRF-1)
}
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselect, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, mustReturnFullResponse)
assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
// Stop one more vmstorage in the first group.
//
@@ -1084,10 +1077,10 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
// because it is unaware of replication across groups. vmselectGroupGlobalRF
// will continue retuning full dataset.
opts.c.storageGroups[0].stopNodes(tc, 1)
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselect, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
// Stop one more vmstoarge in remaining globarRF-1 groups.
//
@@ -1096,56 +1089,19 @@ func testGroupPartialResponse(tc *apptest.TestCase, opts *testGroupReplicationOp
for g := 1; g < opts.globalRF-1; g++ {
opts.c.storageGroups[g].stopNodes(tc, 1)
}
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnFullResponse)
assertSeries(opts.c.vmselect, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnFullResponse)
// Stop one more vmstoarge in one more group.
//
// vmselectGroupGlobalRF must now return partial dataset.
opts.c.storageGroups[opts.globalRF].stopNodes(tc, 1)
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
// Stop all the remaining vmstorage nodes except a single node.
//
// At this point vmselects still must be able to return partial response
// because at least one vmstorage node has successfully returned results.
n := len(opts.c.storageGroups[0].vmstorages)
opts.c.storageGroups[0].stopNodes(tc, n-1)
for g := 1; g < len(opts.c.storageGroups); g++ {
n := len(opts.c.storageGroups[g].vmstorages)
opts.c.storageGroups[g].stopNodes(tc, n)
}
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnPartialResponse)
mustReturnUnavailableError := &apptest.PrometheusAPIV1SeriesResponse{
Status: "error",
ErrorType: "503",
}
// vmselects must return an error for the same request when partial
// responses are denied explicitly.
assertSeries(opts.c.vmselect, denyPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselectGroupRF, denyPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselectGlobalRF, denyPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselectGroupGlobalRF, denyPartialResponse, mustReturnUnavailableError)
// Stop the last remaining vmstorage node.
//
// vmselects must return an error when there are no successful vmstorage
// responses.
opts.c.storageGroups[0].stopNodes(tc, 1)
assertSeries(opts.c.vmselect, allowPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselectGroupRF, allowPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselectGlobalRF, allowPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselectGroupGlobalRF, allowPartialResponse, mustReturnUnavailableError)
assertSeries(opts.c.vmselect, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGlobalRF, mustReturnPartialResponse)
assertSeries(opts.c.vmselectGroupGlobalRF, mustReturnPartialResponse)
}
// TestClusterReplication_PartialResponseMultitenant checks how vmselect handles some

View File

@@ -1,51 +0,0 @@
{
"ulid": "01JFJBS3YP1SHZ3PJQ6HK76EC3",
"minTime": 1734709200000,
"maxTime": 1734709320000,
"stats": {
"numSamples": 400,
"numSeries": 100,
"numChunks": 100
},
"compaction": {
"level": 1,
"sources": [
"01JFJBS3YP1SHZ3PJQ6HK76EC3"
],
"parents": [
{
"ulid": "00000000000000000000000000",
"minTime": 0,
"maxTime": 0
}
],
"hints": [
"from-out-of-order"
]
},
"version": 1,
"out_of_order": false,
"thanos": {
"labels": {},
"downsample": {
"resolution": 0
},
"source": "receive",
"segment_files": [
"000001"
],
"files": [
{
"rel_path": "chunks/000001",
"size_bytes": 4808
},
{
"rel_path": "index",
"size_bytes": 55021
},
{
"rel_path": "meta.json"
}
]
}
}

File diff suppressed because one or more lines are too long

View File

@@ -1,139 +0,0 @@
package tests
import (
"encoding/json"
"fmt"
"io"
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
)
const (
testMimirPath = "testdata/mimir-tsdb"
expectedMimirResponseFile = "./testdata/mimir-tsdb/expected_response.json"
)
func TestSingleVmctlMimirProtocol(t *testing.T) {
fs.MustRemoveDir(t.Name())
tc := apptest.NewTestCase(t)
defer tc.Stop()
vmsingleDst := tc.MustStartDefaultVmsingle()
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
dir, err := os.Getwd()
if err != nil {
t.Fatalf("cannot get current working directory: %s", err)
}
path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
vmctlFlags := []string{
`mimir`,
`--mimir-tenant-id=anonymous`,
`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
`--mimir-custom-s3-endpoint=http://localhost:9000`,
`--mimir-path=` + path,
`--vm-addr=` + vmAddr,
`--disable-progress-bar=true`,
`--vm-concurrency=6`,
`--mimir-concurrency=6`,
}
testMimirProtocol(tc, vmsingleDst, vmctlFlags)
}
func TestClusterVmctlMimirProtocol(t *testing.T) {
fs.MustRemoveDir(t.Name())
tc := apptest.NewTestCase(t)
defer tc.Stop()
cluster := tc.MustStartDefaultCluster()
vmAddr := fmt.Sprintf("http://%s/", cluster.Vminsert.HTTPAddr())
dir, err := os.Getwd()
if err != nil {
t.Fatalf("cannot get current working directory: %s", err)
}
path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
vmctlFlags := []string{
`mimir`,
`--mimir-tenant-id=anonymous`,
`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
`--mimir-custom-s3-endpoint=http://localhost:9000`,
`--mimir-path=` + path,
`--vm-addr=` + vmAddr,
`--disable-progress-bar=true`,
`--vm-concurrency=6`,
`--mimir-concurrency=6`,
}
testMimirProtocol(tc, cluster, vmctlFlags)
}
func testMimirProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
t := tc.T()
t.Helper()
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
// test for empty data request
got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
Step: "5m",
Time: "2025-06-02T17:14:00Z",
})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
tc.MustStartVmctl("vmctl", vmctlFlags)
sut.ForceFlush(t)
// open the expected series response file
file, err := os.Open(expectedMimirResponseFile)
if err != nil {
t.Fatalf("cannot open expected series response file: %s", err)
}
defer file.Close()
bytes, err := io.ReadAll(file)
if err != nil {
t.Fatalf("cannot read expected series response file: %s", err)
}
var wantResponse apptest.PrometheusAPIV1QueryResponse
if err := json.Unmarshal(bytes, &wantResponse); err != nil {
t.Fatalf("cannot unmarshal expected series response file: %s", err)
}
wantResponse.Sort()
tc.Assert(&apptest.AssertOptions{
// For cluster version, we need to wait longer for the metrics to be stored
Retries: 300,
Msg: `unexpected metrics stored on vmsingle via the prometheus protocol`,
Got: func() any {
expected := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
Start: "2024-12-01T15:31:10Z",
End: "2024-12-31T15:32:20Z",
})
expected.Sort()
return expected.Data.Result
},
Want: wantResponse.Data.Result,
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}

View File

@@ -130,7 +130,7 @@
"calcs": [
"lastNotNull"
],
"fields": "/^version$/",
"fields": "/^short_version$/",
"values": false
},
"showPercentChange": false,
@@ -146,10 +146,11 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "vm_app_version{job=~\"$job\",instance=~\"$instance\"}",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "{{short_version}}",
"range": false,
"refId": "A"
}

View File

@@ -791,7 +791,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,

View File

@@ -789,7 +789,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,

View File

@@ -131,7 +131,7 @@
"calcs": [
"lastNotNull"
],
"fields": "/^version$/",
"fields": "/^short_version$/",
"values": false
},
"showPercentChange": false,
@@ -147,10 +147,11 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "vm_app_version{job=~\"$job\",instance=~\"$instance\"}",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "{{short_version}}",
"range": false,
"refId": "A"
}

View File

@@ -792,7 +792,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,

View File

@@ -790,7 +790,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,

View File

@@ -785,7 +785,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,

View File

@@ -566,7 +566,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,
@@ -2804,10 +2804,10 @@
"overrides": []
},
"gridPos": {
"h": 7,
"h": 8,
"w": 12,
"x": 0,
"y": 11
"y": 352
},
"id": 63,
"options": {
@@ -2843,113 +2843,7 @@
],
"title": "Restarts ($job)",
"type": "timeseries"
},
{
"datasource": {
"type": "victoriametrics-metrics-datasource",
"uid": "$ds"
},
"description": "Group iteration reset can be caused by irregular delays during evaluation or by the system wall clock being moved backward.\nIf it is caused by host clock changes, vmalert could generate duplicate results for the group rules, since some evaluations could be repeated.\nCheck the host clock time synchronization configuration if this happens frequently.\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 11
},
"id": 70,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.2.0",
"targets": [
{
"datasource": {
"type": "victoriametrics-metrics-datasource",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(increase(vmalert_iteration_reset_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
"interval": "1m",
"legendFormat": "({{job}}) {{group}}({{file}})",
"range": true,
"refId": "A"
}
],
"title": "Group Iteration Reset ($instance)",
"type": "timeseries"
}
}
],
"title": "Troubleshooting",
"type": "row"

View File

@@ -492,14 +492,14 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by (job, short_version)",
"format": "table",
"instant": true,
"range": false,
"refId": "A"
}
],
"title": "",
"title": "Version",
"type": "table"
},
{

View File

@@ -784,7 +784,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,

View File

@@ -565,7 +565,7 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by(job, short_version)",
"format": "table",
"instant": true,
"range": false,
@@ -2803,10 +2803,10 @@
"overrides": []
},
"gridPos": {
"h": 7,
"h": 8,
"w": 12,
"x": 0,
"y": 11
"y": 352
},
"id": 63,
"options": {
@@ -2842,113 +2842,7 @@
],
"title": "Restarts ($job)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"description": "Group iteration reset can be caused by irregular delays during evaluation or by the system wall clock being moved backward.\nIf it is caused by host clock changes, vmalert could generate duplicate results for the group rules, since some evaluations could be repeated.\nCheck the host clock time synchronization configuration if this happens frequently.\n",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 11
},
"id": 70,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.2.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "$ds"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum(increase(vmalert_iteration_reset_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, group, file) > 0",
"interval": "1m",
"legendFormat": "({{job}}) {{group}}({{file}})",
"range": true,
"refId": "A"
}
],
"title": "Group Iteration Reset ($instance)",
"type": "timeseries"
}
}
],
"title": "Troubleshooting",
"type": "row"

View File

@@ -491,14 +491,14 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum by(job, version) (label_replace(vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version!=\"\"}, \"version\", \"$1\", \"short_version\", \"(.*)\") or vm_app_version{job=~\"$job\", instance=~\"$instance\", short_version=\"\"})",
"expr": "sum(vm_app_version{job=~\"$job\", instance=~\"$instance\"}) by (job, short_version)",
"format": "table",
"instant": true,
"range": false,
"refId": "A"
}
],
"title": "",
"title": "Version",
"type": "table"
},
{

View File

@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.23.4
ROOT_IMAGE_SCRATCH ?= scratch
CERTS_IMAGE := alpine:3.23.4
GO_BUILDER_IMAGE := golang:1.26.4
GO_BUILDER_IMAGE := golang:1.26.3
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)

View File

@@ -64,18 +64,6 @@ groups:
group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups.
If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries."
- alert: GroupIterationReset
expr: increase(vmalert_iteration_reset_total[5m]) > 0
for: 5m
labels:
severity: warning
annotations:
summary: "Evaluation iteration for group {{ $labels.group }} in file {{ $labels.file }} is reset"
description: "Evaluation iteration for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\" is reset on vmalert instance {{ $labels.instance }}.
This can be caused by irregular delays during evaluation or by the system wall clock being moved backward. If it is caused by host clock changes, vmalert could
generate duplicate results for the group rules since some evaluations could be repeated. Check host clock time synchronization configurations if this happens frequently."
- alert: RemoteWriteErrors
expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
for: 15m
@@ -120,3 +108,4 @@ groups:
summary: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
Check vmalert's logs for detailed error message."

View File

@@ -6,45 +6,342 @@ build:
sitemap:
disable: true
---
**Objective**
Setup Victoria Metrics Cluster with support of multiple retention periods within one installation.
[VictoriaMetrics Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports specifying multiple retentions for distinct sets of time series and tenants. If you are an Enterprise user, [configure multiple retentions directly through retention filters](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#retention-filters) instead of following this guide.
**Enterprise Solution**
This guide explains how to set up multiple retentions using an [open-source VictoriaMetrics Cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/).
[VictoriaMetrics Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports specifying multiple retentions
for distinct sets of time series and [tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy)
via [retention filters](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#retention-filters).
## Overview
**Open Source Solution**
VictoriaMetrics retains metrics by default for 1 month. You can change data retention with the [`-retentionPeriod` command-line flag](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention), but this value applies to all time series stored on a given `vmstorage` node and cannot be customized per tenant or per metric in the open source version.
Community version of VictoriaMetrics supports only one retention period per `vmstorage` node via [-retentionPeriod](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) command-line flag.
## Multi-Retention Architecture
A multi-retention setup can be implemented by dividing a [victoriametrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) into logical groups with different retentions.
To support multiple retentions with the open source version of VictoriaMetrics cluster, you can split the cluster into several logical groups of `vmstorage` nodes, where each group is configured with a different `-retentionPeriod` and receives only the data that must follow that retention.
Example:
Setup should handle 3 different retention groups 3months, 1year and 3 years.
Solution contains 3 groups of vmstorages + vminserts and one group of vmselects. Routing is done by [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/)
by [splitting data streams](https://docs.victoriametrics.com/victoriametrics/vmagent/#splitting-data-streams-among-multiple-systems).
The [-retentionPeriod](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) sets how long to keep the metrics.
The diagram below shows a proposed solution
Each storage group is connected to a separate `vminsert`, while a shared `vmselect` layer queries across all storage groups so that dashboards and alerts continue to see a single logical VictoriaMetrics backend.
![Setup](setup.webp)
**Implementation Details**
In the example used throughout this guide, the cluster is divided into three groups:
1. Groups of vminserts A know about only vmstorages A and this is explicitly specified via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup).
1. Groups of vminserts B know about only vmstorages B and this is explicitly specified via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup).
1. Groups of vminserts C know about only vmstorages C and this is explicitly specified via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup).
1. vmselect reads data from all vmstorage nodes via `-storageNode` [configuration](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#cluster-setup)
with [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) setting equal to vmagent's scrape interval or minimum interval between collected samples.
1. vmagent routes incoming metrics to the given set of `vminsert` nodes using relabeling rules specified at `-remoteWrite.urlRelabelConfig` [configuration](https://docs.victoriametrics.com/victoriametrics/relabeling/).
- Group A: 3-month retention.
- Group B: 1-year retention.
- Group C: 3-year retention.
**Multi-Tenant Setup**
Metrics are routed to the appropriate `vminsert` group by [splitting data streams](https://docs.victoriametrics.com/victoriametrics/vmagent/#splitting-data-streams-among-multiple-systems) in `vmagent`. An optional [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) rule can be added on top to enforce per-tenant routing or API access policies.
Every group of vmstorages can handle one tenant or multiple one. Different groups can have overlapping tenants. As vmselect reads from all vmstorage nodes, the data is aggregated on its level.
## Implementing Multi-Retention on Kubernetes
In this section, we'll install and configure the components for a multi-retention deployment of the VictoriaMetrics cluster. See [Kubernetes monitoring with VictoriaMetrics Cluster](https://docs.victoriametrics.com/guides/k8s-monitoring-via-vm-cluster/) for details on running VictoriaMetrics in Kubernetes.
Run the following command to add the VictoriaMetrics Helm repository:
```shell
helm repo add vm https://victoriametrics.github.io/helm-charts/
helm repo update
```
### Step 1: Deploying storage groups
We'll create three retention groups. Each has a different retention period and disk size. Read [Understand Your Setup Size](https://docs.victoriametrics.com/guides/understand-your-setup-size/) to estimate how much space you will need for each group. The following table is shown as an example.
| Group | Retention Period | Disk Size |
|-------------|------------------|-----------|
| `vmcluster-a` | 3 months (`3M`) | 80GB |
| `vmcluster-b` | 1 year (`1Y`) | 300 GB |
| `vmcluster-c` | 3 years (`3Y`) | 900 GB |
Create a Helm values file for Group A.
```shell
cat <<EOF > vmcluster-a.yaml
vmstorage:
enabled: true
persistence:
size: 80Gi
extraArgs:
retentionPeriod: 3M
storageDataPath: /vmstorage-data
dedup.minScrapeInterval: 30s
podLabels:
retention-group: a
retention-period: 3M
vminsert:
enabled: true
podLabels:
retention-group: a
vmselect:
enabled: false
EOF
```
The values file above creates `vminsert` and `vmstorage` services while turning off `vmselect`, which we'll deploy separately. It also defines a 30-second [deduplication](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#deduplication) window to handle possible duplicate metrics. The deduplication window must match the `vmagent` service scrape window (which we'll define later in the guide).
Create the values files for Group B and Group C:
```shell
cat <<EOF > vmcluster-b.yaml
vmstorage:
enabled: true
persistence:
size: 300Gi
extraArgs:
retentionPeriod: 1Y
storageDataPath: /vmstorage-data
dedup.minScrapeInterval: 30s
podLabels:
retention-group: b
retention-period: 1Y
vminsert:
enabled: true
podLabels:
retention-group: b
vmselect:
enabled: false
EOF
cat <<EOF > vmcluster-c.yaml
vmstorage:
enabled: true
persistence:
size: 900Gi
extraArgs:
retentionPeriod: 3Y
storageDataPath: /vmstorage-data
dedup.minScrapeInterval: 30s
podLabels:
retention-group: c
retention-period: 3Y
vminsert:
enabled: true
podLabels:
retention-group: c
vmselect:
enabled: false
EOF
```
Deploy the three storage groups with:
```shell
helm upgrade --install vmcluster-a vm/victoria-metrics-cluster -f vmcluster-a.yaml
helm upgrade --install vmcluster-b vm/victoria-metrics-cluster -f vmcluster-b.yaml
helm upgrade --install vmcluster-c vm/victoria-metrics-cluster -f vmcluster-c.yaml
# Wait for all storage pods to be ready
kubectl rollout status statefulset -l app.kubernetes.io/instance=vmcluster-a
kubectl rollout status statefulset -l app.kubernetes.io/instance=vmcluster-b
kubectl rollout status statefulset -l app.kubernetes.io/instance=vmcluster-c
```
### Step 2: Deploying vmselect
Next, we'll deploy a `vmselect` service to route queries to the storage groups.
Create a Helm values file with:
```shell
cat <<EOF >vmselect.yaml
vmstorage:
enabled: false
vminsert:
enabled: false
vmselect:
enabled: true
replicaCount: 2
suppressStorageFQDNsRender: true
extraArgs:
# Each list item is a single -storageNode flag with comma-separated hosts
# in the same group. The FQDN format is:
# <pod>.<svc>.default.svc
# where pod = <release>-victoria-metrics-cluster-vmstorage-<N>
# and svc = <release>-victoria-metrics-cluster-vmstorage
storageNode:
- "a/vmcluster-a-victoria-metrics-cluster-vmstorage-0.vmcluster-a-victoria-metrics-cluster-vmstorage.default.svc:8401,a/vmcluster-a-victoria-metrics-cluster-vmstorage-1.vmcluster-a-victoria-metrics-cluster-vmstorage.default.svc:8401"
- "b/vmcluster-b-victoria-metrics-cluster-vmstorage-0.vmcluster-b-victoria-metrics-cluster-vmstorage.default.svc:8401,b/vmcluster-b-victoria-metrics-cluster-vmstorage-1.vmcluster-b-victoria-metrics-cluster-vmstorage.default.svc:8401"
- "c/vmcluster-c-victoria-metrics-cluster-vmstorage-0.vmcluster-c-victoria-metrics-cluster-vmstorage.default.svc:8401,c/vmcluster-c-victoria-metrics-cluster-vmstorage-1.vmcluster-c-victoria-metrics-cluster-vmstorage.default.svc:8401"
dedup.minScrapeInterval: 30s
EOF
```
Let's break down the file above:
- Deploys `vmselect` as a separate Helm release
- Disables `vminsert` and `vmstorage` as these services were already deployed in Step 1.
- `supressStorageFQDNsRender: true` turns off automatic FQDN generation for storage nodes. By default, the Helm chart auto-generates `-storageNodes` flags, but since `vmstorage` has been disabled, we need to supply them manually in `extraArgs`.
- In `extraArgs.storageNode:` we define the list of `vmstorage` services to reach for queries. The `storageNode` flags tell vmselect to query all 6 storage pods, which are organized into three groups: `a`, `b`, and `c`.
Deploy the `vmselect` release with:
```shell
helm upgrade --install vmselect vm/victoria-metrics-cluster -f vmselect.yaml
```
### Step 3: Deploying vmagent
We'll use `vmagent` to route incoming metrics to the correct retention group. For example, we can use a `retention` label for mapping metrics to storage groups in the following way:
| `retention` label | Storage Group |
| ----------------| --------------|
| `"3mo"` | `vmcluster-a` |
| `"1yr"` | `vmcluster-b` |
| `"3yr"` | `vmcluster-c` |
Create the values file for vmagent:
```shell
cat <<EOF >vmagent.yaml
service:
enabled: true
remoteWrite:
# Group A: receives metrics with retention="3mo"
- url: http://vmcluster-a-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write
urlRelabelConfig:
- action: keep
source_labels: [retention]
regex: "3mo"
# Group B: receives metrics with retention="1yr"
- url: http://vmcluster-b-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write
urlRelabelConfig:
- action: keep
source_labels: [retention]
regex: "1yr"
# Group C: receives metrics with retention="3yr"
- url: http://vmcluster-c-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write
urlRelabelConfig:
- action: keep
source_labels: [retention]
regex: "3yr"
EOF
```
> Two important notes on scraping:
> - Metrics without a matching `retention` label are silently dropped by the `keep` rules. You must ensure that every metric receives a label, or use a different routing configuration.
> - The [scrape interval](https://docs.victoriametrics.com/victoriametrics/sd_configs/#scrape_configs) should match the `dedup.minScrapeInterval` defined in the vmstorage nodes.
`
Now deploy the vmagent release:
```shell
helm upgrade --install vmagent vm/victoria-metrics-agent -f vmagent.yaml
```
Wait for vmagent to become ready:
```shell
kubectl rollout status deploy/vmagent-victoria-metrics-agent
```
### Step 4: Verification
We can send test data to verify that data is flowing to the correct storage group.
First, port-forward vmagent and vmselect:
```shell
VMAGENT_SVC=$(kubectl get svc -l app.kubernetes.io/instance=vmagent -o jsonpath='{.items[0].metadata.name}')
kubectl port-forward "svc/$VMAGENT_SVC" 8429 &
VMSELECT_SVC=$(kubectl get svc -l app.kubernetes.io/instance=vmselect -o jsonpath='{.items[0].metadata.name}')
kubectl port-forward "svc/$VMSELECT_SVC" 8481 &
```
Send test metrics directly to vmagent's HTTP endpoint to exercise all three retention labels:
```shell
POD=$(kubectl get pod -l app.kubernetes.io/instance=vmagent -o jsonpath='{.items[0].metadata.name}')
for retention in 3mo 1yr 3yr; do
kubectl exec "$POD" -- wget -qO- --post-data="test_routing{retention=\"${retention}\"} 1.0" \
"http://127.0.0.1:8429/api/v1/import/prometheus"
done
```
Query the data back from vmselect (it may take around 30-60 seconds for new data to be available for queries):
```shell
for retention in 3mo 1yr 3yr; do
echo "-> retention=${retention}"
curl -s "http://localhost:8481/select/0/prometheus/api/v1/query" \
--data-urlencode "query=test_routing{retention=\"${retention}\"}"
echo
done
```
You can also check that vmagent is forwarding data to all three groups:
```shell
curl -s http://localhost:8429/metrics | grep vmagent_remotewrite_blocks_sent_total
```
Each `url="N:secret-url"` corresponds to one `remoteWrite` entry (N=1 for Group A, N=2 for Group B, N=3 for Group C). Non-zero values confirm data is flowing.
## Alternative Routing by Existing Labels
The example setup above relies on a synthetic `retention` label to exist in every incoming metric.
If having a `retention` label in every metric isn't practical, you can, as an alternative, rely on existing labels to map data to the correct storage group.
The following example configures `vmagent` to route metrics based on the `environment` and `team` labels:
```yaml
# vmagent.yaml
remoteWrite:
# send dev and staging data to Group A
- url: "http://vmcluster-a-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write"
urlRelabelConfig:
- action: keep
source_labels: [environment]
regex: "dev|staging"
# send prod data to Group B
- url: "http://vmcluster-b-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write"
urlRelabelConfig:
- action: keep
source_labels: [environment]
regex: "prod|production"
# send data from Infra and SRE teams to Group B
- url: "http://vmcluster-b-victoria-metrics-cluster-vminsert:8480/insert/0/prometheus/api/v1/write"
urlRelabelConfig:
- action: keep
source_labels: [team]
regex: "infra|sre"
```
> Metrics that do not match any of the `keep` rules are dropped in the configuration above.
## Alternative Multi-Tenant Routing
VictoriaMetrics Cluster supports [multiple isolated tenants](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenancy) identified by `accountID` (and optionally `projectID`) in the URL path, e.g., `/insert/0/prometheus/...`.
In a standard deployment, a single `vminsert` handles all tenants and distributes data across a shared pool of `vmstorage` nodes. In our current setup, each retention group deploys its own `vminsert`. This means tenant IDs are not required for routing, since data is already isolated by the URL that receives the write.
You can safely use a single tenant (`/insert/0/prometheus`) for all groups and rely on the `retention` label or label-based routing to separate data at query time.
If, however, you prefer tenant-level separation for the query layer, you can assign each group a distinct tenant ID, for instance:
| Group | Insert URL | Query URL |
|-------|------------|-----------|
| A (3mo) | `/insert/0/prometheus` | `/select/0/prometheus` |
| B (1yr) | `/insert/1/prometheus` | `/select/1/prometheus` |
| C (3yr) | `/insert/2/prometheus` | `/select/2/prometheus` |
This lets you query a single retention group directly, e.g., `/select/1/prometheus/api/v1/query?query=up` returns only data written to group B's `vminsert`. Queries to vmselect without a tenant prefix aggregate across all groups, preserving the unified view for dashboards.
The tenant path does not affect where data is stored (routing is always determined by which `vminsert` receives the data). The tenant ID is purely a query-scoping convenience in this architecture.
## Additional Enhancements
You can set up [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) to route data to the specified vminsert group based on the required retention.
**Additional Enhancements**
You can set up [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/) for routing data to the given vminsert group depending on the needed retention.

View File

@@ -205,15 +205,13 @@ curl 'http://vmselect:8481/select/multitenant/prometheus/api/v1/query' \
The precedence for applying filters for tenants follows this order:
1. Filter tenants by `extra_label`, `extra_filters` and `extra_filters[]` filters.
1. Filter tenants by `extra_label` and `extra_filters` filters.
These filters have the highest priority and are applied first when provided through the query arguments.
Filters use `OR` logic - a tenant is selected if it matches any of the filters.
2. Filter tenants from labels selectors defined at metricsQL query expression.
> **Security considerations**
It is recommended restricting access to `multitenant` endpoints only to trusted sources,
since untrusted source may break per-tenant data by writing unwanted samples or get access to data of arbitrary tenants.
See also [vmauth security doc](https://docs.victoriametrics.com/victoriametrics/vmauth/#security).
## Binaries

View File

@@ -1136,8 +1136,6 @@ By default, the last point on the interval `[now - max_lookback ... now]` is scr
For instance, `/federate?match[]=up&max_lookback=1h` would return last points on the `[now - 1h ... now]` interval. This may be useful for time series federation
with scrape intervals exceeding `5m`.
VictoriaMetrics supports Prometheus v3.0 utf-8 content encoding with `Accept` header. If `Accept: allow-utf-8` HTTP header provided, `/federate` API response changes according to [Prometheus utf-8](https://prometheus.io/docs/guides/utf8/#querying) specification - `metric_name{tag="value"}` transforms into `{"metric_name","tag"="value"}`.
## Capacity planning
VictoriaMetrics uses lower amounts of CPU, RAM and storage space on production workloads compared to competing solutions (Prometheus, Thanos, Cortex, TimescaleDB, InfluxDB, QuestDB, M3DB) according to [our case studies](https://docs.victoriametrics.com/victoriametrics/casestudies/).

View File

@@ -26,24 +26,10 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
## tip
* SECURITY: upgrade Go builder from Go1.26.3 to Go1.26.4. See [the list of issues addressed in Go1.26.4](https://github.com/golang/go/issues?q=milestone%3AGo1.26.4%20label%3ACherryPickApproved).
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): support `match[]=<label_selector>` query parameters in `/api/v1/rules` and `/api/v1/alerts` APIs to return only the rules that have configured labels satisfying the provided label selectors. See [11020](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11020).
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/), [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): add `-opentelemetry.promoteAllResourceAttributes` and `-opentelemetry.promoteScopeMetadata` command-line flags to allow managing label promotion for resource attributes and OTel scope metadata. See [OpenTelemetry](https://docs.victoriametrics.com/victoriametrics/integrations/opentelemetry/) docs and [#10931](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10931).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent/) : introduce `vmagent_remotewrite_kafka_outbuf_latency_seconds` and `vmagent_remotewrite_kafka_rtt_seconds` metrics for [kafka integration](https://docs.victoriametrics.com/victoriametrics/integrations/kafka/). The metrics could help identify throughput bottlenecks. See [#10730](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10730).
* FEATURE: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly log user information when a missing route error occurs. See [#11052](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11052).
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl/): add the ability to migrate data from [Mimir](https://docs.victoriametrics.com/victoriametrics/vmctl/mimir/#) object storage to VictoriaMetrics. See [#7717](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7717).
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): show the full `version` label in the `Version` panel when `short_version` label is empty (e.g. custom builds from feature branch). Previously, the panel could appear empty. See [#11047](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11047).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fix the `Notifiers` page in web UI appearing blank despite the API returning notifier data correctly. See [#11035](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11035).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): reset the group evaluation timestamp if it exceeds the current host time. Previously, vmalert could use future timestamps for evaluations if the system clock was shifted backward. See [#10985](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10985).
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): properly parse [Prometheus Native Histograms](https://prometheus.io/docs/specs/native_histograms/), previously Protobuf parser could produce unexpected `vmrange` labels. See [#11041](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11041).
* BUGFIX: [vmauth](https://docs.victoriametrics.com/victoriametrics/vmauth/): properly calculate number of loaded users to be printed in startup log. Previously, it was only accounting for static users and skipped JWT configuration entries. See [#11050](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11050/).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): `integrate()` no longer extrapolates the last sample's value past the end of the time series. Previously, querying `integrate(metric[1h])` at a timestamp where the series had already ended would keep accruing area as if the last value continued indefinitely, producing values much larger than the true integral. See [#9474](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9474). Thanks to @wtfashwin for contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): avoid returning HTTP 503 for queries with partial results when a storage group is unavailable and `-search.denyPartialResponse` is disabled. See [#11009](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11009). Thanks to @fxrlv for the contribution.
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): properly escape `utf-8` label names for [/federate](https://docs.victoriametrics.com/victoriametrics/#federation) API requests. See [#10968](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10968).
* BUGFIX: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the `Disable deduplication` toggle under its own local storage key. Before this fix, the toggle state was lost after reload and could overwrite the `Compact view` table setting. See [#11004](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11004). Thanks to @immanuwell for the contribution.
* BUGFIX: `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix intermittent `write: connection timed out` errors caused by silently dropped TCP connections being reused from the connection pool. See [#10735](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10735#issuecomment-4535832301).
## [v1.144.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.144.0)

View File

@@ -502,7 +502,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
Supports array of values separated by comma or specified via multiple flags.
Empty values are set to default value.
-remoteWrite.roundDigits array
Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics. See also -remoteWrite.significantFigures (default 100)
Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default, digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics (default 100)
Supports array of values separated by comma or specified via multiple flags.
Empty values are set to default value.
-remoteWrite.sendTimeout array

View File

@@ -801,15 +801,17 @@ Please refer to the [VictoriaMetrics Cloud documentation](https://docs.victoriam
`vmalert` runs a web-server (`-httpListenAddr`) for serving metrics and alerts endpoints:
* `http://<vmalert-addr>` - UI;
* `http://<vmalert-addr>/api/v1/rules` - returns a list of all loaded groups and rules. Supports the `datasource_type`, `search`, `group_limit`, and `page_num` parameters, as well as additional [filtering](https://prometheus.io/docs/prometheus/latest/querying/api/#rules);
* `http://<vmalert-addr>/api/v1/alerts` - returns a list of all active alerts. Supports the `datasource_type`, `rule_group[]`, `file[]` and `match[]`(applied on templated alert labels) query parameters;
* `http://<vmalert-addr>/api/v1/notifiers` - returns a list of all available notifiers;
* `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - returns the alert status in JSON format;
* `http://<vmalert-addr>/vmalert/api/v1/rule?group_id=<group_id>&rule_id=<rule_id>` - returns the rule status in JSON format;
* `http://<vmalert-addr>/vmalert/api/v1/group?group_id=<group_id>` - returns the group status in JSON format. Used as the alert source in AlertManager;
* `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - displays the alert status in the web UI;
* `http://<vmalert-addr>/vmalert/rule?group_id=<group_id>&rule_id=<rule_id>` - displays the rule status in the web UI;
* `http://<vmalert-addr>/metrics` - application metrics endpoint;
* `http://<vmalert-addr>/api/v1/rules` - list of all loaded groups and rules. Supports `search`, `group_limit`, and `page_num` parameters, as well as additional [filtering](https://prometheus.io/docs/prometheus/latest/querying/api/#rules);
* `http://<vmalert-addr>/api/v1/alerts` - list of all active alerts;
* `http://<vmalert-addr>/api/v1/notifiers` - list all available notifiers;
* `http://<vmalert-addr>/vmalert/api/v1/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in JSON format.
* `http://<vmalert-addr>/vmalert/api/v1/rule?group_id=<group_id>&rule_id=<rule_id>` - get rule status in JSON format.
* `http://<vmalert-addr>/vmalert/api/v1/group?group_id=<group_id>` - get group status in JSON format.
Used as alert source in AlertManager.
* `http://<vmalert-addr>/vmalert/alert?group_id=<group_id>&alert_id=<alert_id>` - get alert status in web UI.
* `http://<vmalert-addr>/vmalert/rule?group_id=<group_id>&rule_id=<rule_id>` - get rule status in web UI.
* `http://<vmalert-addr>/vmalert/api/v1/rule?group_id=<group_id>&alert_id=<alert_id>` - get rule status in JSON format.
* `http://<vmalert-addr>/metrics` - application metrics.
* `http://<vmalert-addr>/-/reload` - hot configuration reload.
`vmalert` web UI can be accessed from [single-node version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/)

View File

@@ -1536,16 +1536,6 @@ To enable TLS on the public listener while keeping the internal listener non-TLS
`vmauth` also supports restricting access by IP - see [these docs](#ip-filters). See also [concurrency limiting docs](#concurrency-limiting).
When `vmauth` performs tenant routing for [multitenant](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#multitenant-reads) requests, it is crucial to explicitly set `extra_label`, `extra_filters` and `extra_filters[]` in the url_prefix configuration:
```yaml
unauthorized_user:
url_prefix: http://vmselect/select/multitenant?extra_filters[]=&extra_filters=&extra_label=vm_account_id=10&extra_label=vm_project_id=100
```
This is required because `vmselect` uses `OR` logic for tenant filtering. If a client sets `extra_filters[]` or `extra_filters`, it could bypass the tenant restriction configured via `extra_label`.
## Automatic issuing of TLS certificates
`vmauth` [Enterprise](https://docs.victoriametrics.com/victoriametrics/enterprise/) supports automatic issuing of TLS certificates via [Let's Encrypt service](https://letsencrypt.org/).

View File

@@ -73,66 +73,4 @@ ou can define it via the flag `--remote-read-headers=X-Scope-OrgID:demo`.
See [remote-read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/) for more details.
See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).
### Read data from the remote storage like S3, GCS, Azure etc.
If you have data stored in remote storage like S3, GCS, Azure etc. you can use `vmctl` in `mimir` mode to read data from
the remote storage and import it into VictoriaMetrics. In this mode `vmctl` reads data from the remote storage or file system
and checks index file, define needed blocks to be processed. After it downloads blocks by defined filters and
use Prometheus converter to read and sent data to VictoriaMetrics.
The following example shows how to read data from the file system and import it into VictoriaMetrics:
```sh
./vmctl mimir --mimir-path="fs:///mimir/test_data/mimir-tsdb" \ ? ? orbstack
--mimir-tenant-id=anonymous \
--mimir-filter-time-start=2024-12-01T00:00:00 \
--mimir-filter-time-end=2024-12-18T23:59:59 \
--mimir-creds-file-path=creads \
--vm-concurrency=6 \
--mimir-concurrency=6 \
--vm-addr=http://localhost:8428/
```
This approach is useful when you have data stored on the local file system or you have a mounted volume,
download the data from the remote storage etc.
The following example shows how to read data from the remote storage and import it into VictoriaMetrics:
```sh
./vmctl mimir --mimir-path="s3:///mimir-tsdb/anonymous" \
--mimir-filter-time-start=2024-12-01T00:00:00 \
--mimir-filter-time-end=2024-12-17T23:59:59 \
--mimir-creds-file-path=creads \
--mimir-custom-s3-endpoint='http://localhost:9000' \
--vm-concurrency=6 \
--mimir-concurrency=6 \
--vm-addr=http://localhost:8428/
```
In the example above we are used `--mimir-custom-s3-endpoint` flag to specify the custom S3 endpoint if it is needed.
When the process finishes, you will see the following:
```sh
2025/01/18 13:01:59 Fetching blocks from remote storage
Found 204 blocks to import. Continue? [Y/n] y
VM worker 0:? 1589405 samples/s
VM worker 1:? 1911834 samples/s
VM worker 2:? 1849187 samples/s
VM worker 3:? 1648820 samples/s
VM worker 4:? 1539212 samples/s
VM worker 5:? 1411485 samples/s
Processing blocks: 204 / 204 [?????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????] 100.00%
2025/01/18 13:02:18 Import finished!
2025/01/18 13:02:18 VictoriaMetrics importer stats:
idle duration: 18.485875611s;
time spent while importing: 16.40543875s;
total samples: 177961995;
samples/s: 10847743.71;
total bytes: 4.1 GB;
bytes/s: 248.2 MB;
import requests: 893;
import requests retries: 0;
2025/01/18 13:02:18 Total time: 18.867547083s
```
See also general [vmctl migration tips](https://docs.victoriametrics.com/victoriametrics/vmctl/#migration-tips).

View File

@@ -56,7 +56,7 @@ OPTIONS:
--vm-compress Whether to apply gzip compression to import requests (default: true)
--vm-batch-size value How many samples importer collects before sending the import request to VM (default: 200000)
--vm-significant-figures value The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
--vm-extra-label value [ --vm-extra-label value ] Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
--vm-rate-limit value Optional data transfer rate limit in bytes per second.
By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

View File

@@ -51,7 +51,7 @@ OPTIONS:
--vm-compress Whether to apply gzip compression to import requests (default: true)
--vm-batch-size value How many samples importer collects before sending the import request to VM (default: 200000)
--vm-significant-figures value The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
--vm-extra-label value [ --vm-extra-label value ] Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
--vm-rate-limit value Optional data transfer rate limit in bytes per second.
By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

View File

@@ -44,7 +44,7 @@ OPTIONS:
--vm-compress Whether to apply gzip compression to import requests (default: true)
--vm-batch-size value How many samples importer collects before sending the import request to VM (default: 200000)
--vm-significant-figures value The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
--vm-extra-label value [ --vm-extra-label value ] Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
--vm-rate-limit value Optional data transfer rate limit in bytes per second.
By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

View File

@@ -59,7 +59,7 @@ OPTIONS:
--vm-compress Whether to apply gzip compression to import requests (default: true)
--vm-batch-size value How many samples importer collects before sending the import request to VM (default: 200000)
--vm-significant-figures value The number of significant figures to leave in metric values before importing. See https://en.wikipedia.org/wiki/Significant_figures. Zero value saves all the significant figures. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-round-digits option (default: 0)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics. See also --vm-significant-figures option (default: 100)
--vm-round-digits value Round metric values to the given number of decimal digits after the point. This option may be used for increasing on-disk compression level for the stored metrics (default: 100)
--vm-extra-label value [ --vm-extra-label value ] Extra labels, that will be added to imported timeseries. In case of collision, label value defined by flag will have priority. Flag can be set multiple times, to add few additional labels.
--vm-rate-limit value Optional data transfer rate limit in bytes per second.
By default, the rate limit is disabled. It can be useful for limiting load on configured via '--vm-addr' destination. (default: 0)

2
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/VictoriaMetrics/VictoriaMetrics
go 1.26.4
go 1.26.3
require (
cloud.google.com/go/storage v1.62.1

View File

@@ -398,10 +398,6 @@ func (nhctx *nativeHistogramContext) appendTimeSeries(tss []TimeSeries, baseLabe
if baseName == "" {
return tss, labelsPool, samplesPool
}
originName := *nameValueP
defer func() {
*nameValueP = originName
}()
*nameValueP = fb.formatName(baseName, "_count")
tss, labelsPool, samplesPool = appendHistogramSeries(tss, labelsPool, samplesPool, baseLabels, "", tsMillis, count)
@@ -554,11 +550,9 @@ func (nhctx *nativeHistogramContext) reset() {
nhctx.zeroCountInt = 0
nhctx.zeroCountFloat = 0
nhctx.timestamp = 0
clear(nhctx.negativeSpans)
nhctx.negativeSpans = nhctx.negativeSpans[:0]
nhctx.negativeDeltas = nhctx.negativeDeltas[:0]
nhctx.negativeCounts = nhctx.negativeCounts[:0]
clear(nhctx.positiveSpans)
nhctx.positiveSpans = nhctx.positiveSpans[:0]
nhctx.positiveDeltas = nhctx.positiveDeltas[:0]
nhctx.positiveCounts = nhctx.positiveCounts[:0]

View File

@@ -3,9 +3,8 @@ package prompb
import (
"encoding/binary"
"math"
"reflect"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestUnmarshalTimeSeries(t *testing.T) {
@@ -19,8 +18,8 @@ func TestUnmarshalTimeSeries(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if diff := cmp.Diff(wantTSS, tss); len(diff) > 0 {
t.Fatalf("unexpected timeseries (-want, +got):\n%s", diff)
if !reflect.DeepEqual(tss, wantTSS) {
t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, wantTSS)
}
}
@@ -198,93 +197,6 @@ func TestUnmarshalTimeSeries(t *testing.T) {
},
})
}
{
// verify histogram fields are correctly reused
nativeHistogramC := nativeHistogramContext{
countInt: 0,
isCountFloat: true,
countFloat: 2.5,
sum: 1.0,
schema: 1,
zeroThreshold: 0.00001,
isZeroCountFloat: true,
zeroCountFloat: 0.5,
timestamp: 3000,
positiveSpans: []bucketSpan{{offset: 1, length: 2}},
positiveCounts: []float64{1.5, 1.0},
negativeSpans: []bucketSpan{{offset: 0, length: 1}},
}
nativeHistogramC2 := nativeHistogramContext{
countInt: 0,
isCountFloat: true,
countFloat: 0,
sum: 1.0,
schema: 1,
zeroThreshold: 0.00001,
isZeroCountFloat: true,
zeroCountFloat: 0.5,
timestamp: 4000,
positiveSpans: []bucketSpan{{offset: 0, length: 2}},
positiveCounts: []float64{1.5, 1.0},
negativeSpans: []bucketSpan{{offset: 0, length: 1}},
negativeCounts: []float64{1.5, 0},
}
hd1 := encodeHistogram(nativeHistogramC)
hd2 := encodeHistogram(nativeHistogramC2)
src := encodeTimeSeries(
[]Label{{Name: "__name__", Value: "rpc_latency_seconds"}},
nil,
[][]byte{hd1, hd2},
)
f(src, []TimeSeries{
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_count"}},
Samples: []Sample{{Value: 2.5, Timestamp: 3000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_sum"}},
Samples: []Sample{{Value: 1.0, Timestamp: 3000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(-0.00001, 0.00001)}},
Samples: []Sample{{Value: 0.5, Timestamp: 3000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(1, 1.414)}},
Samples: []Sample{{Value: 1.5, Timestamp: 3000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(1.414, 2.0)}},
Samples: []Sample{{Value: 1.0, Timestamp: 3000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_count"}},
Samples: []Sample{{Value: 0, Timestamp: 4000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_sum"}},
Samples: []Sample{{Value: 1.0, Timestamp: 4000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(-0.00001, 0.00001)}},
Samples: []Sample{{Value: 0.5, Timestamp: 4000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(0.7071, 1)}},
Samples: []Sample{{Value: 1.5, Timestamp: 4000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(1, 1.414)}},
Samples: []Sample{{Value: 1.0, Timestamp: 4000}},
},
{
Labels: []Label{{Name: "__name__", Value: "rpc_latency_seconds_bucket"}, {Name: "vmrange", Value: appendVmrangeHelper(-1, -0.7071)}},
Samples: []Sample{{Value: 1.5, Timestamp: 4000}},
},
})
}
}
func encodeTimeSeries(labels []Label, samples []Sample, histograms [][]byte) []byte {