mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-07-30 20:41:44 +03:00
Compare commits
10 Commits
docs-artic
...
issue-5914
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e54c3ecfb6 | ||
|
|
d98336e581 | ||
|
|
13ef9c3ba8 | ||
|
|
5a87c85093 | ||
|
|
f1a9c61ba0 | ||
|
|
6cb014fde5 | ||
|
|
565ecdc4fb | ||
|
|
06f4fde931 | ||
|
|
203eb3a2b4 | ||
|
|
7827647b96 |
@@ -2,7 +2,6 @@
|
||||
|
||||
[](https://github.com/VictoriaMetrics/VictoriaMetrics/releases)
|
||||
[](https://hub.docker.com/u/victoriametrics)
|
||||
[](https://goreportcard.com/report/github.com/VictoriaMetrics/VictoriaMetrics)
|
||||
[](https://github.com/VictoriaMetrics/VictoriaMetrics/actions/workflows/build.yml)
|
||||
[](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/LICENSE)
|
||||
[](https://slack.victoriametrics.com)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
@@ -267,7 +268,11 @@ func (c *Client) do(req *http.Request) (*http.Response, error) {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
return nil, fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body)
|
||||
err = &httpserver.ErrorWithStatusCode{
|
||||
StatusCode: resp.StatusCode,
|
||||
Err: fmt.Errorf("unexpected response code %d for %s. Response body %s", resp.StatusCode, ru, body),
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
@@ -31,6 +31,8 @@ type Alert struct {
|
||||
State AlertState
|
||||
// Expr contains expression that was executed to generate the Alert
|
||||
Expr string
|
||||
// Interval contains the evaluation interval of the Alert's group
|
||||
Interval time.Duration
|
||||
// ActiveAt defines the moment of time when Alert has become active
|
||||
ActiveAt time.Time
|
||||
// Start defines the moment of time when Alert has become firing
|
||||
@@ -84,6 +86,7 @@ type AlertTplData struct {
|
||||
Labels map[string]string
|
||||
Value float64
|
||||
Expr string
|
||||
Interval time.Duration
|
||||
AlertID uint64
|
||||
GroupID uint64
|
||||
ActiveAt time.Time
|
||||
@@ -96,6 +99,7 @@ var tplHeaders = []string{
|
||||
"{{ $type := .Type }}",
|
||||
"{{ $labels := .Labels }}",
|
||||
"{{ $expr := .Expr }}",
|
||||
"{{ $interval := .Interval }}",
|
||||
"{{ $externalLabels := .ExternalLabels }}",
|
||||
"{{ $externalURL := .ExternalURL }}",
|
||||
"{{ $alertID := .AlertID }}",
|
||||
@@ -115,6 +119,7 @@ func (a *Alert) ExecTemplate(q templates.QueryFn, labels, annotations map[string
|
||||
Type: a.Type,
|
||||
Labels: labels,
|
||||
Expr: a.Expr,
|
||||
Interval: a.Interval,
|
||||
AlertID: a.ID,
|
||||
GroupID: a.GroupID,
|
||||
ActiveAt: a.ActiveAt,
|
||||
|
||||
@@ -129,6 +129,17 @@ func TestAlertExecTemplate(t *testing.T) {
|
||||
"exprEscapedHTML": "vm_rows{"label"="bar"}<0",
|
||||
})
|
||||
|
||||
// interval-template
|
||||
f(&Alert{
|
||||
Interval: 10 * time.Second,
|
||||
}, map[string]string{
|
||||
"interval": "{{ .Interval }}",
|
||||
"intervalVariable": "{{ $interval }}",
|
||||
}, map[string]string{
|
||||
"interval": "10s",
|
||||
"intervalVariable": "10s",
|
||||
})
|
||||
|
||||
// query
|
||||
f(&Alert{
|
||||
Expr: `vm_rows{"label"="bar"}>0`,
|
||||
|
||||
@@ -25,11 +25,12 @@ var (
|
||||
replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
|
||||
"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
|
||||
replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
|
||||
"Defines how many retries to make before giving up on rule if request for it returns an error.")
|
||||
"Defines how many retries to make before giving up on rule if request for it returns a retriable error.")
|
||||
disableProgressBar = flag.Bool("replay.disableProgressBar", false, "Whether to disable rendering progress bars during the replay. "+
|
||||
"Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.")
|
||||
ruleEvaluationConcurrency = flag.Int("replay.ruleEvaluationConcurrency", 1, "The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. "+
|
||||
"Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery.")
|
||||
continueWithExecutionErr = flag.Bool("replay.continueWithExecutionErr", false, "Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.")
|
||||
)
|
||||
|
||||
func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewrite.RWClient) (totalRows, droppedRows int, err error) {
|
||||
@@ -73,7 +74,7 @@ func replay(groupsCfg []config.Group, qb datasource.QuerierBuilder, rw remotewri
|
||||
|
||||
for _, cfg := range groupsCfg {
|
||||
ng := rule.NewGroup(cfg, qb, *evaluationInterval, labels)
|
||||
totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency)
|
||||
totalRows += ng.Replay(tFrom, tTo, rw, *replayMaxDatapoints, *replayRuleRetryAttempts, *replayRulesDelay, *disableProgressBar, *ruleEvaluationConcurrency, *continueWithExecutionErr)
|
||||
}
|
||||
logger.Infof("replay evaluation finished, generated %d samples", totalRows)
|
||||
if err := rw.Close(); err != nil {
|
||||
|
||||
@@ -8,8 +8,10 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutil"
|
||||
"github.com/VictoriaMetrics/metricsql"
|
||||
)
|
||||
|
||||
type fakeReplayQuerier struct {
|
||||
@@ -32,6 +34,14 @@ func (fc *fakeRWClient) Close() error {
|
||||
}
|
||||
|
||||
func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to time.Time) (res datasource.Result, err error) {
|
||||
_, err = metricsql.Parse(q)
|
||||
if err != nil {
|
||||
return res, &httpserver.ErrorWithStatusCode{
|
||||
StatusCode: 422,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
key := fmt.Sprintf("%s+%s", from.Format("15:04:05"), to.Format("15:04:05"))
|
||||
dps, ok := fr.registry[q]
|
||||
if !ok {
|
||||
@@ -275,4 +285,28 @@ func TestReplay(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}, 10)
|
||||
|
||||
// rule with wrong expression won't break the other rule with continueWithExecutionErr
|
||||
continueWithExecutionErrOld := *continueWithExecutionErr
|
||||
defer func() {
|
||||
*continueWithExecutionErr = continueWithExecutionErrOld
|
||||
}()
|
||||
*continueWithExecutionErr = true
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 1, time.Millisecond, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
{Rules: []config.Rule{{Record: "bar", Expr: "up ++"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string][]datasource.Metric{
|
||||
"sum(up)": {
|
||||
"12:00:00+12:01:00": {
|
||||
{
|
||||
Timestamps: []int64{1, 2},
|
||||
Values: []float64{1, 2},
|
||||
},
|
||||
},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
},
|
||||
}, 2)
|
||||
}
|
||||
|
||||
@@ -530,6 +530,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
|
||||
ar.logDebugf(ts, a, "INACTIVE => PENDING")
|
||||
}
|
||||
a.Value = m.Values[0]
|
||||
a.Interval = ar.EvalInterval
|
||||
a.Annotations = annotations
|
||||
a.KeepFiringSince = time.Time{}
|
||||
continue
|
||||
@@ -612,6 +613,7 @@ func (ar *AlertingRule) expandAnnotationTemplates(m datasource.Metric, qFn templ
|
||||
Type: ar.Type.String(),
|
||||
Labels: ls.origin,
|
||||
Expr: ar.Expr,
|
||||
Interval: ar.EvalInterval,
|
||||
AlertID: hash(ls.processed),
|
||||
GroupID: ar.GroupID,
|
||||
ActiveAt: activeAt,
|
||||
@@ -673,6 +675,7 @@ func (ar *AlertingRule) newAlert(m datasource.Metric, start time.Time, labels, a
|
||||
Name: ar.Name,
|
||||
Type: ar.Type.String(),
|
||||
Expr: ar.Expr,
|
||||
Interval: ar.EvalInterval,
|
||||
For: ar.For,
|
||||
ActiveAt: start,
|
||||
Value: m.Values[0],
|
||||
|
||||
@@ -662,6 +662,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.GetID(),
|
||||
Name: "for-pending",
|
||||
Type: config.NewPrometheusType().String(),
|
||||
Interval: time.Second,
|
||||
Labels: map[string]string{"alertname": "for-pending"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StatePending,
|
||||
@@ -682,6 +683,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.GetID(),
|
||||
Name: "for-firing",
|
||||
Type: config.NewPrometheusType().String(),
|
||||
Interval: 3 * time.Second,
|
||||
Labels: map[string]string{"alertname": "for-firing"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StateFiring,
|
||||
@@ -703,6 +705,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.GetID(),
|
||||
Name: "for-hold-pending",
|
||||
Type: config.NewPrometheusType().String(),
|
||||
Interval: time.Second,
|
||||
Labels: map[string]string{"alertname": "for-hold-pending"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StatePending,
|
||||
@@ -759,6 +762,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.GetID(),
|
||||
Name: "multi-series",
|
||||
Type: config.NewPrometheusType().String(),
|
||||
Interval: 3 * time.Second,
|
||||
Labels: map[string]string{"alertname": "multi-series"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StateFiring,
|
||||
@@ -771,6 +775,7 @@ func TestAlertingRuleExecRange(t *testing.T) {
|
||||
GroupID: fakeGroup.GetID(),
|
||||
Name: "multi-series",
|
||||
Type: config.NewPrometheusType().String(),
|
||||
Interval: 3 * time.Second,
|
||||
Labels: map[string]string{"alertname": "multi-series", "foo": "bar"},
|
||||
Annotations: map[string]string{},
|
||||
State: notifier.StatePending,
|
||||
@@ -1134,7 +1139,8 @@ func TestAlertingRule_Template(t *testing.T) {
|
||||
fq.Add(metrics...)
|
||||
fq.SetPartialResponse(isResponsePartial)
|
||||
|
||||
if _, err := rule.exec(context.TODO(), time.Now(), 0); err != nil {
|
||||
ts := time.Unix(3600, 0)
|
||||
if _, err := rule.exec(context.TODO(), ts, 0); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
for hash, expAlert := range alertsExpected {
|
||||
@@ -1152,12 +1158,14 @@ func TestAlertingRule_Template(t *testing.T) {
|
||||
}
|
||||
|
||||
f(&AlertingRule{
|
||||
Name: "common",
|
||||
Name: "common",
|
||||
EvalInterval: time.Hour,
|
||||
Labels: map[string]string{
|
||||
"region": "east",
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
|
||||
"summary": `{{ $labels.alertname }}: Too high connection number for "{{ $labels.instance }}"`,
|
||||
"dashboard": `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`,
|
||||
},
|
||||
alerts: make(map[uint64]*notifier.Alert),
|
||||
}, []datasource.Metric{
|
||||
@@ -1166,7 +1174,8 @@ func TestAlertingRule_Template(t *testing.T) {
|
||||
}, false, map[uint64]*notifier.Alert{
|
||||
hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "foo"}): {
|
||||
Annotations: map[string]string{
|
||||
"summary": `common: Too high connection number for "foo"`,
|
||||
"summary": `common: Too high connection number for "foo"`,
|
||||
"dashboard": "&from=0&to=3600000",
|
||||
},
|
||||
Labels: map[string]string{
|
||||
alertNameLabel: "common",
|
||||
@@ -1176,7 +1185,8 @@ func TestAlertingRule_Template(t *testing.T) {
|
||||
},
|
||||
hash(map[string]string{alertNameLabel: "common", "region": "east", "instance": "bar"}): {
|
||||
Annotations: map[string]string{
|
||||
"summary": `common: Too high connection number for "bar"`,
|
||||
"summary": `common: Too high connection number for "bar"`,
|
||||
"dashboard": "&from=0&to=3600000",
|
||||
},
|
||||
Labels: map[string]string{
|
||||
alertNameLabel: "common",
|
||||
@@ -1388,7 +1398,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
|
||||
"alertname": "ConfigurationReloadFailure",
|
||||
"alertgroup": "vmalert",
|
||||
"pod": "vmalert-0",
|
||||
"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
|
||||
"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
|
||||
}
|
||||
|
||||
expectedProcessedLabels := map[string]string{
|
||||
@@ -1398,7 +1408,7 @@ func TestAlertingRule_ToLabels(t *testing.T) {
|
||||
"exported_alertname": "ConfigurationReloadFailure",
|
||||
"group": "vmalert",
|
||||
"alertgroup": "vmalert",
|
||||
"invalid_label": `error evaluating template: template: :1:298: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
|
||||
"invalid_label": `error evaluating template: template: :1:326: executing "" at <.Values.mustRuntimeFail>: can't evaluate field Values in type notifier.tplData`,
|
||||
}
|
||||
|
||||
ls, err := ar.toLabels(metric, nil)
|
||||
|
||||
@@ -548,7 +548,7 @@ func (g *Group) infof(format string, args ...any) {
|
||||
}
|
||||
|
||||
// Replay performs group replay
|
||||
func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int) int {
|
||||
func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoint, replayRuleRetryAttempts int, replayDelay time.Duration, disableProgressBar bool, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
|
||||
var total int
|
||||
step := g.Interval * time.Duration(maxDataPoint)
|
||||
ri := rangeIterator{start: start, end: end, step: step}
|
||||
@@ -576,7 +576,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations)
|
||||
}
|
||||
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
|
||||
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
@@ -598,7 +598,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
|
||||
rule := g.Rules[i]
|
||||
sem <- struct{}{}
|
||||
wg.Go(func() {
|
||||
res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency)
|
||||
res <- replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts, ruleEvaluationConcurrency, continueWithExecutionErr)
|
||||
<-sem
|
||||
})
|
||||
}
|
||||
@@ -618,7 +618,7 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
|
||||
return total
|
||||
}
|
||||
|
||||
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int) int {
|
||||
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts, ruleEvaluationConcurrency int, continueWithExecutionErr bool) int {
|
||||
fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
|
||||
// alerting rule with for>0 can't be replayed concurrently, since the status change might depend on the previous evaluation
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/commit/abcb21aa5ee918ba9a4e9cde495dba06e1e9564c
|
||||
@@ -633,7 +633,7 @@ func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewri
|
||||
start := ri.s
|
||||
end := ri.e
|
||||
wg.Go(func() {
|
||||
n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts)
|
||||
n, err := replayRule(r, start, end, rw, replayRuleRetryAttempts, continueWithExecutionErr)
|
||||
if err != nil {
|
||||
logger.Fatalf("rule %q: %s", r, err)
|
||||
}
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb"
|
||||
)
|
||||
@@ -118,7 +120,7 @@ func (s *ruleState) add(e StateEntry) {
|
||||
s.entries[s.cur] = e
|
||||
}
|
||||
|
||||
func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int) (int, error) {
|
||||
func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRuleRetryAttempts int, continueWithExecutionErr bool) (int, error) {
|
||||
var err error
|
||||
var tss []prompb.TimeSeries
|
||||
for i := range replayRuleRetryAttempts {
|
||||
@@ -126,6 +128,21 @@ func replayRule(r Rule, start, end time.Time, rw remotewrite.RWClient, replayRul
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
// retry request if possible to tolerate temporary network or datasource unavailability issues
|
||||
var esc *httpserver.ErrorWithStatusCode
|
||||
if errors.As(err, &esc) {
|
||||
statusCode := esc.StatusCode
|
||||
// if the status code is 422, it means that the query was executed but failed due to an expression syntax error or a the resource limit being hit,
|
||||
// continue replaying but skip the problematic execution if continueWithExecutionErr is true, otherwise, return the error without retry.
|
||||
if statusCode == http.StatusUnprocessableEntity {
|
||||
if continueWithExecutionErr {
|
||||
logger.Errorf("rule %q: %s", r, err)
|
||||
return 0, nil
|
||||
} else {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.Errorf("attempt %d to execute rule %q failed: %s", i+1, r, err)
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
|
||||
@@ -311,6 +311,8 @@ var (
|
||||
|
||||
const (
|
||||
influxAddr = "influx-addr"
|
||||
influxVersion = "influx-version"
|
||||
influxToken = "influx-token"
|
||||
influxUser = "influx-user"
|
||||
influxPassword = "influx-password"
|
||||
influxDB = "influx-database"
|
||||
@@ -337,6 +339,19 @@ var (
|
||||
Value: "http://localhost:8086",
|
||||
Usage: "InfluxDB server addr",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: influxVersion,
|
||||
Usage: "Major version of the source InfluxDB: 1 or 2.\n" +
|
||||
"InfluxDB 2.x is migrated via its InfluxDB 1.x compatibility API, which requires -influx-token " +
|
||||
"and a database/retention policy mapping (DBRP) pointing at the bucket to migrate.\n" +
|
||||
"See https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/",
|
||||
Value: 1,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: influxToken,
|
||||
Usage: "InfluxDB v2 API token. Requires -influx-version=2",
|
||||
EnvVars: []string{"INFLUX_TOKEN"},
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: influxUser,
|
||||
Usage: "InfluxDB user",
|
||||
@@ -353,9 +368,11 @@ var (
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: influxRetention,
|
||||
Usage: "InfluxDB retention policy",
|
||||
Value: "autogen",
|
||||
Name: influxRetention,
|
||||
Usage: "InfluxDB retention policy.\n" +
|
||||
"Defaults to 'autogen' for -influx-version=1.\n" +
|
||||
"For -influx-version=2 it is the retention policy of a DBRP mapping; if empty, " +
|
||||
"the default mapping of -influx-database is used",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: influxChunkSize,
|
||||
|
||||
@@ -11,6 +11,27 @@ import (
|
||||
influx "github.com/influxdata/influxdb/client/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
// VersionV1 is InfluxDB 1.x, queried via its native /query endpoint.
|
||||
VersionV1 = 1
|
||||
|
||||
// VersionV2 is InfluxDB 2.x, queried via its InfluxDB 1.x compatibility
|
||||
// API. It authenticates with an API token instead of a password.
|
||||
//
|
||||
// See https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/
|
||||
VersionV2 = 2
|
||||
|
||||
// defaultV1CompatUser is sent as the username when migrating from InfluxDB 2.x
|
||||
// with an API token.
|
||||
//
|
||||
// The InfluxDB 1.x compatibility API requires a username whenever an API token
|
||||
// is used as the password, but the value itself is ignored.
|
||||
// See https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/
|
||||
defaultV1CompatUser = "vmctl"
|
||||
|
||||
defaultRetentionV1 = "autogen"
|
||||
)
|
||||
|
||||
// Client represents a wrapper over
|
||||
// influx HTTP client
|
||||
type Client struct {
|
||||
@@ -27,7 +48,9 @@ type Client struct {
|
||||
// Config contains fields required
|
||||
// for Client configuration
|
||||
type Config struct {
|
||||
Version int
|
||||
Addr string
|
||||
Token string
|
||||
Username string
|
||||
Password string
|
||||
Database string
|
||||
@@ -88,10 +111,18 @@ type LabelPair struct {
|
||||
// NewClient creates and returns influx client
|
||||
// configured with passed Config
|
||||
func NewClient(cfg Config) (*Client, error) {
|
||||
if err := cfg.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// InfluxDB 2.x authenticates with an API token passed as the password
|
||||
// of its InfluxDB 1.x compatibility API.
|
||||
username, password := resolveAuth(cfg.Username, cfg.Password, cfg.Token)
|
||||
|
||||
c := influx.HTTPConfig{
|
||||
Addr: cfg.Addr,
|
||||
Username: cfg.Username,
|
||||
Password: cfg.Password,
|
||||
Username: username,
|
||||
Password: password,
|
||||
TLSConfig: cfg.TLSConfig,
|
||||
}
|
||||
hc, err := influx.NewHTTPClient(c)
|
||||
@@ -99,6 +130,7 @@ func NewClient(cfg Config) (*Client, error) {
|
||||
return nil, fmt.Errorf("failed to establish conn: %w", err)
|
||||
}
|
||||
if _, _, err := hc.Ping(time.Second); err != nil {
|
||||
_ = hc.Close()
|
||||
return nil, fmt.Errorf("ping failed: %w", err)
|
||||
}
|
||||
|
||||
@@ -110,7 +142,7 @@ func NewClient(cfg Config) (*Client, error) {
|
||||
client := &Client{
|
||||
Client: hc,
|
||||
database: cfg.Database,
|
||||
retention: cfg.Retention,
|
||||
retention: resolveRetention(cfg.Version, cfg.Retention),
|
||||
chunkSize: chunkSize,
|
||||
filterTime: timeFilter(cfg.Filter.TimeStart, cfg.Filter.TimeEnd),
|
||||
filterSeries: cfg.Filter.Series,
|
||||
@@ -464,3 +496,52 @@ func (c *Client) do(q influx.Query) ([]queryValues, error) {
|
||||
}
|
||||
return parseResult(res.Results[0])
|
||||
}
|
||||
|
||||
// resolveAuth returns the credentials to authenticate with.
|
||||
//
|
||||
// InfluxDB 2.x is queried via its InfluxDB 1.x compatibility API, which accepts
|
||||
// an API token in place of the password. Therefore a non-empty token replaces
|
||||
// the password, and a placeholder username is substituted when none is given.
|
||||
func resolveAuth(username, password, token string) (string, string) {
|
||||
if token == "" {
|
||||
return username, password
|
||||
}
|
||||
if username == "" {
|
||||
username = defaultV1CompatUser
|
||||
}
|
||||
return username, token
|
||||
}
|
||||
|
||||
// resolveRetention returns the retention policy to query.
|
||||
//
|
||||
// In InfluxDB 1.x `autogen` is the retention policy created together with a
|
||||
// database, so it is a meaningful default. In InfluxDB 2.x the retention policy
|
||||
// is one half of a DBRP mapping and its name is arbitrary, so no default can be
|
||||
// assumed: an empty value makes InfluxDB use the default mapping of the
|
||||
// database instead of failing on a non-existent one.
|
||||
func resolveRetention(version int, retention string) string {
|
||||
if retention == "" && version == VersionV1 {
|
||||
return defaultRetentionV1
|
||||
}
|
||||
return retention
|
||||
}
|
||||
|
||||
// validate checks that the configuration is self-consistent.
|
||||
func (cfg *Config) validate() error {
|
||||
if cfg.Version != VersionV1 && cfg.Version != VersionV2 {
|
||||
return fmt.Errorf("unsupported InfluxDB version %d; supported versions are %d and %d",
|
||||
cfg.Version, VersionV1, VersionV2)
|
||||
}
|
||||
if cfg.Version == VersionV2 && cfg.Token == "" {
|
||||
return fmt.Errorf("-influx-token is required for InfluxDB v2")
|
||||
}
|
||||
if cfg.Version == VersionV1 && cfg.Token != "" {
|
||||
return fmt.Errorf("-influx-token is only supported for InfluxDB v2; pass -influx-version=2 to use it")
|
||||
}
|
||||
// The database is the `db` parameter of the query API. For InfluxDB 2.x
|
||||
// it is the database name of a DBRP mapping, which points at a bucket.
|
||||
if cfg.Database == "" {
|
||||
return fmt.Errorf("-influx-database cannot be empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
package influx
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchQuery(t *testing.T) {
|
||||
f := func(s *Series, timeFilter, resultExpected string) {
|
||||
@@ -126,3 +132,255 @@ func TestGetSeriesCommand(t *testing.T) {
|
||||
f("from cpu where arch='x86'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND time >= '2020-01-01T20:07:00Z'")
|
||||
f("from cpu where arch='x86' AND hostname='host_2753'", "time >= '2020-01-01T20:07:00Z'", "show series from cpu where arch='x86' AND hostname='host_2753' AND time >= '2020-01-01T20:07:00Z'")
|
||||
}
|
||||
|
||||
func TestResolveAuth(t *testing.T) {
|
||||
f := func(username, password, token, userExpected, passExpected string) {
|
||||
t.Helper()
|
||||
|
||||
user, pass := resolveAuth(username, password, token)
|
||||
if user != userExpected || pass != passExpected {
|
||||
t.Fatalf("unexpected credentials for (username=%q, password=%q, token=%q)\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
username, password, token, user, pass, userExpected, passExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x: no token, credentials are passed through unchanged.
|
||||
f("", "", "", "", "")
|
||||
f("user", "pass", "", "user", "pass")
|
||||
|
||||
// InfluxDB 2.x: the API token is sent as the password. The username is
|
||||
// required by the v1 compatibility API but may be any value, so a
|
||||
// placeholder is used when the caller did not provide one.
|
||||
f("", "", "token", defaultV1CompatUser, "token")
|
||||
|
||||
// An explicitly provided username is preserved.
|
||||
f("myuser", "", "token", "myuser", "token")
|
||||
|
||||
// The token takes precedence over a password.
|
||||
f("user", "pass", "token", "user", "token")
|
||||
}
|
||||
|
||||
func TestConfigValidateSuccess(t *testing.T) {
|
||||
f := func(cfg Config) {
|
||||
t.Helper()
|
||||
|
||||
if err := cfg.validate(); err != nil {
|
||||
t.Fatalf("unexpected error for config %+v: %s", cfg, err)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x with and without credentials.
|
||||
f(Config{Version: 1, Database: "mydb"})
|
||||
f(Config{Version: 1, Database: "mydb", Username: "user", Password: "pass"})
|
||||
|
||||
// InfluxDB 2.x requires a token.
|
||||
f(Config{Version: 2, Database: "mydb", Token: "my-token"})
|
||||
}
|
||||
|
||||
func TestConfigValidateFailure(t *testing.T) {
|
||||
f := func(cfg Config, errStrExpected string) {
|
||||
t.Helper()
|
||||
|
||||
err := cfg.validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error for config %+v", cfg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), errStrExpected) {
|
||||
t.Fatalf("unexpected error for config %+v\ngot\n%s\nwant it to contain\n%s", cfg, err, errStrExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// unsupported versions
|
||||
f(Config{Version: 0, Database: "mydb"}, "unsupported InfluxDB version")
|
||||
f(Config{Version: 3, Database: "mydb"}, "unsupported InfluxDB version")
|
||||
|
||||
// InfluxDB 2.x without a token cannot authenticate
|
||||
f(Config{Version: 2, Database: "mydb"}, "influx-token")
|
||||
|
||||
// a token is meaningless without opting into v2
|
||||
f(Config{Version: 1, Database: "mydb", Token: "my-token"}, "influx-version=2")
|
||||
|
||||
// the database is mandatory for both versions, since it is the `db`
|
||||
// parameter of the query API
|
||||
f(Config{Version: 1}, "influx-database")
|
||||
f(Config{Version: 2, Token: "my-token"}, "influx-database")
|
||||
}
|
||||
|
||||
func TestResolveRetention(t *testing.T) {
|
||||
f := func(version int, retention, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
result := resolveRetention(version, retention)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected retention policy for (version=%d, retention=%q)\ngot\n%q\nwant\n%q",
|
||||
version, retention, result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x keeps its historical default.
|
||||
f(VersionV1, "", "autogen")
|
||||
f(VersionV1, "all_data", "all_data")
|
||||
|
||||
// For InfluxDB 2.x the retention policy is part of a DBRP mapping whose
|
||||
// name is arbitrary, so no default may be assumed: an empty value lets
|
||||
// InfluxDB pick the default mapping of the database.
|
||||
f(VersionV2, "", "")
|
||||
f(VersionV2, "myrp", "myrp")
|
||||
}
|
||||
|
||||
// TestNewClientValidatesConfig ensures an invalid configuration is rejected
|
||||
// before any connection to InfluxDB is attempted.
|
||||
func TestNewClientValidatesConfig(t *testing.T) {
|
||||
f := func(cfg Config, errStrExpected string) {
|
||||
t.Helper()
|
||||
|
||||
cfg.Addr = "http://127.0.0.1:1"
|
||||
|
||||
c, err := NewClient(cfg)
|
||||
if err == nil {
|
||||
t.Fatalf("expecting non-nil error for config %+v", cfg)
|
||||
}
|
||||
if c != nil {
|
||||
t.Fatalf("expecting nil client for config %+v", cfg)
|
||||
}
|
||||
if !strings.Contains(err.Error(), errStrExpected) {
|
||||
t.Fatalf("unexpected error for config %+v\ngot\n%s\nwant it to contain\n%s", cfg, err, errStrExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f(Config{Version: 0, Database: "mydb"}, "unsupported InfluxDB version")
|
||||
f(Config{Version: 2, Database: "mydb"}, "influx-token")
|
||||
f(Config{Version: 1, Database: "mydb", Token: "my-token"}, "influx-version=2")
|
||||
}
|
||||
|
||||
// TestNewClientTokenAuth ensures the API token is used as the password when
|
||||
// migrating from InfluxDB 2.x.
|
||||
func TestNewClientTokenAuth(t *testing.T) {
|
||||
f := func(cfg Config, userExpected, passExpected string) {
|
||||
t.Helper()
|
||||
|
||||
user, pass := resolveAuth(cfg.Username, cfg.Password, cfg.Token)
|
||||
if user != userExpected || pass != passExpected {
|
||||
t.Fatalf("unexpected credentials for config %+v\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
cfg, user, pass, userExpected, passExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f(Config{Version: 2, Database: "mydb", Token: "my-token"}, defaultV1CompatUser, "my-token")
|
||||
f(Config{Version: 1, Database: "mydb", Username: "user", Password: "pass"}, "user", "pass")
|
||||
}
|
||||
|
||||
func TestQueryRequestAuthAndRetention(t *testing.T) {
|
||||
f := func(cfg Config, userExpected, passExpected, rpExpected string) {
|
||||
t.Helper()
|
||||
|
||||
var lastQuery queryRequest
|
||||
s := newTestServer(t, &lastQuery)
|
||||
defer s.Close()
|
||||
|
||||
cfg.Addr = s.URL
|
||||
c, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating client: %s", err)
|
||||
}
|
||||
|
||||
if _, err := c.fieldsByMeasurement(); err != nil {
|
||||
t.Fatalf("unexpected error querying field keys: %s", err)
|
||||
}
|
||||
|
||||
got := lastQuery.get()
|
||||
if !got.seen {
|
||||
t.Fatalf("the client did not issue a /query request")
|
||||
}
|
||||
if !got.hasAuth {
|
||||
t.Fatalf("expecting basic auth to be set on the request")
|
||||
}
|
||||
if got.user != userExpected || got.password != passExpected {
|
||||
t.Fatalf("unexpected credentials on the wire\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
got.user, got.password, userExpected, passExpected)
|
||||
}
|
||||
if got.rp != rpExpected {
|
||||
t.Fatalf("unexpected rp parameter\ngot\n%q\nwant\n%q", got.rp, rpExpected)
|
||||
}
|
||||
if got.db != cfg.Database {
|
||||
t.Fatalf("unexpected db parameter\ngot\n%q\nwant\n%q", got.db, cfg.Database)
|
||||
}
|
||||
}
|
||||
|
||||
// InfluxDB 1.x: credentials are passed through and `autogen` is the
|
||||
// default retention policy.
|
||||
f(Config{Version: VersionV1, Database: "mydb", Username: "user", Password: "pass"},
|
||||
"user", "pass", "autogen")
|
||||
|
||||
// An explicit retention policy is honoured.
|
||||
f(Config{Version: VersionV1, Database: "mydb", Username: "user", Password: "pass", Retention: "all_data"},
|
||||
"user", "pass", "all_data")
|
||||
|
||||
// InfluxDB 2.x: the API token is sent as the password with a placeholder
|
||||
// username, and no retention policy is assumed so that the default DBRP
|
||||
// mapping of the database applies.
|
||||
f(Config{Version: VersionV2, Database: "mydb", Token: "my-token"},
|
||||
defaultV1CompatUser, "my-token", "")
|
||||
|
||||
// An explicit retention policy names the DBRP mapping to query.
|
||||
f(Config{Version: VersionV2, Database: "mydb", Token: "my-token", Retention: "myrp"},
|
||||
defaultV1CompatUser, "my-token", "myrp")
|
||||
}
|
||||
|
||||
// queryRequest holds what a /query request carried on the wire. Only the values
|
||||
// under test are kept, so nothing of the *http.Request outlives the handler.
|
||||
type queryRequest struct {
|
||||
mu sync.Mutex
|
||||
|
||||
seen bool
|
||||
user string
|
||||
password string
|
||||
hasAuth bool
|
||||
rp string
|
||||
db string
|
||||
}
|
||||
|
||||
func (q *queryRequest) set(r *http.Request) {
|
||||
user, password, hasAuth := r.BasicAuth()
|
||||
params := r.URL.Query()
|
||||
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
q.seen = true
|
||||
q.user, q.password, q.hasAuth = user, password, hasAuth
|
||||
q.rp = params.Get("rp")
|
||||
q.db = params.Get("db")
|
||||
}
|
||||
|
||||
func (q *queryRequest) get() queryRequest {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
return queryRequest{
|
||||
seen: q.seen,
|
||||
user: q.user,
|
||||
password: q.password,
|
||||
hasAuth: q.hasAuth,
|
||||
rp: q.rp,
|
||||
db: q.db,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestServer(t *testing.T, lastQuery *queryRequest) *httptest.Server {
|
||||
t.Helper()
|
||||
|
||||
const fieldKeysResponse = `{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["fieldKey","fieldType"],"values":[["value","float"]]}]}]}`
|
||||
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Influxdb-Version", "test")
|
||||
switch r.URL.Path {
|
||||
case "/ping":
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case "/query":
|
||||
lastQuery.set(r)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(fieldKeysResponse))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -141,7 +141,9 @@ func main() {
|
||||
}
|
||||
|
||||
iCfg := influx.Config{
|
||||
Version: c.Int(influxVersion),
|
||||
Addr: c.String(influxAddr),
|
||||
Token: c.String(influxToken),
|
||||
Username: c.String(influxUser),
|
||||
Password: c.String(influxPassword),
|
||||
Database: c.String(influxDB),
|
||||
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/golang/snappy"
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/prometheus/prometheus/storage/remote"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
@@ -234,9 +236,29 @@ func processResponse(body io.ReadCloser, callback StreamCallback) error {
|
||||
// shouldn't be accounted as an error.
|
||||
for _, res := range readResp.Results {
|
||||
for _, ts := range res.Timeseries {
|
||||
vmTs := convertSamples(ts.Samples, ts.Labels)
|
||||
if err := callback(vmTs); err != nil {
|
||||
return err
|
||||
// A series contains either float samples or native histogram samples.
|
||||
// Both fields are processed independently, since a series may switch
|
||||
// from float to native histogram representation at some point in time,
|
||||
// so the requested time range may contain samples of both types.
|
||||
if len(ts.Samples) > 0 {
|
||||
vmTs := convertSamples(ts.Samples, ts.Labels)
|
||||
if err := callback(vmTs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(ts.Histograms) > 0 {
|
||||
hSamples := make([]histogramSample, 0, len(ts.Histograms))
|
||||
for _, h := range ts.Histograms {
|
||||
hSamples = append(hSamples, histogramSample{
|
||||
timestamp: h.Timestamp,
|
||||
fh: h.ToFloatHistogram(),
|
||||
})
|
||||
}
|
||||
for _, vmTs := range convertHistograms(hSamples, ts.Labels) {
|
||||
if err := callback(vmTs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -263,17 +285,45 @@ func processStreamResponse(body io.ReadCloser, callback StreamCallback) error {
|
||||
|
||||
for _, series := range res.ChunkedSeries {
|
||||
samples := make([]prompb.Sample, 0)
|
||||
var hSamples []histogramSample
|
||||
for _, chunk := range series.Chunks {
|
||||
s, err := parseSamples(chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
switch chunk.Type {
|
||||
case prompb.Chunk_XOR, prompb.Chunk_UNKNOWN:
|
||||
// In proto3 the `type` field may be left unset (UNKNOWN) for XOR chunks.
|
||||
// Prometheus remote.proto: "REQUIREMENT: when using proto3, this field
|
||||
// MUST be set when using anything else than XOR". Senders before native
|
||||
// histograms support (Prometheus < 2.40) do not set this field at all,
|
||||
// so UNKNOWN chunks must be parsed as XOR ones.
|
||||
s, err := parseSamples(chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
samples = append(samples, s...)
|
||||
case prompb.Chunk_HISTOGRAM, prompb.Chunk_FLOAT_HISTOGRAM:
|
||||
hs, err := parseHistograms(chunk.Type, chunk.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hSamples = append(hSamples, hs...)
|
||||
default:
|
||||
return fmt.Errorf("unsupported chunk encoding %q", chunk.Type)
|
||||
}
|
||||
samples = append(samples, s...)
|
||||
}
|
||||
|
||||
ts := convertSamples(samples, series.Labels)
|
||||
if err := callback(ts); err != nil {
|
||||
return err
|
||||
// A series contains either XOR chunks or native histogram chunks.
|
||||
// Both are processed independently, since a series may switch
|
||||
// from float to native histogram representation at some point in time,
|
||||
// so the requested time range may contain chunks of both types.
|
||||
if len(samples) > 0 {
|
||||
ts := convertSamples(samples, series.Labels)
|
||||
if err := callback(ts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, ts := range convertHistograms(hSamples, series.Labels) {
|
||||
if err := callback(ts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -312,6 +362,151 @@ func parseSamples(chunk []byte) ([]prompb.Sample, error) {
|
||||
return samples, it.Err()
|
||||
}
|
||||
|
||||
// histogramSample represents a single native histogram sample.
|
||||
type histogramSample struct {
|
||||
timestamp int64
|
||||
fh *histogram.FloatHistogram
|
||||
}
|
||||
|
||||
func parseHistograms(encoding prompb.Chunk_Encoding, chunk []byte) ([]histogramSample, error) {
|
||||
var enc chunkenc.Encoding
|
||||
switch encoding {
|
||||
case prompb.Chunk_HISTOGRAM:
|
||||
enc = chunkenc.EncHistogram
|
||||
case prompb.Chunk_FLOAT_HISTOGRAM:
|
||||
enc = chunkenc.EncFloatHistogram
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported histogram chunk encoding %q", encoding)
|
||||
}
|
||||
c, err := chunkenc.FromData(enc, chunk)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error read chunk: %w", err)
|
||||
}
|
||||
|
||||
var hSamples []histogramSample
|
||||
it := c.Iterator(nil)
|
||||
for {
|
||||
typ := it.Next()
|
||||
if typ == chunkenc.ValNone {
|
||||
break
|
||||
}
|
||||
switch typ {
|
||||
case chunkenc.ValHistogram:
|
||||
ts, h := it.AtHistogram(nil)
|
||||
hSamples = append(hSamples, histogramSample{
|
||||
timestamp: ts,
|
||||
fh: h.ToFloat(nil),
|
||||
})
|
||||
case chunkenc.ValFloatHistogram:
|
||||
ts, fh := it.AtFloatHistogram(nil)
|
||||
hSamples = append(hSamples, histogramSample{
|
||||
timestamp: ts,
|
||||
fh: fh,
|
||||
})
|
||||
default:
|
||||
// Skip unsupported values
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := it.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error iterate over chunks: %w", err)
|
||||
}
|
||||
|
||||
return hSamples, nil
|
||||
}
|
||||
|
||||
// convertHistograms converts native histogram samples into VictoriaMetrics histogram
|
||||
// time series in the same way as VictoriaMetrics converts native histograms
|
||||
// received via Prometheus remote write protocol: every native histogram sample
|
||||
// is converted into `<name>_count` and `<name>_sum` series plus a set of
|
||||
// `<name>_bucket` series with `vmrange` labels containing non-cumulative bucket counts.
|
||||
// The only difference is that for native histograms with custom buckets (NHCB)
|
||||
// bucket bounds are taken from the custom values, while the remote write protocol
|
||||
// parser ignores custom values and estimates the bounds with the exponential formula.
|
||||
// See https://prometheus.io/docs/specs/native_histograms/#data-model
|
||||
func convertHistograms(hSamples []histogramSample, labels []prompb.Label) []*vm.TimeSeries {
|
||||
if len(hSamples) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
labelPairs := make([]vm.LabelPair, 0, len(labels))
|
||||
nameValue := ""
|
||||
for _, label := range labels {
|
||||
if label.Name == "__name__" {
|
||||
nameValue = label.Value
|
||||
continue
|
||||
}
|
||||
labelPairs = append(labelPairs, vm.LabelPair{Name: label.Name, Value: label.Value})
|
||||
}
|
||||
// the metric has no name, skip it in the same way as VictoriaMetrics does
|
||||
// when it receives a native histogram without the metric name via remote write protocol.
|
||||
if nameValue == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
countSeries := &vm.TimeSeries{
|
||||
Name: nameValue + "_count",
|
||||
LabelPairs: labelPairs,
|
||||
}
|
||||
sumSeries := &vm.TimeSeries{
|
||||
Name: nameValue + "_sum",
|
||||
LabelPairs: labelPairs,
|
||||
}
|
||||
bucketSeries := make(map[string]*vm.TimeSeries)
|
||||
// vmranges preserves the order of bucketSeries creation
|
||||
// in order to get deterministic results.
|
||||
var vmranges []string
|
||||
|
||||
for _, hs := range hSamples {
|
||||
fh := hs.fh
|
||||
countSeries.Timestamps = append(countSeries.Timestamps, hs.timestamp)
|
||||
countSeries.Values = append(countSeries.Values, fh.Count)
|
||||
sumSeries.Timestamps = append(sumSeries.Timestamps, hs.timestamp)
|
||||
sumSeries.Values = append(sumSeries.Values, fh.Sum)
|
||||
|
||||
it := fh.AllBucketIterator()
|
||||
for it.Next() {
|
||||
b := it.At()
|
||||
if b.Count <= 0 {
|
||||
continue
|
||||
}
|
||||
vmrange := formatVmrange(b.Lower, b.Upper)
|
||||
s := bucketSeries[vmrange]
|
||||
if s == nil {
|
||||
bucketLabelPairs := make([]vm.LabelPair, len(labelPairs), len(labelPairs)+1)
|
||||
copy(bucketLabelPairs, labelPairs)
|
||||
bucketLabelPairs = append(bucketLabelPairs, vm.LabelPair{Name: "vmrange", Value: vmrange})
|
||||
s = &vm.TimeSeries{
|
||||
Name: nameValue + "_bucket",
|
||||
LabelPairs: bucketLabelPairs,
|
||||
}
|
||||
bucketSeries[vmrange] = s
|
||||
vmranges = append(vmranges, vmrange)
|
||||
}
|
||||
s.Timestamps = append(s.Timestamps, hs.timestamp)
|
||||
s.Values = append(s.Values, b.Count)
|
||||
}
|
||||
}
|
||||
|
||||
tss := make([]*vm.TimeSeries, 0, 2+len(vmranges))
|
||||
tss = append(tss, countSeries, sumSeries)
|
||||
for _, vmrange := range vmranges {
|
||||
tss = append(tss, bucketSeries[vmrange])
|
||||
}
|
||||
return tss
|
||||
}
|
||||
|
||||
// formatVmrange formats the given bucket bounds into `vmrange` label value
|
||||
// in the same way as VictoriaMetrics does for native histograms
|
||||
// received via Prometheus remote write protocol.
|
||||
func formatVmrange(lower, upper float64) string {
|
||||
b := make([]byte, 0, 24)
|
||||
b = strconv.AppendFloat(b, lower, 'e', 3, 64)
|
||||
b = append(b, "..."...)
|
||||
b = strconv.AppendFloat(b, upper, 'e', 3, 64)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
type keyValue struct {
|
||||
key string
|
||||
value string
|
||||
|
||||
334
app/vmctl/remoteread/remoteread_test.go
Normal file
334
app/vmctl/remoteread/remoteread_test.go
Normal file
@@ -0,0 +1,334 @@
|
||||
package remoteread
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/golang/snappy"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/prometheus/prometheus/storage/remote"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
||||
)
|
||||
|
||||
func testHistogram(mul int64) *histogram.Histogram {
|
||||
return &histogram.Histogram{
|
||||
Schema: 0,
|
||||
Count: uint64(10 * mul),
|
||||
Sum: 25.5 * float64(mul),
|
||||
ZeroThreshold: 0.001,
|
||||
ZeroCount: uint64(2 * mul),
|
||||
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}},
|
||||
PositiveBuckets: []int64{1 * mul, 2 * mul},
|
||||
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
|
||||
NegativeBuckets: []int64{4 * mul},
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertHistograms(t *testing.T) {
|
||||
f := func(hSamples []histogramSample, labels []prompb.Label, expected []*vm.TimeSeries) {
|
||||
t.Helper()
|
||||
|
||||
tss := convertHistograms(hSamples, labels)
|
||||
if !reflect.DeepEqual(tss, expected) {
|
||||
t.Fatalf("unexpected result\ngot:\n%v\nwant:\n%v", tss, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// series without samples
|
||||
f(nil, []prompb.Label{{Name: "__name__", Value: "foo"}}, nil)
|
||||
|
||||
// series without the metric name must be skipped
|
||||
f([]histogramSample{
|
||||
{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
|
||||
}, []prompb.Label{{Name: "job", Value: "bar"}}, nil)
|
||||
|
||||
// native histogram must be converted to _count, _sum and _bucket series
|
||||
// in the same way as VictoriaMetrics does for Prometheus remote write protocol
|
||||
labels := []prompb.Label{
|
||||
{Name: "__name__", Value: "request_duration_seconds"},
|
||||
{Name: "job", Value: "bar"},
|
||||
}
|
||||
jobLabel := []vm.LabelPair{{Name: "job", Value: "bar"}}
|
||||
bucketLabels := func(vmrange string) []vm.LabelPair {
|
||||
return []vm.LabelPair{
|
||||
{Name: "job", Value: "bar"},
|
||||
{Name: "vmrange", Value: vmrange},
|
||||
}
|
||||
}
|
||||
f([]histogramSample{
|
||||
{timestamp: 1000, fh: testHistogram(1).ToFloat(nil)},
|
||||
{timestamp: 2000, fh: testHistogram(2).ToFloat(nil)},
|
||||
}, labels, []*vm.TimeSeries{
|
||||
{
|
||||
Name: "request_duration_seconds_count",
|
||||
LabelPairs: jobLabel,
|
||||
Timestamps: []int64{1000, 2000},
|
||||
Values: []float64{10, 20},
|
||||
},
|
||||
{
|
||||
Name: "request_duration_seconds_sum",
|
||||
LabelPairs: jobLabel,
|
||||
Timestamps: []int64{1000, 2000},
|
||||
Values: []float64{25.5, 51},
|
||||
},
|
||||
{
|
||||
Name: "request_duration_seconds_bucket",
|
||||
LabelPairs: bucketLabels("-1.000e+00...-5.000e-01"),
|
||||
Timestamps: []int64{1000, 2000},
|
||||
Values: []float64{4, 8},
|
||||
},
|
||||
{
|
||||
Name: "request_duration_seconds_bucket",
|
||||
LabelPairs: bucketLabels("-1.000e-03...1.000e-03"),
|
||||
Timestamps: []int64{1000, 2000},
|
||||
Values: []float64{2, 4},
|
||||
},
|
||||
{
|
||||
Name: "request_duration_seconds_bucket",
|
||||
LabelPairs: bucketLabels("5.000e-01...1.000e+00"),
|
||||
Timestamps: []int64{1000, 2000},
|
||||
Values: []float64{1, 2},
|
||||
},
|
||||
{
|
||||
Name: "request_duration_seconds_bucket",
|
||||
LabelPairs: bucketLabels("1.000e+00...2.000e+00"),
|
||||
Timestamps: []int64{1000, 2000},
|
||||
Values: []float64{3, 6},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseHistograms(t *testing.T) {
|
||||
c := chunkenc.NewHistogramChunk()
|
||||
app, err := c.Appender()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create chunk appender: %s", err)
|
||||
}
|
||||
if _, _, _, err := app.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
|
||||
t.Fatalf("cannot append histogram: %s", err)
|
||||
}
|
||||
if _, _, _, err := app.AppendHistogram(nil, 0, 2000, testHistogram(2), true); err != nil {
|
||||
t.Fatalf("cannot append histogram: %s", err)
|
||||
}
|
||||
|
||||
hSamples, err := parseHistograms(prompb.Chunk_HISTOGRAM, c.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("cannot parse histogram chunk: %s", err)
|
||||
}
|
||||
if len(hSamples) != 2 {
|
||||
t.Fatalf("unexpected number of histogram samples; got %d; want 2", len(hSamples))
|
||||
}
|
||||
for i, expected := range []struct {
|
||||
timestamp int64
|
||||
count float64
|
||||
sum float64
|
||||
}{
|
||||
{timestamp: 1000, count: 10, sum: 25.5},
|
||||
{timestamp: 2000, count: 20, sum: 51},
|
||||
} {
|
||||
if hSamples[i].timestamp != expected.timestamp {
|
||||
t.Fatalf("unexpected timestamp; got %d; want %d", hSamples[i].timestamp, expected.timestamp)
|
||||
}
|
||||
if hSamples[i].fh.Count != expected.count {
|
||||
t.Fatalf("unexpected count; got %f; want %f", hSamples[i].fh.Count, expected.count)
|
||||
}
|
||||
if hSamples[i].fh.Sum != expected.sum {
|
||||
t.Fatalf("unexpected sum; got %f; want %f", hSamples[i].fh.Sum, expected.sum)
|
||||
}
|
||||
}
|
||||
|
||||
// unsupported chunk encoding must return error
|
||||
if _, err := parseHistograms(prompb.Chunk_XOR, c.Bytes()); err == nil {
|
||||
t.Fatalf("expecting non-nil error for unsupported chunk encoding")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessResponse(t *testing.T) {
|
||||
readResp := &prompb.ReadResponse{
|
||||
Results: []*prompb.QueryResult{
|
||||
{
|
||||
Timeseries: []*prompb.TimeSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "__name__", Value: "cpu_usage"},
|
||||
{Name: "job", Value: "bar"},
|
||||
},
|
||||
Samples: []prompb.Sample{
|
||||
{Timestamp: 1000, Value: 1.5},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "__name__", Value: "request_duration_seconds"},
|
||||
{Name: "job", Value: "bar"},
|
||||
},
|
||||
Histograms: []prompb.Histogram{
|
||||
prompb.FromIntHistogram(1000, testHistogram(1)),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
data, err := proto.Marshal(readResp)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal ReadResponse: %s", err)
|
||||
}
|
||||
compressed := snappy.Encode(nil, data)
|
||||
|
||||
var tss []*vm.TimeSeries
|
||||
err = processResponse(io.NopCloser(bytes.NewReader(compressed)), func(ts *vm.TimeSeries) error {
|
||||
tss = append(tss, ts)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot process response: %s", err)
|
||||
}
|
||||
|
||||
// 1 float series + _count + _sum + 4 buckets
|
||||
if len(tss) != 7 {
|
||||
t.Fatalf("unexpected number of time series; got %d; want 7", len(tss))
|
||||
}
|
||||
if tss[0].Name != "cpu_usage" || !reflect.DeepEqual(tss[0].Values, []float64{1.5}) {
|
||||
t.Fatalf("unexpected float series: %v", tss[0])
|
||||
}
|
||||
if tss[1].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[1].Values, []float64{10}) {
|
||||
t.Fatalf("unexpected _count series: %v", tss[1])
|
||||
}
|
||||
if tss[2].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[2].Values, []float64{25.5}) {
|
||||
t.Fatalf("unexpected _sum series: %v", tss[2])
|
||||
}
|
||||
for _, ts := range tss[3:] {
|
||||
if ts.Name != "request_duration_seconds_bucket" {
|
||||
t.Fatalf("unexpected bucket series name %q", ts.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type nopFlusher struct{}
|
||||
|
||||
func (nopFlusher) Flush() {}
|
||||
|
||||
func TestProcessStreamResponse(t *testing.T) {
|
||||
// build a histogram chunk
|
||||
hc := chunkenc.NewHistogramChunk()
|
||||
hApp, err := hc.Appender()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create histogram chunk appender: %s", err)
|
||||
}
|
||||
if _, _, _, err := hApp.AppendHistogram(nil, 0, 1000, testHistogram(1), true); err != nil {
|
||||
t.Fatalf("cannot append histogram: %s", err)
|
||||
}
|
||||
|
||||
// build a float chunk
|
||||
xc := chunkenc.NewXORChunk()
|
||||
xApp, err := xc.Appender()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create xor chunk appender: %s", err)
|
||||
}
|
||||
xApp.Append(0, 1000, 1.5)
|
||||
|
||||
res := &prompb.ChunkedReadResponse{
|
||||
ChunkedSeries: []*prompb.ChunkedSeries{
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "__name__", Value: "request_duration_seconds"},
|
||||
{Name: "job", Value: "bar"},
|
||||
},
|
||||
Chunks: []prompb.Chunk{
|
||||
{Type: prompb.Chunk_HISTOGRAM, Data: hc.Bytes()},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "__name__", Value: "cpu_usage"},
|
||||
},
|
||||
Chunks: []prompb.Chunk{
|
||||
{Type: prompb.Chunk_XOR, Data: xc.Bytes()},
|
||||
},
|
||||
},
|
||||
{
|
||||
Labels: []prompb.Label{
|
||||
{Name: "__name__", Value: "memory_usage"},
|
||||
},
|
||||
Chunks: []prompb.Chunk{
|
||||
// the `type` field may be unset for XOR chunks,
|
||||
// such chunks must be parsed as XOR ones
|
||||
{Type: prompb.Chunk_UNKNOWN, Data: xc.Bytes()},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
data, err := proto.Marshal(res)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal ChunkedReadResponse: %s", err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
cw := remote.NewChunkedWriter(&buf, nopFlusher{})
|
||||
if _, err := cw.Write(data); err != nil {
|
||||
t.Fatalf("cannot write chunked response: %s", err)
|
||||
}
|
||||
|
||||
var tss []*vm.TimeSeries
|
||||
err = processStreamResponse(io.NopCloser(&buf), func(ts *vm.TimeSeries) error {
|
||||
tss = append(tss, ts)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cannot process stream response: %s", err)
|
||||
}
|
||||
|
||||
// _count + _sum + 4 buckets + 1 float series + 1 float series from UNKNOWN chunk
|
||||
if len(tss) != 8 {
|
||||
t.Fatalf("unexpected number of time series; got %d; want 8", len(tss))
|
||||
}
|
||||
if tss[0].Name != "request_duration_seconds_count" || !reflect.DeepEqual(tss[0].Values, []float64{10}) {
|
||||
t.Fatalf("unexpected _count series: %v", tss[0])
|
||||
}
|
||||
if tss[1].Name != "request_duration_seconds_sum" || !reflect.DeepEqual(tss[1].Values, []float64{25.5}) {
|
||||
t.Fatalf("unexpected _sum series: %v", tss[1])
|
||||
}
|
||||
for _, ts := range tss[2:6] {
|
||||
if ts.Name != "request_duration_seconds_bucket" {
|
||||
t.Fatalf("unexpected bucket series name %q", ts.Name)
|
||||
}
|
||||
}
|
||||
if tss[6].Name != "cpu_usage" || !reflect.DeepEqual(tss[6].Values, []float64{1.5}) {
|
||||
t.Fatalf("unexpected float series: %v", tss[6])
|
||||
}
|
||||
if tss[7].Name != "memory_usage" || !reflect.DeepEqual(tss[7].Values, []float64{1.5}) {
|
||||
t.Fatalf("unexpected float series from UNKNOWN chunk: %v", tss[7])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFloatHistograms(t *testing.T) {
|
||||
c := chunkenc.NewFloatHistogramChunk()
|
||||
app, err := c.Appender()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create chunk appender: %s", err)
|
||||
}
|
||||
fh := testHistogram(1).ToFloat(nil)
|
||||
if _, _, _, err := app.AppendFloatHistogram(nil, 0, 1000, fh, true); err != nil {
|
||||
t.Fatalf("cannot append float histogram: %s", err)
|
||||
}
|
||||
|
||||
hSamples, err := parseHistograms(prompb.Chunk_FLOAT_HISTOGRAM, c.Bytes())
|
||||
if err != nil {
|
||||
t.Fatalf("cannot parse float histogram chunk: %s", err)
|
||||
}
|
||||
if len(hSamples) != 1 {
|
||||
t.Fatalf("unexpected number of histogram samples; got %d; want 1", len(hSamples))
|
||||
}
|
||||
if hSamples[0].timestamp != 1000 {
|
||||
t.Fatalf("unexpected timestamp; got %d; want 1000", hSamples[0].timestamp)
|
||||
}
|
||||
if hSamples[0].fh.Count != 10 {
|
||||
t.Fatalf("unexpected count; got %f; want 10", hSamples[0].fh.Count)
|
||||
}
|
||||
}
|
||||
342
apptest/tests/influx_server_test.go
Normal file
342
apptest/tests/influx_server_test.go
Normal file
@@ -0,0 +1,342 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// influxPoint is a single field value at a single timestamp of a single series,
|
||||
// as served by the mock InfluxDB server.
|
||||
type influxPoint struct {
|
||||
Measurement string
|
||||
Tags map[string]string
|
||||
Field string
|
||||
FieldType string
|
||||
Timestamp int64 // unix seconds
|
||||
Value any // float64, int64, bool or string
|
||||
}
|
||||
|
||||
// influxRequest is a /query request recorded by the mock server, used to assert
|
||||
// how vmctl authenticates and which database and retention policy it asks for.
|
||||
type influxRequest struct {
|
||||
Query url.Values
|
||||
User string
|
||||
Password string
|
||||
HasAuth bool
|
||||
}
|
||||
|
||||
// influxMockServer implements the subset of the InfluxDB 1.x query API used by
|
||||
// `vmctl influx`: /ping plus /query serving `SHOW FIELD KEYS`, `SHOW TAG KEYS`,
|
||||
// `SHOW SERIES` and `SELECT`.
|
||||
//
|
||||
// InfluxDB 2.x is migrated through the very same API - its 1.x compatibility
|
||||
// endpoint - so one mock covers both -influx-version=1 and -influx-version=2.
|
||||
type influxMockServer struct {
|
||||
server *httptest.Server
|
||||
points []influxPoint
|
||||
|
||||
mu sync.Mutex
|
||||
requests []influxRequest
|
||||
}
|
||||
|
||||
// newInfluxMockServer starts an httptest server serving the given points.
|
||||
func newInfluxMockServer(t *testing.T, points []influxPoint) *influxMockServer {
|
||||
t.Helper()
|
||||
s := &influxMockServer{points: points}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/ping", s.handlePing)
|
||||
mux.HandleFunc("/query", s.handleQuery)
|
||||
s.server = httptest.NewServer(mux)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *influxMockServer) close() { s.server.Close() }
|
||||
|
||||
func (s *influxMockServer) httpAddr() string { return s.server.URL }
|
||||
|
||||
// recordQuery stores a /query request for later assertions.
|
||||
//
|
||||
// Every request is kept, not just the last one: vmctl issues several kinds of
|
||||
// query - `SHOW FIELD KEYS` unchunked, `SHOW TAG KEYS` and `SHOW SERIES`
|
||||
// chunked, and one `SELECT` per series - and the assertions verify that all of
|
||||
// them authenticate and address the database the same way. The assertions
|
||||
// cannot live in this handler because it runs on a server goroutine, where
|
||||
// t.Fatalf must not be called.
|
||||
func (s *influxMockServer) recordQuery(r *http.Request) {
|
||||
user, pass, ok := r.BasicAuth()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.requests = append(s.requests, influxRequest{
|
||||
Query: r.URL.Query(),
|
||||
User: user,
|
||||
Password: pass,
|
||||
HasAuth: ok,
|
||||
})
|
||||
}
|
||||
|
||||
// queryRequests returns a copy of the recorded /query requests.
|
||||
func (s *influxMockServer) queryRequests() []influxRequest {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return slices.Clone(s.requests)
|
||||
}
|
||||
|
||||
// handlePing must answer 204: the client library treats any other status as a
|
||||
// failed ping, and vmctl pings before querying.
|
||||
//
|
||||
// X-Influxdb-Version is set because real InfluxDB sets it. The client returns
|
||||
// the value from Ping, but vmctl discards it, so it is not asserted anywhere and
|
||||
// its content is arbitrary.
|
||||
//
|
||||
// The request is not recorded: /ping carries no db or rp to assert.
|
||||
func (s *influxMockServer) handlePing(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("X-Influxdb-Version", "1.8.0-mock")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// influxResponse mirrors the InfluxDB 1.x JSON query response.
|
||||
type influxResponse struct {
|
||||
Results []influxResult `json:"results"`
|
||||
}
|
||||
|
||||
type influxResult struct {
|
||||
StatementID int `json:"statement_id"`
|
||||
Series []influxRow `json:"series,omitempty"`
|
||||
Err string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type influxRow struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Columns []string `json:"columns"`
|
||||
Values [][]any `json:"values,omitempty"`
|
||||
}
|
||||
|
||||
// parseSelect splits a SELECT built by vmctl into its field, measurement and
|
||||
// WHERE clause. The statement shape is fixed by Series.fetchQuery:
|
||||
//
|
||||
// select "value" from "cpu" where "host"::tag='h1' and "env"::tag=''
|
||||
//
|
||||
// Identifiers containing a double quote are not supported, which is fine
|
||||
// because no test fixture uses one.
|
||||
func parseSelect(q string) (field, measurement, where string, err error) {
|
||||
rest, ok := strings.CutPrefix(q, `select "`)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("cannot parse select statement: %s", q)
|
||||
}
|
||||
field, rest, ok = strings.Cut(rest, `" from "`)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("cannot parse measurement in: %s", q)
|
||||
}
|
||||
measurement, rest, ok = strings.Cut(rest, `"`)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("unterminated measurement in: %s", q)
|
||||
}
|
||||
return field, measurement, strings.TrimPrefix(rest, " where "), nil
|
||||
}
|
||||
|
||||
// handleQuery serves the InfluxQL statements vmctl issues. Both the plain and
|
||||
// the chunked code paths of the client accept a single JSON response object,
|
||||
// so one implementation covers both.
|
||||
func (s *influxMockServer) handleQuery(w http.ResponseWriter, r *http.Request) {
|
||||
s.recordQuery(r)
|
||||
|
||||
q := strings.TrimSpace(r.URL.Query().Get("q"))
|
||||
lower := strings.ToLower(q)
|
||||
|
||||
// Content-Type is load-bearing: the client rejects any response that is not
|
||||
// application/json. X-Influxdb-Version only matters for 5xx replies, where
|
||||
// the client uses its absence to report a downstream proxy error instead of
|
||||
// an InfluxDB one; it is set here to mirror real InfluxDB.
|
||||
w.Header().Set("X-Influxdb-Version", "1.8.0-mock")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
var resp influxResponse
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "show field keys"):
|
||||
resp = influxResponse{Results: []influxResult{{Series: s.fieldKeys()}}}
|
||||
case strings.HasPrefix(lower, "show tag keys"):
|
||||
resp = influxResponse{Results: []influxResult{{Series: s.tagKeys()}}}
|
||||
case strings.HasPrefix(lower, "show series"):
|
||||
resp = influxResponse{Results: []influxResult{{Series: s.series()}}}
|
||||
case strings.HasPrefix(lower, "select"):
|
||||
rows, err := s.selectRows(q)
|
||||
if err != nil {
|
||||
resp = influxResponse{Results: []influxResult{{Err: err.Error()}}}
|
||||
break
|
||||
}
|
||||
resp = influxResponse{Results: []influxResult{{Series: rows}}}
|
||||
default:
|
||||
resp = influxResponse{Results: []influxResult{{Err: fmt.Sprintf("unsupported query: %s", q)}}}
|
||||
}
|
||||
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// fieldKeys serves `SHOW FIELD KEYS`: one row per measurement listing every
|
||||
// field key together with its type. vmctl uses the type to skip string fields.
|
||||
//
|
||||
// Rows are built by walking the points in order, so the output is deterministic
|
||||
// without sorting.
|
||||
func (s *influxMockServer) fieldKeys() []influxRow {
|
||||
rows := make(map[string]*influxRow)
|
||||
var order []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, p := range s.points {
|
||||
row, ok := rows[p.Measurement]
|
||||
if !ok {
|
||||
row = &influxRow{Name: p.Measurement, Columns: []string{"fieldKey", "fieldType"}}
|
||||
rows[p.Measurement] = row
|
||||
order = append(order, p.Measurement)
|
||||
}
|
||||
key := p.Measurement + "\x00" + p.Field
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
row.Values = append(row.Values, []any{p.Field, p.FieldType})
|
||||
}
|
||||
out := make([]influxRow, 0, len(order))
|
||||
for _, m := range order {
|
||||
out = append(out, *rows[m])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tagKeys serves `SHOW TAG KEYS`: one row per measurement listing its tag keys.
|
||||
//
|
||||
// Tag keys come from a map, so each row is sorted to keep the output stable.
|
||||
func (s *influxMockServer) tagKeys() []influxRow {
|
||||
rows := make(map[string]*influxRow)
|
||||
var order []string
|
||||
seen := make(map[string]struct{})
|
||||
for _, p := range s.points {
|
||||
row, ok := rows[p.Measurement]
|
||||
if !ok {
|
||||
row = &influxRow{Name: p.Measurement, Columns: []string{"tagKey"}}
|
||||
rows[p.Measurement] = row
|
||||
order = append(order, p.Measurement)
|
||||
}
|
||||
for k := range p.Tags {
|
||||
key := p.Measurement + "\x00" + k
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
row.Values = append(row.Values, []any{k})
|
||||
}
|
||||
}
|
||||
out := make([]influxRow, 0, len(order))
|
||||
for _, m := range order {
|
||||
row := *rows[m]
|
||||
sort.Slice(row.Values, func(a, b int) bool {
|
||||
return row.Values[a][0].(string) < row.Values[b][0].(string)
|
||||
})
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// series serves `SHOW SERIES`: a single row of series keys in the
|
||||
// `measurement,tag=value,...` form.
|
||||
func (s *influxMockServer) series() []influxRow {
|
||||
seen := make(map[string]struct{})
|
||||
var keys []string
|
||||
for _, p := range s.points {
|
||||
key := influxSeriesKey(p.Measurement, p.Tags)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
row := influxRow{Columns: []string{"key"}}
|
||||
for _, k := range keys {
|
||||
row.Values = append(row.Values, []any{k})
|
||||
}
|
||||
return []influxRow{row}
|
||||
}
|
||||
|
||||
// selectRows serves the per-series SELECT issued for every measurement/field
|
||||
// combination, returning the `time` and field columns.
|
||||
func (s *influxMockServer) selectRows(q string) ([]influxRow, error) {
|
||||
field, measurement, where, err := parseSelect(q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Tags the series must carry. An empty value means the tag must be absent,
|
||||
// which is how vmctl addresses series that lack a tag of the measurement.
|
||||
wantTags := make(map[string]string)
|
||||
for _, cond := range strings.Split(where, " and ") {
|
||||
// Conditions that are not tag comparisons - such as the time filter
|
||||
// added by -influx-filter-time-start/-end - are not series selectors.
|
||||
name, value, ok := strings.Cut(cond, "::tag=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
wantTags[strings.Trim(name, `"`)] = strings.Trim(value, `'`)
|
||||
}
|
||||
|
||||
row := influxRow{Name: measurement, Columns: []string{"time", field}}
|
||||
for _, p := range s.points {
|
||||
if p.Measurement != measurement || p.Field != field {
|
||||
continue
|
||||
}
|
||||
if !influxTagsMatch(p.Tags, wantTags) {
|
||||
continue
|
||||
}
|
||||
ts := time.Unix(p.Timestamp, 0).UTC().Format(time.RFC3339)
|
||||
row.Values = append(row.Values, []any{ts, p.Value})
|
||||
}
|
||||
if len(row.Values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return []influxRow{row}, nil
|
||||
}
|
||||
|
||||
// influxTagsMatch reports whether the series tags satisfy the conditions of a
|
||||
// SELECT built by vmctl. Every requested tag must match exactly; a requested
|
||||
// empty value requires the tag to be absent from the series.
|
||||
func influxTagsMatch(got, want map[string]string) bool {
|
||||
for k, v := range want {
|
||||
if v == "" {
|
||||
if _, ok := got[k]; ok {
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Every tag of the series must be constrained, otherwise the SELECT would
|
||||
// address more than one series.
|
||||
for k := range got {
|
||||
if _, ok := want[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// influxSeriesKey builds the `measurement,tag=value,...` key used by SHOW SERIES.
|
||||
//
|
||||
// The tags must be sorted: the key doubles as the deduplication key, so an
|
||||
// unstable order would report the same series more than once. tagsKey already
|
||||
// sorts, so it is reused here.
|
||||
func influxSeriesKey(measurement string, tags map[string]string) string {
|
||||
if len(tags) == 0 {
|
||||
return measurement
|
||||
}
|
||||
return measurement + "," + tagsKey(tags)
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func (ms *PrometheusMockStorage) Read(_ context.Context, query *prompb.Query, so
|
||||
}
|
||||
|
||||
if !notMatch {
|
||||
q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples})
|
||||
q.Timeseries = append(q.Timeseries, &prompb.TimeSeries{Labels: s.Labels, Samples: s.Samples, Histograms: s.Histograms})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/golang/snappy"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/prometheus/prometheus/storage/remote"
|
||||
@@ -86,10 +87,17 @@ func (rrs *RemoteReadServer) getReadHandler(t *testing.T) http.Handler {
|
||||
samples = append(samples, sample)
|
||||
}
|
||||
}
|
||||
var histograms []prompb.Histogram
|
||||
for _, h := range s.Histograms {
|
||||
if h.Timestamp >= startTs && h.Timestamp < endTs {
|
||||
histograms = append(histograms, h)
|
||||
}
|
||||
}
|
||||
var series prompb.TimeSeries
|
||||
if len(samples) > 0 {
|
||||
if len(samples) > 0 || len(histograms) > 0 {
|
||||
series.Labels = s.Labels
|
||||
series.Samples = samples
|
||||
series.Histograms = histograms
|
||||
}
|
||||
ts[i] = &series
|
||||
}
|
||||
@@ -317,6 +325,37 @@ func generateRemoteReadSamples(idx int, startTime, endTime, numOfSamples int64)
|
||||
return samples
|
||||
}
|
||||
|
||||
// GenerateRemoteReadHistogramSeries generates a remote read series
|
||||
// with native histogram samples within the given time range.
|
||||
func GenerateRemoteReadHistogramSeries(start, end, numOfSamples int64) []*prompb.TimeSeries {
|
||||
timeSeries := &prompb.TimeSeries{
|
||||
Labels: []prompb.Label{
|
||||
{Name: labels.MetricName, Value: "vm_histogram_metric"},
|
||||
{Name: "job", Value: "0"},
|
||||
},
|
||||
}
|
||||
|
||||
delta := (end - start) / numOfSamples
|
||||
mul := int64(0)
|
||||
for t := start; t != end; t += delta {
|
||||
mul++
|
||||
h := &histogram.Histogram{
|
||||
Schema: 0,
|
||||
Count: uint64(10 * mul),
|
||||
Sum: 25.5 * float64(mul),
|
||||
ZeroThreshold: 0.001,
|
||||
ZeroCount: uint64(2 * mul),
|
||||
PositiveSpans: []histogram.Span{{Offset: 0, Length: 2}},
|
||||
PositiveBuckets: []int64{mul, 2 * mul},
|
||||
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
|
||||
NegativeBuckets: []int64{4 * mul},
|
||||
}
|
||||
timeSeries.Histograms = append(timeSeries.Histograms, prompb.FromIntHistogram(t*1000, h))
|
||||
}
|
||||
|
||||
return []*prompb.TimeSeries{timeSeries}
|
||||
}
|
||||
|
||||
func labelsToLabelsProto(ls labels.Labels) []prompb.Label {
|
||||
result := make([]prompb.Label, 0, ls.Len())
|
||||
ls.Range(func(l labels.Label) {
|
||||
|
||||
272
apptest/tests/vmctl_influx_migration_test.go
Normal file
272
apptest/tests/vmctl_influx_migration_test.go
Normal file
@@ -0,0 +1,272 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
influxTestDatabase = "testdb"
|
||||
influxTestToken = "my-secret-token"
|
||||
// defaultV1CompatUser mirrors the username vmctl sends when authenticating
|
||||
// to InfluxDB 2.x with an API token. The 1.x compatibility API requires a
|
||||
// username but ignores its value.
|
||||
influxV1CompatUser = "vmctl"
|
||||
)
|
||||
|
||||
func TestSingleVmctlInfluxV2Migration(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
|
||||
baseTS := time.Now().Add(-2 * time.Hour).Truncate(time.Minute).Unix()
|
||||
points := newInfluxTestPoints(baseTS)
|
||||
|
||||
influx := newInfluxMockServer(t, points)
|
||||
defer influx.close()
|
||||
|
||||
vmctlFlags := []string{
|
||||
`influx`,
|
||||
`--influx-version=2`,
|
||||
`--influx-addr=` + influx.httpAddr(),
|
||||
`--influx-token=` + influxTestToken,
|
||||
`--influx-database=` + influxTestDatabase,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
`-s`,
|
||||
}
|
||||
|
||||
testVmctlInfluxMigration(tc, vmsingleDst, vmctlFlags, points, baseTS)
|
||||
|
||||
assertInfluxRequests(t, influx, influxV1CompatUser, influxTestToken, "")
|
||||
}
|
||||
|
||||
func TestSingleVmctlInfluxV1Migration(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
|
||||
baseTS := time.Now().Add(-2 * time.Hour).Truncate(time.Minute).Unix()
|
||||
points := newInfluxTestPoints(baseTS)
|
||||
|
||||
influx := newInfluxMockServer(t, points)
|
||||
defer influx.close()
|
||||
|
||||
vmctlFlags := []string{
|
||||
`influx`,
|
||||
`--influx-addr=` + influx.httpAddr(),
|
||||
`--influx-user=user`,
|
||||
`--influx-password=pass`,
|
||||
`--influx-database=` + influxTestDatabase,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
`-s`,
|
||||
}
|
||||
|
||||
testVmctlInfluxMigration(tc, vmsingleDst, vmctlFlags, points, baseTS)
|
||||
assertInfluxRequests(t, influx, "user", "pass", "autogen")
|
||||
}
|
||||
|
||||
func TestClusterVmctlInfluxV2Migration(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
cluster := tc.MustStartDefaultCluster()
|
||||
vmAddr := fmt.Sprintf("http://%s/", cluster.Vminsert.HTTPAddr())
|
||||
|
||||
baseTS := time.Now().Add(-2 * time.Hour).Truncate(time.Minute).Unix()
|
||||
points := newInfluxTestPoints(baseTS)
|
||||
|
||||
influx := newInfluxMockServer(t, points)
|
||||
defer influx.close()
|
||||
|
||||
vmctlFlags := []string{
|
||||
`influx`,
|
||||
`--influx-version=2`,
|
||||
`--influx-addr=` + influx.httpAddr(),
|
||||
`--influx-token=` + influxTestToken,
|
||||
`--influx-database=` + influxTestDatabase,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--vm-account-id=0`,
|
||||
`--disable-progress-bar=true`,
|
||||
`-s`,
|
||||
}
|
||||
|
||||
testVmctlInfluxMigration(tc, cluster, vmctlFlags, points, baseTS)
|
||||
assertInfluxRequests(t, influx, influxV1CompatUser, influxTestToken, "")
|
||||
}
|
||||
|
||||
func testVmctlInfluxMigration(
|
||||
tc *apptest.TestCase,
|
||||
queries apptest.PrometheusWriteQuerier,
|
||||
vmctlFlags []string,
|
||||
points []influxPoint,
|
||||
baseTS int64,
|
||||
) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
queryStart := time.Unix(baseTS-3600, 0).UTC().Format(time.RFC3339)
|
||||
queryEnd := time.Unix(baseTS+7200, 0).UTC().Format(time.RFC3339)
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
// Nothing is stored before the migration runs.
|
||||
got := queries.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: queryStart,
|
||||
})
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response before migration (-want, +got):\n%s", diff)
|
||||
}
|
||||
|
||||
tc.MustStartVmctl("vmctl", vmctlFlags)
|
||||
queries.ForceFlush(t)
|
||||
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Retries: 300,
|
||||
Msg: `unexpected metrics migrated from influx`,
|
||||
Got: func() any {
|
||||
r := queries.PrometheusAPIV1Export(t, `{__name__!=""}`, apptest.QueryOpts{
|
||||
Start: queryStart,
|
||||
End: queryEnd,
|
||||
})
|
||||
r.Sort()
|
||||
return r.Data.Result
|
||||
},
|
||||
Want: buildExpectedInfluxResult(t, points),
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// buildExpectedInfluxResult derives the expected VictoriaMetrics contents from
|
||||
// the same points the mock server serves: metric name is
|
||||
// `<measurement>_<field>`, tags become labels, the source database is added as
|
||||
// the `db` label, booleans become 1/0 and string fields are skipped.
|
||||
func buildExpectedInfluxResult(t *testing.T, points []influxPoint) []*apptest.QueryResult {
|
||||
t.Helper()
|
||||
|
||||
grouped := map[string]*apptest.QueryResult{}
|
||||
for _, p := range points {
|
||||
if p.FieldType == "string" {
|
||||
continue
|
||||
}
|
||||
name := fmt.Sprintf("%s_%s", p.Measurement, p.Field)
|
||||
metric := map[string]string{
|
||||
"__name__": name,
|
||||
"db": influxTestDatabase,
|
||||
}
|
||||
for k, v := range p.Tags {
|
||||
metric[k] = v
|
||||
}
|
||||
key := tagsKey(metric)
|
||||
if _, ok := grouped[key]; !ok {
|
||||
grouped[key] = &apptest.QueryResult{Metric: metric}
|
||||
}
|
||||
grouped[key].Samples = append(grouped[key].Samples, &apptest.Sample{
|
||||
Timestamp: p.Timestamp * 1000,
|
||||
Value: influxExpectedValue(t, p.Value),
|
||||
})
|
||||
}
|
||||
out := make([]*apptest.QueryResult, 0, len(grouped))
|
||||
for _, v := range grouped {
|
||||
out = append(out, v)
|
||||
}
|
||||
resp := apptest.PrometheusAPIV1QueryResponse{
|
||||
Data: &apptest.QueryData{Result: out},
|
||||
}
|
||||
resp.Sort()
|
||||
return resp.Data.Result
|
||||
}
|
||||
|
||||
// influxExpectedValue converts a field value the way vmctl does: numbers pass
|
||||
// through and booleans become 1/0.
|
||||
func influxExpectedValue(t *testing.T, v any) float64 {
|
||||
t.Helper()
|
||||
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return val
|
||||
case int64:
|
||||
return float64(val)
|
||||
case bool:
|
||||
if val {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
default:
|
||||
t.Fatalf("unexpected field value type %T in test fixture; only float64, int64 and bool are supported", v)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// assertInfluxRequests verifies how vmctl authenticated to InfluxDB and which
|
||||
// database and retention policy it requested.
|
||||
func assertInfluxRequests(t *testing.T, s *influxMockServer, wantUser, wantPass, wantRP string) {
|
||||
t.Helper()
|
||||
|
||||
reqs := s.queryRequests()
|
||||
if len(reqs) == 0 {
|
||||
t.Fatalf("vmctl issued no /query requests")
|
||||
}
|
||||
for _, r := range reqs {
|
||||
if !r.HasAuth {
|
||||
t.Fatalf("no basic auth on %s", r.Query)
|
||||
}
|
||||
if r.User != wantUser || r.Password != wantPass {
|
||||
t.Fatalf("unexpected credentials for q=%q\ngot\n(%q, %q)\nwant\n(%q, %q)",
|
||||
r.Query.Get("q"), r.User, r.Password, wantUser, wantPass)
|
||||
}
|
||||
if rp := r.Query.Get("rp"); rp != wantRP {
|
||||
t.Fatalf("unexpected rp for q=%q\ngot\n%q\nwant\n%q", r.Query.Get("q"), rp, wantRP)
|
||||
}
|
||||
if db := r.Query.Get("db"); db != influxTestDatabase {
|
||||
t.Fatalf("unexpected db for q=%q\ngot\n%q\nwant\n%q", r.Query.Get("q"), db, influxTestDatabase)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newInfluxTestPoints returns points covering the cases that matter for the
|
||||
// migration: numeric field types, a boolean, a string field which must be
|
||||
// skipped, a series missing one of the measurement tags, and a measurement name
|
||||
// containing special characters.
|
||||
func newInfluxTestPoints(baseTS int64) []influxPoint {
|
||||
var points []influxPoint
|
||||
for i := range 10 {
|
||||
ts := baseTS + int64(i*60)
|
||||
|
||||
// Two series of `cpu`: the second one has no `env` tag, which exercises
|
||||
// the empty-tag conditions vmctl adds to its SELECT statements.
|
||||
points = append(points,
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "value", "float", ts, float64(i) + 0.5},
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "count", "integer", ts, int64(i)},
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "flag", "boolean", ts, i%2 == 0},
|
||||
influxPoint{"cpu", map[string]string{"host": "h1", "env": "prod"}, "note", "string", ts, "ignored"},
|
||||
influxPoint{"cpu", map[string]string{"host": "h2"}, "value", "float", ts, float64(i) - 2.25},
|
||||
influxPoint{"cpu", map[string]string{"host": "h2"}, "note", "string", ts, "ignored"},
|
||||
// Special characters in a measurement name, cf. issue #10892.
|
||||
influxPoint{"user_percent.mem.zwickel+", map[string]string{"host": "h1"}, "value", "float", ts, float64(i * 3)},
|
||||
)
|
||||
}
|
||||
return points
|
||||
}
|
||||
@@ -75,6 +75,118 @@ func TestClusterVmctlRemoteReadProtocol(t *testing.T) {
|
||||
testRemoteReadProtocol(tc, clusterDst, newRemoteReadServer, vmctlFlags)
|
||||
}
|
||||
|
||||
func TestSingleVmctlRemoteReadNativeHistograms(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
vmctlFlags := []string{
|
||||
`remote-read`,
|
||||
`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
|
||||
`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
|
||||
`--remote-read-step-interval=minute`,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
}
|
||||
|
||||
testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadServer, vmctlFlags)
|
||||
}
|
||||
|
||||
func TestSingleVmctlRemoteReadStreamNativeHistograms(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
vmctlFlags := []string{
|
||||
`remote-read`,
|
||||
`--remote-read-filter-time-start=2025-06-11T15:31:10Z`,
|
||||
`--remote-read-filter-time-end=2025-06-11T15:31:20Z`,
|
||||
`--remote-read-step-interval=minute`,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--remote-read-use-stream=true`,
|
||||
`--disable-progress-bar=true`,
|
||||
}
|
||||
|
||||
testRemoteReadNativeHistograms(tc, vmsingleDst, NewRemoteReadStreamServer, vmctlFlags)
|
||||
}
|
||||
|
||||
// testRemoteReadNativeHistograms verifies that native histograms are migrated
|
||||
// as _count, _sum and _bucket series with vmrange labels in the same way
|
||||
// as VictoriaMetrics converts native histograms received via Prometheus remote write protocol.
|
||||
func testRemoteReadNativeHistograms(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T, series []*prompb.TimeSeries) *RemoteReadServer, vmctlFlags []string) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
series := GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2)
|
||||
|
||||
rrs := newRemoteReadServer(t, series)
|
||||
defer rrs.Close()
|
||||
|
||||
vmctlFlags = append(vmctlFlags, `--remote-read-src-addr=`+rrs.HTTPAddr())
|
||||
tc.MustStartVmctl("vmctl", vmctlFlags)
|
||||
|
||||
sut.ForceFlush(t)
|
||||
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Retries: 300,
|
||||
Msg: `unexpected native histogram metrics stored on vmsingle via the prometheus protocol`,
|
||||
Got: func() any {
|
||||
got := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Start: "2025-06-11T15:31:10Z",
|
||||
End: "2025-06-11T15:32:20Z",
|
||||
})
|
||||
got.Sort()
|
||||
return got.Data.Result
|
||||
},
|
||||
Want: expectedNativeHistogramQueryResult(),
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// expectedNativeHistogramQueryResult returns the series expected to be stored in VictoriaMetrics
|
||||
// after migrating the series generated by GenerateRemoteReadHistogramSeries(1749655870, 1749655880, 2).
|
||||
func expectedNativeHistogramQueryResult() []*apptest.QueryResult {
|
||||
metric := func(name, vmrange string) map[string]string {
|
||||
m := map[string]string{
|
||||
"__name__": name,
|
||||
"job": "0",
|
||||
}
|
||||
if vmrange != "" {
|
||||
m["vmrange"] = vmrange
|
||||
}
|
||||
return m
|
||||
}
|
||||
samples := func(v1, v2 float64) []*apptest.Sample {
|
||||
return []*apptest.Sample{
|
||||
{Timestamp: 1749655870000, Value: v1},
|
||||
{Timestamp: 1749655875000, Value: v2},
|
||||
}
|
||||
}
|
||||
resp := &apptest.PrometheusAPIV1QueryResponse{
|
||||
Data: &apptest.QueryData{
|
||||
Result: []*apptest.QueryResult{
|
||||
{Metric: metric("vm_histogram_metric_count", ""), Samples: samples(10, 20)},
|
||||
{Metric: metric("vm_histogram_metric_sum", ""), Samples: samples(25.5, 51)},
|
||||
{Metric: metric("vm_histogram_metric_bucket", "-1.000e+00...-5.000e-01"), Samples: samples(4, 8)},
|
||||
{Metric: metric("vm_histogram_metric_bucket", "-1.000e-03...1.000e-03"), Samples: samples(2, 4)},
|
||||
{Metric: metric("vm_histogram_metric_bucket", "5.000e-01...1.000e+00"), Samples: samples(1, 2)},
|
||||
{Metric: metric("vm_histogram_metric_bucket", "1.000e+00...2.000e+00"), Samples: samples(3, 6)},
|
||||
},
|
||||
},
|
||||
}
|
||||
// sort in the same way as the exported result
|
||||
resp.Sort()
|
||||
return resp.Data.Result
|
||||
}
|
||||
|
||||
func testRemoteReadProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, newRemoteReadServer func(t *testing.T) *RemoteReadServer, vmctlFlags []string) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
@@ -156,13 +156,6 @@ See [our blog](https://victoriametrics.com/blog) for the latest articles written
|
||||
* [Why irate from Prometheus doesn't capture spikes](https://valyala.medium.com/why-irate-from-prometheus-doesnt-capture-spikes-45f9896d7832)
|
||||
* [VictoriaMetrics: PromQL compliance](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e)
|
||||
* [How do open source solutions for logs work: Elasticsearch, Loki and VictoriaLogs](https://itnext.io/how-do-open-source-solutions-for-logs-work-elasticsearch-loki-and-victorialogs-9f7097ecbc2f)
|
||||
* [How vmagent Collects and Ships Metrics Fast with Aggregation, Deduplication, and More](https://victoriametrics.com/blog/vmagent-how-it-works/)
|
||||
* [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/)
|
||||
* [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/)
|
||||
* [How vmstorage Processes Data: Retention, Merging, Deduplication...](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/)
|
||||
* [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/)
|
||||
* [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/)
|
||||
* [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/)
|
||||
|
||||
### Tutorials, guides and how-to articles
|
||||
|
||||
@@ -180,12 +173,6 @@ See [our guides](https://docs.victoriametrics.com/guides/) for the up-to-date gu
|
||||
* [Prometheus storage: tech terms for humans](https://valyala.medium.com/prometheus-storage-technical-terms-for-humans-4ab4de6c3d48)
|
||||
* [Cardinality explorer](https://victoriametrics.com/blog/cardinality-explorer/)
|
||||
* [Rules backfilling via vmalert](https://victoriametrics.com/blog/rules-replay/)
|
||||
* [vmagent: Key Features Explained in Under 15 Minutes](https://victoriametrics.com/blog/vmagent-key-features-explained/)
|
||||
* [Prometheus Metrics Explained: Counters, Gauges, Histograms & Summaries](https://victoriametrics.com/blog/prometheus-monitoring-metrics-counters-gauges-histogram-summaries/)
|
||||
* [Prometheus Monitoring: Instant Queries and Range Queries Explained](https://victoriametrics.com/blog/prometheus-monitoring-instant-range-query/)
|
||||
* [Prometheus Monitoring: Functions, Subqueries, Operators, and Modifiers](https://victoriametrics.com/blog/prometheus-monitoring-function-operator-modifier/)
|
||||
* [Prometheus Alerting 101: Rules, Recording Rules, and Alertmanager](https://victoriametrics.com/blog/alerting-recording-rules-alertmanager/)
|
||||
* [Alerting Best Practices](https://victoriametrics.com/blog/alerting-best-practices/)
|
||||
|
||||
### Other articles
|
||||
|
||||
|
||||
@@ -59,11 +59,6 @@ It increases cluster availability, and simplifies cluster maintenance as well as
|
||||
|
||||

|
||||
|
||||
> Further reading, deep dives into how each service works internally:
|
||||
> - `vmstorage`: [How vmstorage Handles Data Ingestion From vminsert](https://victoriametrics.com/blog/vmstorage-how-it-handles-data-ingestion/), [How vmstorage's IndexDB Works](https://victoriametrics.com/blog/vmstorage-how-indexdb-works/), [How vmstorage Handles Query Requests From vmselect](https://victoriametrics.com/blog/vmstorage-how-it-handles-query-requests/).
|
||||
> - `vminsert`: [When Metrics Meet vminsert: A Data-Delivery Story](https://victoriametrics.com/blog/vminsert-how-it-works/).
|
||||
> - `vmselect`: [Inside vmselect: The Query Processing Engine of VictoriaMetrics](https://victoriametrics.com/blog/vmselect-how-it-works/).
|
||||
|
||||
## vmui
|
||||
|
||||
VictoriaMetrics cluster version provides UI for query troubleshooting and exploration. The UI is available at
|
||||
@@ -834,9 +829,9 @@ See also [minimum downtime strategy](#minimum-downtime-strategy).
|
||||
|
||||
## Slowness-based re-routing
|
||||
|
||||
By default{{% available_from "#" %}}, `vminsert` automatically [re-route writes](https://victoriametrics.com/blog/vminsert-how-it-works/#31-rerouting)
|
||||
away from the slowest `vmstorage` node to preserve maximum ingestion throughput. This prevents a single slow `vmstorage`
|
||||
node from throttling the entire cluster.
|
||||
By default{{% available_from "#" %}}, `vminsert` automatically re-routes writes away from the slowest `vmstorage` node
|
||||
to preserve maximum ingestion throughput. This prevents a single slow `vmstorage` node
|
||||
from throttling the entire cluster.
|
||||
|
||||
Re-routing occurs only when all of the following conditions hold:
|
||||
- the storage send buffer is full.
|
||||
@@ -883,7 +878,7 @@ See also [resource usage limits docs](#resource-usage-limits).
|
||||
|
||||
## Rebalancing
|
||||
|
||||
Every `vminsert` node [evenly spreads (shards) incoming data](https://victoriametrics.com/blog/vminsert-how-it-works/#3-sharding-and-buffering) among `vmstorage` nodes specified in the `-storageNode` command-line flag.
|
||||
Every `vminsert` node evenly spreads (shards) incoming data among `vmstorage` nodes specified in the `-storageNode` command-line flag.
|
||||
This guarantees even distribution of the ingested data among `vmstorage` nodes. When new `vmstorage` nodes are added to the `-storageNode`
|
||||
command-line flag at `vminsert`, then only newly ingested data is distributed evenly among old and new `vmstorage` nodes, while
|
||||
historical data remains on the old `vmstorage` nodes. This speeds up data ingestion and querying for the majority of production workloads,
|
||||
@@ -1031,7 +1026,7 @@ By default, VictoriaMetrics offloads replication to the underlying storage point
|
||||
which guarantees data durability. VictoriaMetrics supports application-level replication if replicated durable persistent disks cannot be used for some reason.
|
||||
|
||||
The replication can be enabled by passing `-replicationFactor=N` command-line flag to `vminsert`. This instructs `vminsert` to store `N` copies for every ingested sample
|
||||
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable. See [how `vminsert` replicates each sample to `N` `vmstorage` nodes](https://victoriametrics.com/blog/vminsert-how-it-works/#4-replication-and-sending-data-to-vmstorage) for details.
|
||||
on `N` distinct `vmstorage` nodes. This guarantees that all the stored data remains available for querying if up to `N-1` `vmstorage` nodes are unavailable.
|
||||
|
||||
Passing `-replicationFactor=N` command-line flag to `vmselect` instructs it to not mark responses as `partial` if less than `-replicationFactor` vmstorage nodes are unavailable during the query.
|
||||
See [cluster availability docs](#cluster-availability) for details.
|
||||
@@ -1066,7 +1061,7 @@ deduplication can't be guaranteed when samples and sample duplicates for the sam
|
||||
- when `vmstorage` node has no enough capacity for processing incoming data stream. Then `vminsert` re-routes new samples to other `vmstorage` nodes.
|
||||
|
||||
It is recommended to set **the same** `-dedup.minScrapeInterval` command-line flag value to both `vmselect` and `vmstorage` nodes
|
||||
to ensure query results consistency, even if [storage layer didn't complete deduplication](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#deduplication) yet.
|
||||
to ensure query results consistency, even if storage layer didn't complete deduplication yet.
|
||||
|
||||
## Metrics Metadata
|
||||
|
||||
|
||||
@@ -1405,9 +1405,6 @@ for fast block lookups, which belong to the given `TSID` and cover the given tim
|
||||
* various background maintenance tasks such as [de-duplication](#deduplication), [downsampling](#downsampling)
|
||||
and [freeing up disk space for the deleted time series](#how-to-delete-time-series) are performed during the merge
|
||||
|
||||
See how `vmstorage` [selects parts for background merging](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#merge-process),
|
||||
including merge limits and monitoring metrics.
|
||||
|
||||
Newly added `parts` either successfully appear in the storage or fail to appear.
|
||||
The newly added `part` is atomically registered in the `parts.json` file under the corresponding partition
|
||||
after it is fully written and [fsynced](https://man7.org/linux/man-pages/man2/fsync.2.html) to the storage.
|
||||
@@ -1535,8 +1532,6 @@ are **eventually deleted** during [background merge](https://medium.com/@valyala
|
||||
The time range covered by data part is **not limited by retention period unit**. One data part can cover hours or days of
|
||||
data. Hence, a data part can be deleted only **when fully outside the configured retention**.
|
||||
See more about partitions and parts in the [Storage section](#storage).
|
||||
See how the [retention and free-disk watchers manage storage](https://victoriametrics.com/blog/vmstorage-retention-merging-deduplication/#retention-free-disk-space-guard-and-downsampling)
|
||||
for implementation details and monitoring metrics.
|
||||
|
||||
The maximum disk space usage for a given `-retentionPeriod` is going to be (`-retentionPeriod` + 1) months.
|
||||
For example, if `-retentionPeriod` is set to 1, data for January is deleted on March 1st.
|
||||
|
||||
@@ -32,14 +32,20 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
* SECURITY: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmselect` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): restrict `/api/v1/admin/tsdb/delete_series`, `/tags/delSeries` endpoints to `POST` method only to prevent some [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery)-based data deletion attacks. See [#5552](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5552).
|
||||
|
||||
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `name` label identifying the corresponding `-remoteWrite.url` target to the `vm_persistentqueue_*` metrics exposed by persistent queue. See [#7944](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7944). Thanks to @tIGO for contribution.
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add `-replay.continueWithExecutionErr` flag to allow continuing to replay other rules when a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit. See [11313](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11313).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): add template variable `$interval` to expose the alerting rule group's evaluation interval. This allows generating dashboard links with a lookback window relative to the rule's interval, for example `&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}`. See this issue [#11232](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11232) for more details. Thanks to @1solomonwakhungu for contribution.
|
||||
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`,
|
||||
`NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): support [Prometheus native histograms](https://prometheus.io/docs/specs/native_histograms/) migration in [remote read mode](https://docs.victoriametrics.com/victoriametrics/vmctl/remoteread/). Native histograms are converted into `_count`, `_sum` and `_bucket` series with `vmrange` labels in the same way as VictoriaMetrics converts native histograms received via Prometheus remote write protocol, except that for native histograms with custom buckets the original bucket bounds are preserved instead of being estimated with the exponential formula. Previously native histograms were silently ignored in `SAMPLES` mode, while in stream mode the migration failed with `EOF` error. See [#11292](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11292). Thanks to @liuxu623 for contribution.
|
||||
* FEATURE: `vminsert` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): enable [slowness-based re-routing](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#slowness-based-re-routing) by default. Previously, `-disableRerouting` defaulted to `true`, which limited ingestion throughput to the slowest `vmstorage` node. Now `-disableRerouting` defaults to `false`, so `vminsert` automatically routes data away from the slowest `vmstorage` node, improving overall ingestion performance. Slowness re-routing is automatically disabled when `-replicationFactor` is greater than 1. See [#11287](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11287).
|
||||
* FEATURE: [vmui](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#vmui): persist the selected auto-refresh interval in the URL. See [VictoriaLogs#1310](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1310).
|
||||
* FEATURE: [dashboards](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/dashboards): add `Fsync avg duration` panel to the Troubleshooting section of the single-node, cluster, and vmagent dashboards. This panel surfaces degradation of IO operation for faster incident triage. See [#10432](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10432).
|
||||
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/) and [alerts](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules): introduce `vm_backup_last_success_at` metric to track the last successful backup by type. Add [alerting rules](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/deployment/docker/rules/alerts-vmbackupmanager.yml) `NoLatestBackupWithinLastDay`, `NoHourlyBackupWithinLastDay`, `NoDailyBackupWithinLast3Days`, `NoWeeklyBackupWithinLast14Days` and `NoMonthlyBackupWithinLast62Days` to remind users about the missing backups. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
* FEATURE: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/): add `-remoteWrite.obfuscateLabels` flag for hashing values of the specified labels before sending metrics to the corresponding `-remoteWrite.url`. This allows sharing metrics with external systems while keeping sensitive label values hidden. See [#10599](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10599).
|
||||
* FEATURE: [vmctl](https://docs.victoriametrics.com/victoriametrics/vmctl/): add the ability to migrate data from InfluxDB 2.x in the `influx` mode via the new `-influx-version` and `-influx-token` command-line flags. Pass `-influx-version=2` together with `-influx-token` to authenticate with an InfluxDB 2.x API token; `-influx-database` then accepts the bucket name. Migration goes through the [InfluxDB 1.x compatibility API](https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/), which InfluxDB OSS exposes for every bucket automatically, so no database and retention policy mapping has to be created manually. See [#5914](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5914).
|
||||
|
||||
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): properly drop data points filtered out by an inner [comparison operation](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators) when its result is used on the right side of another comparison. Previously, queries like `foo != (bar > 100)` could return unexpected results because filtered-out data points are represented internally as `NaN`, and `value != NaN` evaluates to `true`. Comparisons against explicitly present `NaN` values keep the previous behavior. See [#10018](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10018). Thanks to @zasdaym for contribution.
|
||||
* BUGFIX: [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) and [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/): ignore HTTP proxy environment variables when scraping targets over Unix domain sockets. See [#11318](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/11318). Thanks to @lwmacct for contribution.
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): fixed the display of rule state badges on the `Groups` page in the web UI. See [#11160](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11160).
|
||||
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/victoriametrics/vmbackupmanager/): previously, `vmbackupmanager` was crashing on startup when it failed to restore backup state from remote storage, causing a crash loop. Now it logs the error and continues running, retrying the state restore before each scheduled backup. Added `vm_backup_errors_total{type="restoreState"}` metric to track backup state restore failures. See [#11217](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/11217).
|
||||
|
||||
|
||||
@@ -141,7 +141,6 @@ to other remote storage systems that support Prometheus `remote_write` protocol
|
||||
If a single remote storage instance is temporarily unavailable, the collected data remains available on the other remote storage instances.
|
||||
`vmagent` buffers the collected data in files at `-remoteWrite.tmpDataPath` until the remote storage becomes available again.
|
||||
Then it sends the buffered data to the remote storage in order to prevent data gaps.
|
||||
See how `vmagent` [selects shards and places replicas](https://victoriametrics.com/blog/vmagent-how-it-works/#step-4-sharding--replication) for implementation details.
|
||||
|
||||
[VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) already supports replication,
|
||||
so there is no need to specify multiple `-remoteWrite.url` flags when writing data to the same cluster.
|
||||
@@ -151,8 +150,7 @@ See [these docs](https://docs.victoriametrics.com/victoriametrics/cluster-victor
|
||||
|
||||
`vmagent` can add, remove, or update labels on the collected data before sending it to the remote storage.
|
||||
It can filter scrape targets or remove unwanted samples via Prometheus-like relabeling.
|
||||
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for configuration examples.
|
||||
For ingestion pipeline internals, see how `vmagent` [applies global relabeling and cardinality limits](https://victoriametrics.com/blog/vmagent-how-it-works/#step-2-global-relabeling-cardinality-reduction).
|
||||
Please see the [Relabeling cookbook](https://docs.victoriametrics.com/victoriametrics/relabeling/) for details.
|
||||
|
||||
### Sharding among remote storages
|
||||
|
||||
@@ -270,9 +268,7 @@ for the collected samples. Examples:
|
||||
```sh
|
||||
./vmagent -remoteWrite.url=http://remote-storage/api/v1/write -streamAggr.dropInputLabels=replica -streamAggr.dedupInterval=60s
|
||||
```
|
||||
|
||||
See how `vmagent` [orders global deduplication and stream aggregation](https://victoriametrics.com/blog/vmagent-how-it-works/#step-3-global-deduplication--stream-aggregation) in the ingestion pipeline.
|
||||
|
||||
|
||||
### Monitoring Data eXchange
|
||||
|
||||
The Monitoring Data eXchange (MDX){{% available_from "v1.147.0" %}} feature allows `vmagent` to forward only VictoriaMetrics metrics to selected `-remoteWrite.url` destinations while dropping metrics from non-VictoriaMetrics services.
|
||||
@@ -363,8 +359,6 @@ in addition to the pull-based Prometheus-compatible targets' scraping:
|
||||
* Prometheus exposition format via `http://<vmagent>:8429/api/v1/import/prometheus`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-data-in-prometheus-exposition-format) for details.
|
||||
* Arbitrary CSV data via `http://<vmagent>:8429/api/v1/import/csv`. See [these docs](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#how-to-import-csv-data).
|
||||
|
||||
See how `vmagent` [handles concurrency, decompression, and stream parsing](https://victoriametrics.com/blog/vmagent-how-it-works/#step-1-receiving-data-via-api-or-scrape) during ingestion.
|
||||
|
||||
## How to collect metrics in Prometheus format
|
||||
|
||||
Specify the path to the `prometheus.yml` file via the `-promscrape.config` command-line flag. `vmagent` takes into account the following
|
||||
@@ -1072,7 +1066,6 @@ This behavior can be changed with the `-remoteWrite.inmemoryQueues` {{% availabl
|
||||
When set to a non-zero value, vmagent starts the given number of additional workers,
|
||||
which send only recently ingested data from the in-memory queue, while the workers configured via `-remoteWrite.queues` drain the file-based backlog concurrently.
|
||||
This reduces the delivery lag for fresh samples after remote storage outages or slowdowns. The flag can be set individually per each `-remoteWrite.url`.
|
||||
See how the [in-memory and file-based queues manage blocks](https://victoriametrics.com/blog/vmagent-how-it-works/#in-memory-queue) for implementation details.
|
||||
|
||||
Note that these workers are started in addition to the workers configured via `-remoteWrite.queues`, so the total number of concurrent connections to
|
||||
the remote storage becomes the sum of both flags. Take this into account if the remote storage limits the number of concurrent requests.
|
||||
|
||||
@@ -341,6 +341,7 @@ The following variables are available in templating:
|
||||
| $externalLabels or .ExternalLabels | List of labels configured via `-external.label` command-line flag. | `Issues with {{ $labels.instance }} (datacenter-{{ $externalLabels.dc }})` |
|
||||
| $externalURL or .ExternalURL | URL configured via `-external.url` command-line flag. Used for cases when vmalert is hidden behind proxy. | `Visit {{ $externalURL }} for more details` |
|
||||
| $isPartial or .IsPartial | Indicates whether the latest rule query response from the datasource(that supports returning `isPartial` option, such as vmcluster) could be partial. | `{{ if $isPartial }}WARNING: The latest alert state may be a false alarm due to a partial response from the datasource.{{ end }}` |
|
||||
| $interval or .Interval | Alerting rule group's evaluation interval. | `http://vm-grafana.com/<dashboard-id>?viewPanel=<panel-id>&from={{ ($activeAt.Add (parseDurationTime (printf "-%s" .Interval))).UnixMilli }}&to={{ $activeAt.UnixMilli }}` |
|
||||
|
||||
Additionally, `vmalert` provides some extra templating functions listed in [template functions](#template-functions) and [reusable templates](#reusable-templates).
|
||||
|
||||
@@ -1024,9 +1025,6 @@ Try the following tips to avoid common issues:
|
||||
In that case, the default step will be used (`-datasource.queryStep`) and may cause unexpected results compared to
|
||||
executing this query in vmui/Grafana, where step is adjusted differently.
|
||||
|
||||
See [practical examples for reducing alert noise](https://victoriametrics.com/blog/alerting-best-practices/#reducing-noise),
|
||||
including aggregating alerts and configuring inhibition.
|
||||
|
||||
### Rule state
|
||||
|
||||
vmalert keeps the last `-rule.updateEntriesLimit` updates (or `update_entries_limit` [per-rule config](https://docs.victoriametrics.com/victoriametrics/vmalert/#alerting-rules))
|
||||
@@ -1082,9 +1080,6 @@ Sometimes, it's hard to understand why a specific alert fired or not. Keep in mi
|
||||
If evaluation returns error (i.e. datasource is unavailable), alert state doesn't change.
|
||||
If at least one evaluation returns no data, then alert's `for` state resets.
|
||||
|
||||
See [how to tune the `for` parameter](https://victoriametrics.com/blog/alerting-best-practices/#the-for-param),
|
||||
including its tradeoff with the query lookbehind window.
|
||||
|
||||
> Note: The alert state is tracked separately for each time series returned during evaluation.
|
||||
> For example, if the 1st evaluation returns series A and B, and the 2nd evaluation returns only B – the alert will remain active **only for B**.
|
||||
|
||||
|
||||
@@ -404,6 +404,8 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
|
||||
Optional TLS server name to use for connections to -remoteWrite.url. By default, the server name from -remoteWrite.url is used
|
||||
-remoteWrite.url string
|
||||
Optional URL to persist alerts state and recording rules results in form of timeseries. It must support either VictoriaMetrics remote write protocol or Prometheus remote_write protocol. Supports address in the form of IP address with a port (e.g., http://127.0.0.1:8428) or DNS SRV record. For example, if -remoteWrite.url=http://127.0.0.1:8428 is specified, then the alerts state will be written to http://127.0.0.1:8428/api/v1/write . See also -remoteWrite.disablePathAppend, '-remoteWrite.showURL'.
|
||||
-replay.continueWithExecutionErr bool
|
||||
Whether to continue replaying other rules if a rule execution fails with a 422 response code, which can happen due to an expression syntax error or a resource limit being hit.
|
||||
-replay.disableProgressBar
|
||||
Whether to disable rendering progress bars during the replay. Progress bar rendering might be verbose or break the logs parsing, so it is recommended to be disabled when not used in interactive mode.
|
||||
-replay.maxDatapointsPerQuery int
|
||||
@@ -411,7 +413,7 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmalert/ .
|
||||
-replay.ruleEvaluationConcurrency int
|
||||
The maximum number of concurrent '/query_range' requests when replay recording rule or alerting rule with for=0. Increasing this value when replaying for a long time, since each request is limited by -replay.maxDatapointsPerQuery. (default 1)
|
||||
-replay.ruleRetryAttempts int
|
||||
Defines how many retries to make before giving up on rule if request for it returns an error. (default 5)
|
||||
Defines how many retries to make before giving up on rule if request for it returns a retriable error. (default 5)
|
||||
-replay.rulesDelay duration
|
||||
Delay before evaluating the next rule within the group. Is important for chained rules. Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting. (default 1s)
|
||||
-replay.timeFrom string
|
||||
|
||||
@@ -7,9 +7,12 @@ menu:
|
||||
identifier: "vmctl-influxdb"
|
||||
weight: 2
|
||||
---
|
||||
`vmctl` can migrate historical data from InfluxDB (v1) to VictoriaMetrics. See `./vmctl influx --help` for details and
|
||||
`vmctl` can migrate historical data from InfluxDB v1 and v2 to VictoriaMetrics. See `./vmctl influx --help` for details and
|
||||
full list of flags. Also see [migrating data from InfluxDB to VictoriaMetrics](https://docs.victoriametrics.com/guides/migrate-from-influx/) article.
|
||||
|
||||
For InfluxDB v2 see [InfluxDB v2](#influxdb-v2) below. The examples in this section use InfluxDB v1,
|
||||
which is the default (`--influx-version=1`).
|
||||
|
||||
To start migration, specify the InfluxDB address `--influx-addr`, database `--influx-database` and VictoriaMetrics address `--vm-addr`:
|
||||
```sh
|
||||
./vmctl influx --influx-addr=http://<influx-addr>:8086 \
|
||||
@@ -92,8 +95,45 @@ See more about [time filtering in InfluxDB](https://docs.influxdata.com/influxdb
|
||||
|
||||
## InfluxDB v2
|
||||
|
||||
Migrating data from InfluxDB v2.x is not supported yet ([#32](https://github.com/VictoriaMetrics/vmctl/issues/32)).
|
||||
You may find useful a 3rd party solution for this - <https://github.com/jonppe/influx_to_victoriametrics>.
|
||||
Migrating data from InfluxDB v2 is supported {{% available_from "#" %}} via the `--influx-version=2` flag. In this mode vmctl reads
|
||||
data through the [InfluxDB 1.x compatibility API](https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/)
|
||||
of InfluxDB v2, so migration uses the same queries and produces the same
|
||||
[data mapping](#data-mapping) as for InfluxDB v1. Only authentication differs:
|
||||
|
||||
- `--influx-token` must be set to an InfluxDB v2 [API token](https://docs.influxdata.com/influxdb/v2/admin/tokens/).
|
||||
It is sent as the password of the compatibility API. The username is required by that API but its value
|
||||
is ignored, so `--influx-user` may be left unset.
|
||||
- `--influx-database` accepts the name of the **bucket** to migrate.
|
||||
- `--influx-retention-policy` should be left unset. InfluxDB then resolves the default database and
|
||||
retention policy (DBRP) mapping of the bucket. Set it only if the bucket is exposed through an explicit
|
||||
mapping with a non-default retention policy name.
|
||||
|
||||
```sh
|
||||
./vmctl influx --influx-version=2 \
|
||||
--influx-addr=http://<influx-addr>:8086 \
|
||||
--influx-token=<influx-token> \
|
||||
--influx-database=<bucket-name> \
|
||||
--vm-addr=http://<victoriametrics-addr>:8428
|
||||
```
|
||||
|
||||
InfluxDB OSS automatically creates a *virtual* DBRP mapping for every bucket, where the database name equals
|
||||
the bucket name. No mapping has to be created manually before the migration. Existing mappings can be listed
|
||||
with:
|
||||
|
||||
```sh
|
||||
curl -s 'http://<influx-addr>:8086/api/v2/dbrps?org=<org>' \
|
||||
--header 'Authorization: Token <influx-token>'
|
||||
```
|
||||
|
||||
If the bucket must be reachable under a different database name, or if several retention policies are mapped
|
||||
to different buckets, create the mapping explicitly. See
|
||||
[Database and retention policy mapping](https://docs.influxdata.com/influxdb/v2/api-guide/influxdb-1x/dbrp/)
|
||||
and pass its `database` and `retention_policy` values via `--influx-database` and `--influx-retention-policy`.
|
||||
|
||||
All the other `--influx-*` flags behave the same as for InfluxDB v1, including
|
||||
[filtering](#filtering) via `--influx-filter-series` and the time filters, since the compatibility API
|
||||
accepts the same InfluxQL statements. When using `--influx-filter-series` with an `ON <database_name>`
|
||||
clause, the database name is the one of the DBRP mapping, which for a virtual mapping is the bucket name.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -90,7 +90,11 @@ func newClient(ctx context.Context, sw *ScrapeWork) (*client, error) {
|
||||
}
|
||||
|
||||
tr := httputil.NewTransport(false, "vm_promscrape")
|
||||
if proxyURLFunc != nil {
|
||||
if sw.UnixSocket != "" {
|
||||
// Unix sockets are direct local endpoints and cannot be reached via HTTP proxies.
|
||||
// Proxy could be set implicitly via global env variable HTTP_PROXY
|
||||
tr.Proxy = nil
|
||||
} else if proxyURLFunc != nil {
|
||||
tr.Proxy = proxyURLFunc
|
||||
}
|
||||
tr.TLSHandshakeTimeout = 10 * time.Second
|
||||
|
||||
Reference in New Issue
Block a user