mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-01 16:12:18 +03:00
Compare commits
93 Commits
docs-fix-c
...
issue-7717
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
35d388dff9 | ||
|
|
1038153097 | ||
|
|
18c32d7385 | ||
|
|
754a9bd563 | ||
|
|
2fcbf75539 | ||
|
|
676a88793a | ||
|
|
8d3e9d1dac | ||
|
|
09251f0a1e | ||
|
|
4ea5f8a84d | ||
|
|
cd52978096 | ||
|
|
f65e24b2ab | ||
|
|
0579e68409 | ||
|
|
f2aea8532f | ||
|
|
94473ed262 | ||
|
|
c646a66b60 | ||
|
|
ccf97a4143 | ||
|
|
66df8a5003 | ||
|
|
cea9505bab | ||
|
|
30ac8cd3fa | ||
|
|
a1f0b792af | ||
|
|
50f75d751f | ||
|
|
27f7bc81e0 | ||
|
|
90d23d7c9f | ||
|
|
f68c028673 | ||
|
|
f24bf391a4 | ||
|
|
bc64ecfa3d | ||
|
|
f0bbf6ec15 | ||
|
|
cff4bde4d6 | ||
|
|
1716f11677 | ||
|
|
b4932ed2da | ||
|
|
77f2ab139f | ||
|
|
5537140074 | ||
|
|
5d766bf7f1 | ||
|
|
5907239181 | ||
|
|
720c2bfa1d | ||
|
|
e971e6102e | ||
|
|
5cd6d7cfba | ||
|
|
907aa1973a | ||
|
|
d6dacd9771 | ||
|
|
5bb67a7f00 | ||
|
|
8c1c92d4c9 | ||
|
|
95ca45d05a | ||
|
|
828a2aaf17 | ||
|
|
007ae5a3f0 | ||
|
|
dcd23da4ba | ||
|
|
e33dbaf3d2 | ||
|
|
c68973a247 | ||
|
|
2c72ef0f38 | ||
|
|
bd0551da3b | ||
|
|
9f52c40b0b | ||
|
|
ba3b50df1d | ||
|
|
3cfeae7f1a | ||
|
|
32da04725b | ||
|
|
8ce4636bc0 | ||
|
|
6167ce655e | ||
|
|
f1e294aa2b | ||
|
|
b72bf6961d | ||
|
|
2b880fe7db | ||
|
|
9898743fbd | ||
|
|
ca372168ae | ||
|
|
323974164b | ||
|
|
d0b948289b | ||
|
|
aa429631a6 | ||
|
|
9e3cf9ab64 | ||
|
|
94601365ca | ||
|
|
02b5849d92 | ||
|
|
933f5b39d6 | ||
|
|
367cdb089f | ||
|
|
2a1b3866e1 | ||
|
|
327a103367 | ||
|
|
94830ea064 | ||
|
|
e84d88d390 | ||
|
|
51cdf23e1d | ||
|
|
55486b66ba | ||
|
|
095910342f | ||
|
|
16a4269de1 | ||
|
|
0cd2f800ba | ||
|
|
655406584a | ||
|
|
7d3175b438 | ||
|
|
e67e01bec1 | ||
|
|
2e0082f5c4 | ||
|
|
a7a06b3be6 | ||
|
|
31c92c4009 | ||
|
|
d332cac491 | ||
|
|
554c72aa60 | ||
|
|
0852b56fe2 | ||
|
|
b5c41a1f48 | ||
|
|
d1ffd83bf0 | ||
|
|
2c9c5d0366 | ||
|
|
8a8ff2fa14 | ||
|
|
ea569e7f51 | ||
|
|
06039e6f93 | ||
|
|
426cbff5f7 |
11
SECURITY.md
11
SECURITY.md
@@ -4,12 +4,11 @@
|
||||
|
||||
The following versions of VictoriaMetrics receive regular security fixes:
|
||||
|
||||
| Version | Supported |
|
||||
|---------|--------------------|
|
||||
| [latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
|
||||
| v1.102.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
|
||||
| v1.110.x [LTS line](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
|
||||
| other releases | :x: |
|
||||
| Version | Supported |
|
||||
|--------------------------------------------------------------------------------|--------------------|
|
||||
| [Latest release](https://docs.victoriametrics.com/victoriametrics/changelog/) | :white_check_mark: |
|
||||
| [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-releases/) | :white_check_mark: |
|
||||
| other releases | :x: |
|
||||
|
||||
See [this page](https://victoriametrics.com/security/) for more details.
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ type Group struct {
|
||||
// EvalDelay will adjust the `time` parameter of rule evaluation requests to compensate intentional query delay from datasource.
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5155
|
||||
EvalDelay *promutil.Duration `yaml:"eval_delay,omitempty"`
|
||||
Limit int `yaml:"limit,omitempty"`
|
||||
Limit *int `yaml:"limit,omitempty"`
|
||||
Rules []Rule `yaml:"rules"`
|
||||
Concurrency int `yaml:"concurrency"`
|
||||
// Labels is a set of label value pairs, that will be added to every rule.
|
||||
@@ -91,8 +91,8 @@ func (g *Group) Validate(validateTplFn ValidateTplFn, validateExpressions bool)
|
||||
if g.EvalOffset != nil && g.EvalDelay != nil {
|
||||
return fmt.Errorf("eval_offset cannot be used with eval_delay")
|
||||
}
|
||||
if g.Limit < 0 {
|
||||
return fmt.Errorf("invalid limit %d, shouldn't be less than 0", g.Limit)
|
||||
if g.Limit != nil && *g.Limit < 0 {
|
||||
return fmt.Errorf("invalid limit %d, shouldn't be less than 0", *g.Limit)
|
||||
}
|
||||
if g.Concurrency < 0 {
|
||||
return fmt.Errorf("invalid concurrency %d, shouldn't be less than 0", g.Concurrency)
|
||||
|
||||
@@ -181,9 +181,10 @@ func TestGroupValidate_Failure(t *testing.T) {
|
||||
EvalOffset: promutil.NewDuration(2 * time.Minute),
|
||||
}, false, "eval_offset should be smaller than interval")
|
||||
|
||||
limit := -1
|
||||
f(&Group{
|
||||
Name: "wrong limit",
|
||||
Limit: -1,
|
||||
Limit: &limit,
|
||||
}, false, "invalid limit")
|
||||
|
||||
f(&Group{
|
||||
|
||||
@@ -389,7 +389,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
|
||||
return []datasource.Metric{{Timestamps: []int64{0}, Values: []float64{math.NaN()}}}, nil
|
||||
}
|
||||
for _, s := range res.Data {
|
||||
ls, err := ar.expandLabelTemplates(s)
|
||||
ls, err := ar.expandLabelTemplates(s, qFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -482,7 +482,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
|
||||
expandedLabels := make([]*labelSet, len(res.Data))
|
||||
expandedAnnotations := make([]map[string]string, len(res.Data))
|
||||
for i, m := range res.Data {
|
||||
ls, err := ar.expandLabelTemplates(m)
|
||||
ls, err := ar.expandLabelTemplates(m, qFn)
|
||||
if err != nil {
|
||||
curState.Err = err
|
||||
return nil, curState.Err
|
||||
@@ -604,10 +604,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
|
||||
return append(tss, ar.toTimeSeries(ts.Unix())...), nil
|
||||
}
|
||||
|
||||
func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric) (*labelSet, error) {
|
||||
qFn := func(_ string) ([]datasource.Metric, error) {
|
||||
return nil, fmt.Errorf("`query` template isn't supported in rule label")
|
||||
}
|
||||
func (ar *AlertingRule) expandLabelTemplates(m datasource.Metric, qFn templates.QueryFn) (*labelSet, error) {
|
||||
ls, err := ar.toLabels(m, qFn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to expand label templates: %s", err)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
@@ -1429,3 +1430,142 @@ func TestAlertingRuleExec_Partial(t *testing.T) {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertingRule_QueryTemplateInLabels(t *testing.T) {
|
||||
fq := &datasource.FakeQuerier{}
|
||||
fakeGroup := Group{
|
||||
Name: "TestQueryTemplateInLabels",
|
||||
}
|
||||
|
||||
ar := &AlertingRule{
|
||||
Name: "test_alert",
|
||||
Labels: map[string]string{
|
||||
"suppress_for_mass_alert": `{{ if (printf "ALERTS{alertname='SomeAlert', alertstate='firing', device='%s'} == 1" $labels.device | query) }}true{{ else }}false{{ end }}`,
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"summary": "Test alert with query template in labels",
|
||||
},
|
||||
alerts: make(map[uint64]*notifier.Alert),
|
||||
}
|
||||
ar.GroupID = fakeGroup.GetID()
|
||||
ar.q = fq
|
||||
ar.state = &ruleState{
|
||||
entries: make([]StateEntry, 10),
|
||||
}
|
||||
|
||||
// Add a metric that should trigger the alert
|
||||
fq.Add(metricWithValueAndLabels(t, 1, "device", "sda1"))
|
||||
|
||||
ts := time.Now()
|
||||
_, err := ar.exec(context.TODO(), ts, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error with query template in labels: %s", err)
|
||||
}
|
||||
|
||||
// Verify that the alert was created and the query template was executed
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
|
||||
alert := ar.GetAlerts()[0]
|
||||
suppressLabel, exists := alert.Labels["suppress_for_mass_alert"]
|
||||
if !exists {
|
||||
t.Fatalf("expected 'suppress_for_mass_alert' label to exist")
|
||||
}
|
||||
// The query template should have been executed (even if it returns false due to mock data)
|
||||
if suppressLabel != "true" && suppressLabel != "false" {
|
||||
t.Fatalf("expected 'suppress_for_mass_alert' label to be 'true' or 'false', got '%s'", suppressLabel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAlertingRule_ActiveAtPreservedInAnnotations ensures that the fix for
|
||||
// https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9543 is preserved
|
||||
// while allowing query templates in labels (https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9783)
|
||||
func TestAlertingRule_ActiveAtPreservedInAnnotations(t *testing.T) {
|
||||
// wrap into synctest because of time manipulations
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
fq := &datasource.FakeQuerier{}
|
||||
|
||||
ar := &AlertingRule{
|
||||
Name: "TestActiveAtPreservation",
|
||||
Labels: map[string]string{
|
||||
"test_query_in_label": `{{ "static_value" }}`,
|
||||
},
|
||||
Annotations: map[string]string{
|
||||
"description": "Alert active since {{ $activeAt }}",
|
||||
},
|
||||
alerts: make(map[uint64]*notifier.Alert),
|
||||
q: fq,
|
||||
state: &ruleState{
|
||||
entries: make([]StateEntry, 10),
|
||||
},
|
||||
}
|
||||
|
||||
// Mock query result - return empty result to make suppress_for_mass_alert = false
|
||||
// (no need to add anything to fq for empty result)
|
||||
|
||||
// Add a metric that should trigger the alert
|
||||
fq.Add(metricWithValueAndLabels(t, 1, "instance", "server1"))
|
||||
|
||||
// First execution - creates new alert
|
||||
ts1 := time.Now()
|
||||
_, err := ar.exec(context.TODO(), ts1, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on first exec: %s", err)
|
||||
}
|
||||
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
|
||||
firstAlert := ar.GetAlerts()[0]
|
||||
// Verify first execution: activeAt should be ts1 and annotation should reflect it
|
||||
if !firstAlert.ActiveAt.Equal(ts1) {
|
||||
t.Fatalf("expected activeAt to be %v, got %v", ts1, firstAlert.ActiveAt)
|
||||
}
|
||||
|
||||
// Extract time from annotation (format will be like "Alert active since 2025-09-30 08:55:13.638551611 -0400 EDT m=+0.002928464")
|
||||
expectedTimeStr := ts1.Format("2006-01-02 15:04:05")
|
||||
if !strings.Contains(firstAlert.Annotations["description"], expectedTimeStr) {
|
||||
t.Fatalf("first exec annotation should contain time %s, got: %s", expectedTimeStr, firstAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Second execution - should preserve activeAt in annotation
|
||||
|
||||
// Ensure different timestamp with different seconds
|
||||
// sleep is non-blocking thanks to synctest
|
||||
time.Sleep(2 * time.Second)
|
||||
ts2 := time.Now()
|
||||
_, err = ar.exec(context.TODO(), ts2, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on second exec: %s", err)
|
||||
}
|
||||
|
||||
// Get the alert again (should be the same alert)
|
||||
if len(ar.alerts) != 1 {
|
||||
t.Fatalf("expected 1 alert, got %d", len(ar.alerts))
|
||||
}
|
||||
secondAlert := ar.GetAlerts()[0]
|
||||
|
||||
// Critical test: activeAt should still be ts1, not ts2
|
||||
if !secondAlert.ActiveAt.Equal(ts1) {
|
||||
t.Fatalf("activeAt should be preserved as %v, but got %v", ts1, secondAlert.ActiveAt)
|
||||
}
|
||||
|
||||
// Critical test: annotation should still contain ts1 time, not ts2
|
||||
if !strings.Contains(secondAlert.Annotations["description"], expectedTimeStr) {
|
||||
t.Fatalf("second exec annotation should still contain original time %s, got: %s", expectedTimeStr, secondAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Additional verification: annotation should NOT contain ts2 time
|
||||
ts2TimeStr := ts2.Format("2006-01-02 15:04:05")
|
||||
if strings.Contains(secondAlert.Annotations["description"], ts2TimeStr) {
|
||||
t.Fatalf("annotation should NOT contain new eval time %s, got: %s", ts2TimeStr, secondAlert.Annotations["description"])
|
||||
}
|
||||
|
||||
// Verify query template in labels still works (this would fail if query templates were broken)
|
||||
if firstAlert.Labels["test_query_in_label"] != "static_value" {
|
||||
t.Fatalf("expected test_query_in_label=static_value, got %s", firstAlert.Labels["test_query_in_label"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -24,6 +24,10 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ruleResultsLimit = flag.Int("rule.resultsLimit", 0, "Limits the number of alerts or recording results a single rule can produce. "+
|
||||
"Can be overridden by the limit option under group if specified. "+
|
||||
"If exceeded, the rule will be marked with an error and all its results will be discarded. "+
|
||||
"0 means no limit.")
|
||||
ruleUpdateEntriesLimit = flag.Int("rule.updateEntriesLimit", 20, "Defines the max number of rule's state updates stored in-memory. "+
|
||||
"Rule's updates are available on rule's Details page and are used for debugging purposes. The number of stored updates can be overridden per rule via update_entries_limit param.")
|
||||
resendDelay = flag.Duration("rule.resendDelay", 0, "MiniMum amount of time to wait before resending an alert to notifier.")
|
||||
@@ -111,7 +115,6 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
|
||||
Name: cfg.Name,
|
||||
File: cfg.File,
|
||||
Interval: cfg.Interval.Duration(),
|
||||
Limit: cfg.Limit,
|
||||
Concurrency: cfg.Concurrency,
|
||||
checksum: cfg.Checksum,
|
||||
Params: cfg.Params,
|
||||
@@ -128,6 +131,11 @@ func NewGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval ti
|
||||
if g.Interval == 0 {
|
||||
g.Interval = defaultInterval
|
||||
}
|
||||
if cfg.Limit != nil {
|
||||
g.Limit = *cfg.Limit
|
||||
} else {
|
||||
g.Limit = *ruleResultsLimit
|
||||
}
|
||||
if g.Concurrency < 1 {
|
||||
g.Concurrency = 1
|
||||
}
|
||||
|
||||
@@ -372,20 +372,54 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
|
||||
updateHeadersByConfig(w.Header(), hc.ResponseHeaders)
|
||||
w.WriteHeader(res.StatusCode)
|
||||
|
||||
copyBuf := copyBufPool.Get()
|
||||
copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
|
||||
_, err = io.CopyBuffer(w, res.Body, copyBuf.B)
|
||||
copyBufPool.Put(copyBuf)
|
||||
err = copyStreamToClient(w, res.Body)
|
||||
_ = res.Body.Close()
|
||||
if err != nil && !netutil.IsTrivialNetworkError(err) {
|
||||
if err != nil && !netutil.IsTrivialNetworkError(err) && !errors.Is(err, context.Canceled) {
|
||||
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
|
||||
requestURI := httpserver.GetRequestURI(r)
|
||||
|
||||
logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
|
||||
return true, false
|
||||
}
|
||||
return true, false
|
||||
}
|
||||
|
||||
func copyStreamToClient(client io.Writer, backend io.Reader) error {
|
||||
copyBuf := copyBufPool.Get()
|
||||
copyBuf.B = bytesutil.ResizeNoCopyNoOverallocate(copyBuf.B, 16*1024)
|
||||
defer copyBufPool.Put(copyBuf)
|
||||
buf := copyBuf.B
|
||||
|
||||
flusher, ok := client.(http.Flusher)
|
||||
if !ok {
|
||||
logger.Panicf("BUG: client must implement net/http.Flusher interface; got %T", client)
|
||||
}
|
||||
|
||||
for {
|
||||
n, backendErr := backend.Read(buf)
|
||||
if n > 0 {
|
||||
data := buf[:n]
|
||||
n, clientErr := client.Write(data)
|
||||
if clientErr != nil {
|
||||
return fmt.Errorf("cannot write data to client: %w", clientErr)
|
||||
}
|
||||
if n != len(data) {
|
||||
logger.Panicf("BUG: unexpected number of bytes written returned by client.Write; got %d; want %d", n, len(data))
|
||||
}
|
||||
// Flush the read data from the backend to the client as fast as possible
|
||||
// in order to reduce delays for data propagation.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaLogs/issues/667
|
||||
flusher.Flush()
|
||||
}
|
||||
if backendErr != nil {
|
||||
if backendErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("cannot read data from backend: %w", backendErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var copyBufPool bytesutil.ByteBufferPool
|
||||
|
||||
func copyHeader(dst, src http.Header) {
|
||||
|
||||
@@ -514,6 +514,11 @@ func (w *fakeResponseWriter) getResponse() string {
|
||||
return w.bb.String()
|
||||
}
|
||||
|
||||
// Flush implements net/http.Flusher
|
||||
func (w *fakeResponseWriter) Flush() {
|
||||
// Nothing to do.
|
||||
}
|
||||
|
||||
func (w *fakeResponseWriter) Header() http.Header {
|
||||
if w.h == nil {
|
||||
w.h = http.Header{}
|
||||
|
||||
@@ -115,7 +115,7 @@ func main() {
|
||||
if err != nil {
|
||||
logger.Fatalf("cannot create backup: %s", err)
|
||||
}
|
||||
pushmetrics.Stop()
|
||||
pushmetrics.StopAndPush()
|
||||
|
||||
startTime := time.Now()
|
||||
logger.Infof("gracefully shutting down http server for metrics at %q", listenAddrs)
|
||||
|
||||
@@ -424,6 +424,84 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
mimirPath = "mimir-path"
|
||||
mimirTenantID = "mimir-tenant-id"
|
||||
mimirConcurrency = "mimir-concurrency"
|
||||
mimirFilterTimeStart = "mimir-filter-time-start"
|
||||
mimirFilterTimeEnd = "mimir-filter-time-end"
|
||||
mimirFilterLabel = "mimir-filter-label"
|
||||
mimirFilterLabelValue = "mimir-filter-label-value"
|
||||
|
||||
mimirCredsFilePath = "mimir-creds-file-path"
|
||||
mimirConfigFilePath = "mimir-config-file-path"
|
||||
mimirConfigProfile = "mimir-config-profile"
|
||||
mimirCustomS3Endpoint = "mimir-custom-s3-endpoint"
|
||||
mimirS3ForcePathStyle = "mimir-s3-force-path-style"
|
||||
mimirS3TLSInsecureSkipVerify = "mimir-s3-tls-insecure-skip-verify"
|
||||
)
|
||||
|
||||
var (
|
||||
mimirFlags = []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: mimirPath,
|
||||
Usage: "Path to Mimir storage bucket or local folder.",
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirTenantID,
|
||||
Usage: "Tenant ID for Mimir storage",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: mimirConcurrency,
|
||||
Usage: "Number of concurrently running block readers",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirFilterTimeStart,
|
||||
Usage: "The time filter in RFC3339 format to select timeseries with timestamp equal or higher than provided value. E.g. '2020-01-01T20:07:00Z'",
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirFilterTimeEnd,
|
||||
Usage: "The time filter in RFC3339 format to select timeseries with timestamp equal or lower than provided value. E.g. '2020-01-01T20:07:00Z'",
|
||||
Required: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirFilterLabel,
|
||||
Usage: "Prometheus label name to filter timeseries by. E.g. '__name__' will filter timeseries by name.",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirFilterLabelValue,
|
||||
Usage: fmt.Sprintf("Prometheus regular expression to filter label from %q flag.", promFilterLabel),
|
||||
Value: ".*",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirCredsFilePath,
|
||||
Usage: "Path to file with GCS or S3 credentials. Credentials are loaded from default locations if not set. See https://cloud.google.com/iam/docs/creating-managing-service-account-keys and https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirConfigFilePath,
|
||||
Usage: "Path to file with S3 configs. Configs are loaded from default location if not set. See https://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirConfigProfile,
|
||||
Usage: "Profile name for S3 configs. If no set, the value of the environment variable will be loaded (AWS_PROFILE or AWS_DEFAULT_PROFILE), or if both not set, DefaultSharedConfigProfile is used",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: mimirCustomS3Endpoint,
|
||||
Usage: "Custom S3 endpoint for use with S3-compatible storages (e.g. MinIO). S3 is used if not set",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: mimirS3ForcePathStyle,
|
||||
Usage: "Prefixing endpoint with bucket name when set false, true by default.",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: mimirS3TLSInsecureSkipVerify,
|
||||
Usage: "Whether to skip TLS verification when connecting to the S3 endpoint.",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
const (
|
||||
vmNativeFilterMatch = "vm-native-filter-match"
|
||||
vmNativeFilterTimeStart = "vm-native-filter-time-start"
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/auth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/backoff"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/mimir"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/remoteread"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
|
||||
@@ -271,7 +272,54 @@ func main() {
|
||||
cc: c.Int(promConcurrency),
|
||||
isVerbose: c.Bool(globalVerbose),
|
||||
}
|
||||
return pp.run()
|
||||
return pp.run(ctx)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "mimir",
|
||||
Usage: "Migrate time series from Mimir object storage or local filesystem",
|
||||
Flags: mergeFlags(globalFlags, mimirFlags, vmFlags),
|
||||
Before: beforeFn,
|
||||
Action: func(c *cli.Context) error {
|
||||
fmt.Println("Mimir import mode")
|
||||
|
||||
vmCfg, err := initConfigVM(c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init VM configuration: %s", err)
|
||||
}
|
||||
|
||||
importer, err = vm.NewImporter(ctx, vmCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create VM importer: %s", err)
|
||||
}
|
||||
|
||||
mCfg := mimir.Config{
|
||||
Filter: mimir.Filter{
|
||||
TimeMin: c.String(mimirFilterTimeStart),
|
||||
TimeMax: c.String(mimirFilterTimeEnd),
|
||||
Label: c.String(mimirFilterLabel),
|
||||
LabelValue: c.String(mimirFilterLabelValue),
|
||||
},
|
||||
Path: c.String(mimirPath),
|
||||
TenantID: c.String(mimirTenantID),
|
||||
CredsFilePath: c.String(mimirCredsFilePath),
|
||||
ConfigFilePath: c.String(mimirConfigFilePath),
|
||||
ConfigProfile: c.String(mimirConfigProfile),
|
||||
CustomS3Endpoint: c.String(mimirCustomS3Endpoint),
|
||||
S3ForcePathStyle: c.Bool(mimirS3ForcePathStyle),
|
||||
S3TLSInsecureSkipVerify: c.Bool(mimirS3TLSInsecureSkipVerify),
|
||||
}
|
||||
cl, err := mimir.NewClient(ctx, mCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mimir client: %s", err)
|
||||
}
|
||||
pp := prometheusProcessor{
|
||||
cl: cl,
|
||||
im: importer,
|
||||
cc: c.Int(mimirConcurrency),
|
||||
isVerbose: c.Bool(globalVerbose),
|
||||
}
|
||||
return pp.run(ctx)
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
184
app/vmctl/mimir/lazyreader.go
Normal file
184
app/vmctl/mimir/lazyreader.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package mimir
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/oklog/ulid/v2"
|
||||
"github.com/prometheus/prometheus/tsdb"
|
||||
"github.com/prometheus/prometheus/tsdb/tombstones"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
|
||||
)
|
||||
|
||||
var _ tsdb.BlockReader = (*LazyBlockReader)(nil)
|
||||
|
||||
// LazyBlockReader is stores block id and segment num information.
|
||||
// It is used to lazily fetch and parse block data.
|
||||
// It implements tsdb.BlockReader interface.
|
||||
type LazyBlockReader struct {
|
||||
// Block ID.
|
||||
ID ulid.ULID
|
||||
// SegmentsNum stores the number of chunks segments in the block.
|
||||
SegmentsNum int
|
||||
|
||||
mu sync.Mutex
|
||||
reader tsdb.BlockReader
|
||||
fs common.RemoteFS
|
||||
err error
|
||||
}
|
||||
|
||||
// NewLazyBlockReader returns a new LazyBlockReader for the given block.
|
||||
func NewLazyBlockReader(block *Block, fs common.RemoteFS) (*LazyBlockReader, error) {
|
||||
if block.SegmentsFormat != "1b6d" {
|
||||
return nil, fmt.Errorf("unsupported segments format: %s", block.SegmentsFormat)
|
||||
}
|
||||
|
||||
return &LazyBlockReader{
|
||||
ID: block.ID,
|
||||
SegmentsNum: block.SegmentsNum,
|
||||
fs: fs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (lbr *LazyBlockReader) initialize() error {
|
||||
lbr.mu.Lock()
|
||||
defer lbr.mu.Unlock()
|
||||
if lbr.reader != nil {
|
||||
return nil
|
||||
}
|
||||
// fetching block and parse it and store it in lbr.reader
|
||||
temp, err := lbr.mkTempDir()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp dir: %s", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := os.RemoveAll(temp); err != nil {
|
||||
log.Printf("failed to remove temp dir: %s", err)
|
||||
}
|
||||
log.Printf("removed temp dir: %s", temp)
|
||||
}()
|
||||
|
||||
meta, err := lbr.fetchFile(metaFilename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := lbr.writeFile(temp, metaFilename, meta); err != nil {
|
||||
log.Printf("failed to write meta file: %s", err)
|
||||
return err
|
||||
}
|
||||
idx, err := lbr.fetchFile(indexFilename)
|
||||
if err != nil {
|
||||
log.Printf("failed to fetch index file %q: %s", indexFilename, err)
|
||||
return err
|
||||
}
|
||||
if err := lbr.writeFile(temp, indexFilename, idx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 1; i <= lbr.SegmentsNum; i++ {
|
||||
// segments formats has format 1b06d
|
||||
// https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L32
|
||||
chunkName := fmt.Sprintf("%06d", i)
|
||||
blockChunkPath := filepath.Join("chunks", chunkName)
|
||||
chunk, err := lbr.fetchFile(blockChunkPath)
|
||||
if err != nil {
|
||||
log.Printf("failed to fetch chunk file: %q: %s", chunkName, err)
|
||||
return err
|
||||
}
|
||||
if err := lbr.writeFile(temp, blockChunkPath, chunk); err != nil {
|
||||
log.Printf("failed to write chunk file: %q: %s", chunkName, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Set postingDecoder to nil because
|
||||
// If it is nil then a default decoder is used, compatible with Prometheus v2.
|
||||
pb, err := tsdb.OpenBlock(nil, temp, nil, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open block %q: %s", lbr.ID, err)
|
||||
}
|
||||
lbr.reader = pb
|
||||
return nil
|
||||
}
|
||||
|
||||
// Index returns an IndexReader over the block's data.
|
||||
func (lbr *LazyBlockReader) Index() (tsdb.IndexReader, error) {
|
||||
if err := lbr.initialize(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lbr.reader.Index()
|
||||
}
|
||||
|
||||
// Chunks returns a ChunkReader over the block's data.
|
||||
func (lbr *LazyBlockReader) Chunks() (tsdb.ChunkReader, error) {
|
||||
if err := lbr.initialize(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lbr.reader.Chunks()
|
||||
}
|
||||
|
||||
// Tombstones returns a tombstones.Reader over the block's deleted data.
|
||||
func (lbr *LazyBlockReader) Tombstones() (tombstones.Reader, error) {
|
||||
if err := lbr.initialize(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lbr.reader.Tombstones()
|
||||
}
|
||||
|
||||
// Meta provides meta information about the block reader.
|
||||
func (lbr *LazyBlockReader) Meta() tsdb.BlockMeta {
|
||||
if err := lbr.initialize(); err != nil {
|
||||
lbr.err = fmt.Errorf("error get Block Meta: %s; return empty block", err)
|
||||
return tsdb.BlockMeta{}
|
||||
}
|
||||
return lbr.reader.Meta()
|
||||
}
|
||||
|
||||
// Size returns the number of bytes that the block takes up on disk.
|
||||
func (lbr *LazyBlockReader) Size() int64 {
|
||||
if err := lbr.initialize(); err != nil {
|
||||
lbr.err = fmt.Errorf("error get Size of the block: %s, return zero size", err)
|
||||
return 0
|
||||
}
|
||||
return lbr.reader.Size()
|
||||
}
|
||||
|
||||
// Err returns the last error that occurred on the block reader.
|
||||
func (lbr *LazyBlockReader) Err() error {
|
||||
return lbr.err
|
||||
}
|
||||
|
||||
func (lbr *LazyBlockReader) mkTempDir() (string, error) {
|
||||
temp, err := os.MkdirTemp("", lbr.ID.String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp dir: %s", err)
|
||||
}
|
||||
err = os.Mkdir(filepath.Join(temp, "chunks"), 0755)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp dir: %s", err)
|
||||
}
|
||||
return temp, nil
|
||||
}
|
||||
|
||||
func (lbr *LazyBlockReader) fetchFile(filePath string) ([]byte, error) {
|
||||
blockID := lbr.ID.String()
|
||||
blockPath := filepath.Join(blockID, filePath)
|
||||
has, err := lbr.fs.HasFile(blockPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !has {
|
||||
return nil, fmt.Errorf("block meta %s not found", blockID)
|
||||
}
|
||||
return lbr.fs.ReadFile(blockPath)
|
||||
}
|
||||
|
||||
func (lbr *LazyBlockReader) writeFile(folder string, filename string, file []byte) error {
|
||||
fileName := filepath.Join(folder, filename)
|
||||
return os.WriteFile(fileName, file, 0644)
|
||||
}
|
||||
241
app/vmctl/mimir/mimir.go
Normal file
241
app/vmctl/mimir/mimir.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package mimir
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/oklog/ulid/v2"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb"
|
||||
|
||||
utils "github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
|
||||
)
|
||||
|
||||
const (
|
||||
bucketIndex = "bucket-index.json"
|
||||
bucketIndexCompressedFilename = bucketIndex + ".gz"
|
||||
metaFilename = "meta.json"
|
||||
indexFilename = "index"
|
||||
)
|
||||
|
||||
// BlockDeletionMark holds the information about a block's deletion mark in the index.
|
||||
// This type was copied from the mimir repository https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L234.
|
||||
type BlockDeletionMark struct {
|
||||
// Block ID.
|
||||
ID ulid.ULID `json:"block_id"`
|
||||
|
||||
// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
|
||||
DeletionTime int64 `json:"deletion_time"`
|
||||
}
|
||||
|
||||
// Block holds the information about a block in the index.
|
||||
// This is a partial implementation of the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L73
|
||||
type Block struct {
|
||||
// Block ID.
|
||||
ID ulid.ULID `json:"block_id"`
|
||||
|
||||
// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
|
||||
MinTime int64 `json:"min_time"`
|
||||
MaxTime int64 `json:"max_time"`
|
||||
|
||||
// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
|
||||
// in the block.
|
||||
SegmentsFormat string `json:"segments_format,omitempty"`
|
||||
SegmentsNum int `json:"segments_num,omitempty"`
|
||||
}
|
||||
|
||||
// Index contains all known blocks and markers of a tenant.
|
||||
// This is a partial implementation pof the https://github.com/grafana/mimir/blob/main/pkg/storage/tsdb/bucketindex/index.go#L36
|
||||
type Index struct {
|
||||
// Version of the index format.
|
||||
Version int `json:"version"`
|
||||
|
||||
// List of complete blocks (partial blocks are excluded from the index).
|
||||
Blocks []*Block `json:"blocks"`
|
||||
}
|
||||
|
||||
// Config contains a list of params needed
|
||||
// for reading Prometheus snapshots
|
||||
type Config struct {
|
||||
// Path to remote storage bucket
|
||||
Path string
|
||||
// TenantID is the tenant id for the storage
|
||||
TenantID string
|
||||
|
||||
Filter Filter
|
||||
|
||||
CredsFilePath string
|
||||
ConfigFilePath string
|
||||
ConfigProfile string
|
||||
CustomS3Endpoint string
|
||||
S3ForcePathStyle bool
|
||||
S3TLSInsecureSkipVerify bool
|
||||
}
|
||||
|
||||
// Filter contains configuration for filtering
|
||||
// the timeseries
|
||||
type Filter struct {
|
||||
TimeMin string
|
||||
TimeMax string
|
||||
Label string
|
||||
LabelValue string
|
||||
}
|
||||
|
||||
// Client is a wrapper over Prometheus tsdb.DBReader
|
||||
type Client struct {
|
||||
common.RemoteFS
|
||||
filter filter
|
||||
}
|
||||
|
||||
type filter struct {
|
||||
min, max int64
|
||||
label string
|
||||
labelValue string
|
||||
}
|
||||
|
||||
func (f filter) inRange(minTime, maxTime int64) bool {
|
||||
fmin, fmax := f.min, f.max
|
||||
if minTime == 0 {
|
||||
fmin = minTime
|
||||
}
|
||||
if fmax == 0 {
|
||||
fmax = maxTime
|
||||
}
|
||||
return minTime <= fmax && fmin <= maxTime
|
||||
}
|
||||
|
||||
// NewClient creates and validates new Client
|
||||
// with given Config
|
||||
func NewClient(ctx context.Context, cfg Config) (*Client, error) {
|
||||
if cfg.Path == "" {
|
||||
return nil, fmt.Errorf("path cannot be empty")
|
||||
}
|
||||
|
||||
if cfg.TenantID != "" {
|
||||
cfg.Path = fmt.Sprintf("%s/%s", cfg.Path, cfg.TenantID)
|
||||
}
|
||||
|
||||
var c Client
|
||||
rfs, err := NewRemoteFS(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse `-src`=%q: %w", cfg.Path, err)
|
||||
}
|
||||
|
||||
c.RemoteFS = rfs
|
||||
timeMin, err := utils.ParseTime(cfg.Filter.TimeMin)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse min time in filter: %s", err)
|
||||
}
|
||||
timeMax, err := utils.ParseTime(cfg.Filter.TimeMax)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse max time in filter: %s", err)
|
||||
}
|
||||
c.filter = filter{
|
||||
min: timeMin.UnixMilli(),
|
||||
max: timeMax.UnixMilli(),
|
||||
label: cfg.Filter.Label,
|
||||
labelValue: cfg.Filter.LabelValue,
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// Explore a fetches bucket-index.json file from a remote storage or local filesystem
|
||||
// and filter blocks via the defined time range, but does not take into account label filters.
|
||||
func (c *Client) Explore() ([]tsdb.BlockReader, error) {
|
||||
|
||||
s := &utils.Stats{
|
||||
Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
|
||||
}
|
||||
|
||||
log.Printf("Fetching blocks from remote storage")
|
||||
|
||||
indexFile, err := c.fetchIndexFile()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch index file: %s", err)
|
||||
}
|
||||
|
||||
var blocksToImport []tsdb.BlockReader
|
||||
for _, block := range indexFile.Blocks {
|
||||
if !c.filter.inRange(block.MinTime, block.MaxTime) {
|
||||
// Skipping block outside of time range
|
||||
continue
|
||||
}
|
||||
|
||||
if block.ID.String() == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
lazyBlockReader, err := NewLazyBlockReader(block, c.RemoteFS)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create lazy block reader: %s", err)
|
||||
}
|
||||
blocksToImport = append(blocksToImport, lazyBlockReader)
|
||||
}
|
||||
|
||||
s.Blocks = len(blocksToImport)
|
||||
return blocksToImport, nil
|
||||
}
|
||||
|
||||
// Read reads the given BlockReader according to configured
|
||||
// time and label filters.
|
||||
func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (storage.SeriesSet, error) {
|
||||
meta := block.Meta()
|
||||
if b, ok := block.(*LazyBlockReader); ok && b.Err() != nil {
|
||||
return nil, fmt.Errorf("failed to read block: %s", b.Err())
|
||||
}
|
||||
|
||||
if meta.ULID.String() == "" {
|
||||
log.Printf("got block without the id. it is empty")
|
||||
return nil, fmt.Errorf("block without id")
|
||||
}
|
||||
|
||||
minTime, maxTime := meta.MinTime, meta.MaxTime
|
||||
if c.filter.min != 0 {
|
||||
minTime = c.filter.min
|
||||
}
|
||||
if c.filter.max != 0 {
|
||||
maxTime = c.filter.max
|
||||
}
|
||||
q, err := tsdb.NewBlockQuerier(block, minTime, maxTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
func (c *Client) fetchIndexFile() (*Index, error) {
|
||||
has, err := c.HasFile(bucketIndexCompressedFilename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !has {
|
||||
return nil, fmt.Errorf("bucket-index.json.gz not found")
|
||||
}
|
||||
|
||||
file, err := c.ReadFile(bucketIndexCompressedFilename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read bucket index: %s", err)
|
||||
}
|
||||
|
||||
r := bytes.NewReader(file)
|
||||
// Read all the content.
|
||||
gzipReader, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create gzip reader: %s", err)
|
||||
}
|
||||
|
||||
var indexFile Index
|
||||
err = json.NewDecoder(gzipReader).Decode(&indexFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode bucket index: %s", err)
|
||||
}
|
||||
|
||||
return &indexFile, nil
|
||||
}
|
||||
91
app/vmctl/mimir/remotefs.go
Normal file
91
app/vmctl/mimir/remotefs.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package mimir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/azremote"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/common"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/fsremote"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/gcsremote"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/backup/s3remote"
|
||||
)
|
||||
|
||||
// NewRemoteFS returns new remote fs from the given Config.
|
||||
func NewRemoteFS(ctx context.Context, cfg Config) (common.RemoteFS, error) {
|
||||
if len(cfg.Path) == 0 {
|
||||
return nil, fmt.Errorf("path cannot be empty")
|
||||
}
|
||||
n := strings.Index(cfg.Path, "://")
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("missing scheme in path %q. Supported schemes: `gs://`, `s3://`, `azblob://`, `fs://`", cfg.Path)
|
||||
}
|
||||
scheme := cfg.Path[:n]
|
||||
dir := cfg.Path[n+len("://"):]
|
||||
switch scheme {
|
||||
case "fs":
|
||||
if !filepath.IsAbs(dir) {
|
||||
return nil, fmt.Errorf("dir must be absolute; got %q", dir)
|
||||
}
|
||||
fsr := &fsremote.FS{
|
||||
Dir: filepath.Clean(dir),
|
||||
}
|
||||
return fsr, nil
|
||||
case "gcs", "gs":
|
||||
n := strings.Index(dir, "/")
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("missing directory on the gcs bucket %q", dir)
|
||||
}
|
||||
bucket := dir[:n]
|
||||
dir = dir[n:]
|
||||
fsr := &gcsremote.FS{
|
||||
CredsFilePath: cfg.CredsFilePath,
|
||||
Bucket: bucket,
|
||||
Dir: dir,
|
||||
}
|
||||
if err := fsr.Init(ctx); err != nil {
|
||||
return nil, fmt.Errorf("cannot initialize connection to gcs: %w", err)
|
||||
}
|
||||
return fsr, nil
|
||||
case "azblob":
|
||||
n := strings.Index(dir, "/")
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("missing directory on the AZBlob container %q", dir)
|
||||
}
|
||||
bucket := dir[:n]
|
||||
dir = dir[n:]
|
||||
fsr := &azremote.FS{
|
||||
Container: bucket,
|
||||
Dir: dir,
|
||||
}
|
||||
if err := fsr.Init(ctx); err != nil {
|
||||
return nil, fmt.Errorf("cannot initialize connection to AZBlob: %w", err)
|
||||
}
|
||||
return fsr, nil
|
||||
case "s3":
|
||||
n := strings.Index(dir, "/")
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("missing directory on the s3 bucket %q", dir)
|
||||
}
|
||||
bucket := dir[:n]
|
||||
dir = dir[n:]
|
||||
fsr := &s3remote.FS{
|
||||
CredsFilePath: cfg.CredsFilePath,
|
||||
ConfigFilePath: cfg.ConfigFilePath,
|
||||
CustomEndpoint: cfg.CustomS3Endpoint,
|
||||
TLSInsecureSkipVerify: cfg.S3TLSInsecureSkipVerify,
|
||||
S3ForcePathStyle: cfg.S3ForcePathStyle,
|
||||
ProfileName: cfg.ConfigProfile,
|
||||
Bucket: bucket,
|
||||
Dir: dir,
|
||||
}
|
||||
if err := fsr.Init(ctx); err != nil {
|
||||
return nil, fmt.Errorf("cannot initialize connection to s3: %w", err)
|
||||
}
|
||||
return fsr, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported scheme %q", scheme)
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/barpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/prometheus"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
|
||||
)
|
||||
|
||||
// Runner is an interface for fetching and reading
|
||||
// snapshot blocks
|
||||
type Runner interface {
|
||||
Explore() ([]tsdb.BlockReader, error)
|
||||
Read(context.Context, tsdb.BlockReader) (storage.SeriesSet, error)
|
||||
}
|
||||
|
||||
type prometheusProcessor struct {
|
||||
// prometheus client fetches and reads
|
||||
// Runner fetches and reads
|
||||
// snapshot blocks
|
||||
cl *prometheus.Client
|
||||
cl Runner
|
||||
// importer performs import requests
|
||||
// for timeseries data returned from
|
||||
// snapshot blocks
|
||||
@@ -30,7 +38,7 @@ type prometheusProcessor struct {
|
||||
isVerbose bool
|
||||
}
|
||||
|
||||
func (pp *prometheusProcessor) run() error {
|
||||
func (pp *prometheusProcessor) run(ctx context.Context) error {
|
||||
blocks, err := pp.cl.Explore()
|
||||
if err != nil {
|
||||
return fmt.Errorf("explore failed: %s", err)
|
||||
@@ -43,7 +51,7 @@ func (pp *prometheusProcessor) run() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := pp.processBlocks(blocks); err != nil {
|
||||
if err := pp.processBlocks(ctx, blocks); err != nil {
|
||||
return fmt.Errorf("migration failed: %s", err)
|
||||
}
|
||||
|
||||
@@ -52,8 +60,8 @@ func (pp *prometheusProcessor) run() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
|
||||
ss, err := pp.cl.Read(b)
|
||||
func (pp *prometheusProcessor) do(ctx context.Context, b tsdb.BlockReader) error {
|
||||
ss, err := pp.cl.Read(ctx, b)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read block: %s", err)
|
||||
}
|
||||
@@ -109,7 +117,7 @@ func (pp *prometheusProcessor) do(b tsdb.BlockReader) error {
|
||||
return ss.Err()
|
||||
}
|
||||
|
||||
func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
func (pp *prometheusProcessor) processBlocks(ctx context.Context, blocks []tsdb.BlockReader) error {
|
||||
bar := barpool.AddWithTemplate(fmt.Sprintf(barTpl, "Processing blocks"), len(blocks))
|
||||
if err := barpool.Start(); err != nil {
|
||||
return err
|
||||
@@ -126,7 +134,7 @@ func (pp *prometheusProcessor) processBlocks(blocks []tsdb.BlockReader) error {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for br := range blockReadersCh {
|
||||
if err := pp.do(br); err != nil {
|
||||
if err := pp.do(ctx, br); err != nil {
|
||||
errCh <- fmt.Errorf("read failed for block %q: %s", br.Meta().ULID, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vmctlutil"
|
||||
)
|
||||
|
||||
// Config contains a list of params needed
|
||||
@@ -60,13 +62,13 @@ func NewClient(cfg Config) (*Client, error) {
|
||||
return nil, fmt.Errorf("failed to open snapshot %q: %s", cfg.Snapshot, err)
|
||||
}
|
||||
c := &Client{DBReadOnly: db}
|
||||
minTime, maxTime, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
|
||||
timeMin, timeMax, err := parseTime(cfg.Filter.TimeMin, cfg.Filter.TimeMax)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse time in filter: %s", err)
|
||||
}
|
||||
c.filter = filter{
|
||||
min: minTime,
|
||||
max: maxTime,
|
||||
min: timeMin,
|
||||
max: timeMax,
|
||||
label: cfg.Filter.Label,
|
||||
labelValue: cfg.Filter.LabelValue,
|
||||
}
|
||||
@@ -83,7 +85,7 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch blocks: %s", err)
|
||||
}
|
||||
s := &Stats{
|
||||
s := &vmctlutil.Stats{
|
||||
Filtered: c.filter.min != 0 || c.filter.max != 0 || c.filter.label != "",
|
||||
Blocks: len(blocks),
|
||||
}
|
||||
@@ -110,7 +112,7 @@ func (c *Client) Explore() ([]tsdb.BlockReader, error) {
|
||||
|
||||
// Read reads the given BlockReader according to configured
|
||||
// time and label filters.
|
||||
func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
|
||||
func (c *Client) Read(ctx context.Context, block tsdb.BlockReader) (storage.SeriesSet, error) {
|
||||
minTime, maxTime := block.Meta().MinTime, block.Meta().MaxTime
|
||||
if c.filter.min != 0 {
|
||||
minTime = c.filter.min
|
||||
@@ -122,7 +124,7 @@ func (c *Client) Read(block tsdb.BlockReader) (storage.SeriesSet, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ss := q.Select(context.Background(), false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
|
||||
ss := q.Select(ctx, false, nil, labels.MustNewMatcher(labels.MatchRegexp, c.filter.label, c.filter.labelValue))
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -207,7 +207,6 @@ func (im *Importer) Input(ts *TimeSeries) error {
|
||||
// and waits until they are finished
|
||||
func (im *Importer) Close() {
|
||||
im.once.Do(func() {
|
||||
close(im.close)
|
||||
close(im.input)
|
||||
im.wg.Wait()
|
||||
close(im.errors)
|
||||
@@ -237,7 +236,17 @@ func (im *Importer) startWorker(ctx context.Context, bar barpool.Bar, batchSize,
|
||||
return
|
||||
case ts, ok := <-im.input:
|
||||
if !ok {
|
||||
continue
|
||||
// drain all batches before exit
|
||||
exitErr := &ImportError{
|
||||
Batch: batch,
|
||||
}
|
||||
retryableFunc := func() error { return im.Import(batch) }
|
||||
_, err := im.backoff.Retry(ctx, retryableFunc)
|
||||
if err != nil {
|
||||
exitErr.Err = err
|
||||
}
|
||||
im.errors <- exitErr
|
||||
return
|
||||
}
|
||||
// init waitForBatch when first
|
||||
// value was received
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package prometheus
|
||||
package vmctlutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -68,7 +68,7 @@ func main() {
|
||||
if err := a.Run(ctx); err != nil {
|
||||
logger.Fatalf("cannot restore from backup: %s", err)
|
||||
}
|
||||
pushmetrics.Stop()
|
||||
pushmetrics.StopAndPush()
|
||||
srcFS.MustStop()
|
||||
dstFS.MustStop()
|
||||
|
||||
|
||||
@@ -197,13 +197,13 @@ func newNextSeriesForSearchQuery(ec *evalConfig, sq *storage.SearchQuery, expr g
|
||||
}
|
||||
s.summarize(aggrAvg, ec.startTime, ec.endTime, ec.storageStep, 0)
|
||||
t := timerpool.Get(30 * time.Second)
|
||||
defer timerpool.Put(t)
|
||||
select {
|
||||
case seriesCh <- s:
|
||||
case <-t.C:
|
||||
logger.Errorf("resource leak when processing the %s (full query: %s); please report this error to VictoriaMetrics developers",
|
||||
expr.AppendString(nil), ec.originalQuery)
|
||||
}
|
||||
timerpool.Put(t)
|
||||
return nil
|
||||
})
|
||||
close(seriesCh)
|
||||
|
||||
@@ -1150,15 +1150,23 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
|
||||
}
|
||||
qt.Printf("optimized calculation for instant rollup avg_over_time(m[d]) as (sum_over_time(m[d]) / count_over_time(m[d]))")
|
||||
fe := expr.(*metricsql.FuncExpr)
|
||||
feSum := *fe
|
||||
feSum.Name = "sum_over_time"
|
||||
feCount := *fe
|
||||
feCount.Name = "count_over_time"
|
||||
// copy RollupExpr to drop possible offset,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
|
||||
newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
|
||||
newArg.Offset = nil
|
||||
be := &metricsql.BinaryOpExpr{
|
||||
Op: "/",
|
||||
KeepMetricNames: fe.KeepMetricNames,
|
||||
Left: &feSum,
|
||||
Right: &feCount,
|
||||
Left: &metricsql.FuncExpr{
|
||||
Name: "sum_over_time",
|
||||
Args: []metricsql.Expr{newArg},
|
||||
KeepMetricNames: fe.KeepMetricNames,
|
||||
},
|
||||
Right: &metricsql.FuncExpr{
|
||||
Name: "count_over_time",
|
||||
Args: []metricsql.Expr{newArg},
|
||||
KeepMetricNames: fe.KeepMetricNames,
|
||||
},
|
||||
}
|
||||
return evalExpr(qt, ec, be)
|
||||
case "rate":
|
||||
@@ -1172,8 +1180,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
|
||||
fe := afe.Args[0].(*metricsql.FuncExpr)
|
||||
feIncrease := *fe
|
||||
feIncrease.Name = "increase"
|
||||
re := fe.Args[0].(*metricsql.RollupExpr)
|
||||
d := re.Window.Duration(ec.Step)
|
||||
// copy RollupExpr to drop possible offset,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
|
||||
newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
|
||||
newArg.Offset = nil
|
||||
feIncrease.Args = []metricsql.Expr{newArg}
|
||||
d := newArg.Window.Duration(ec.Step)
|
||||
if d == 0 {
|
||||
d = ec.Step
|
||||
}
|
||||
@@ -1193,8 +1205,12 @@ func evalInstantRollup(qt *querytracer.Tracer, ec *EvalConfig, funcName string,
|
||||
fe := expr.(*metricsql.FuncExpr)
|
||||
feIncrease := *fe
|
||||
feIncrease.Name = "increase"
|
||||
re := fe.Args[0].(*metricsql.RollupExpr)
|
||||
d := re.Window.Duration(ec.Step)
|
||||
// copy RollupExpr to drop possible offset,
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
|
||||
newArg := copyRollupExpr(fe.Args[0].(*metricsql.RollupExpr))
|
||||
newArg.Offset = nil
|
||||
feIncrease.Args = []metricsql.Expr{newArg}
|
||||
d := newArg.Window.Duration(ec.Step)
|
||||
if d == 0 {
|
||||
d = ec.Step
|
||||
}
|
||||
@@ -1999,3 +2015,23 @@ func dropStaleNaNs(funcName string, values []float64, timestamps []int64) ([]flo
|
||||
}
|
||||
return dstValues, dstTimestamps
|
||||
}
|
||||
|
||||
func copyRollupExpr(re *metricsql.RollupExpr) *metricsql.RollupExpr {
|
||||
var newRe metricsql.RollupExpr
|
||||
newRe.Expr = re.Expr
|
||||
newRe.InheritStep = re.InheritStep
|
||||
newRe.At = re.At
|
||||
if re.Window != nil {
|
||||
newRe.Window = &metricsql.DurationExpr{}
|
||||
*newRe.Window = *re.Window
|
||||
}
|
||||
if re.Offset != nil {
|
||||
newRe.Offset = &metricsql.DurationExpr{}
|
||||
*newRe.Offset = *re.Offset
|
||||
}
|
||||
if re.Step != nil {
|
||||
newRe.Step = &metricsql.DurationExpr{}
|
||||
*newRe.Step = *re.Step
|
||||
}
|
||||
return &newRe
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
209
app/vmselect/vmui/assets/index-DQcPcJrn.js
Normal file
209
app/vmselect/vmui/assets/index-DQcPcJrn.js
Normal file
File diff suppressed because one or more lines are too long
1
app/vmselect/vmui/assets/index-dWApsAEM.css
Normal file
1
app/vmselect/vmui/assets/index-dWApsAEM.css
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -36,10 +36,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-DK22yiEQ.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-DBOs1yKE.js">
|
||||
<script type="module" crossorigin src="./assets/index-DQcPcJrn.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-DY9kCvzk.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-Ccv_zSYG.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-dWApsAEM.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25.0 AS build-web-stage
|
||||
FROM golang:1.25.1 AS build-web-stage
|
||||
COPY build /build
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
<link rel="apple-touch-icon" href="/favicon.svg"/>
|
||||
<link rel="mask-icon" href="/favicon.svg" color="#000000">
|
||||
|
||||
<meta name="robots" content="noindex">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=5"/>
|
||||
<meta name="theme-color" content="#000000"/>
|
||||
<meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/>
|
||||
|
||||
56
app/vmui/packages/vmui/package-lock.json
generated
56
app/vmui/packages/vmui/package-lock.json
generated
@@ -17,7 +17,7 @@
|
||||
"react-input-mask": "^2.0.4",
|
||||
"react-router-dom": "^7.6.3",
|
||||
"uplot": "^1.6.32",
|
||||
"vite": "^7.0.4",
|
||||
"vite": "^7.1.5",
|
||||
"web-vitals": "^5.0.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -7321,13 +7321,13 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tinyglobby": {
|
||||
"version": "0.2.14",
|
||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.14.tgz",
|
||||
"integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==",
|
||||
"version": "0.2.15",
|
||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",
|
||||
"integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"fdir": "^6.4.4",
|
||||
"picomatch": "^4.0.2"
|
||||
"fdir": "^6.5.0",
|
||||
"picomatch": "^4.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
@@ -7337,10 +7337,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tinyglobby/node_modules/fdir": {
|
||||
"version": "6.4.6",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz",
|
||||
"integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==",
|
||||
"version": "6.5.0",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
|
||||
"integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"picomatch": "^3 || ^4"
|
||||
},
|
||||
@@ -7351,9 +7354,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tinyglobby/node_modules/picomatch": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
|
||||
"integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
|
||||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
@@ -7657,17 +7660,17 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/vite": {
|
||||
"version": "7.0.4",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-7.0.4.tgz",
|
||||
"integrity": "sha512-SkaSguuS7nnmV7mfJ8l81JGBFV7Gvzp8IzgE8A8t23+AxuNX61Q5H1Tpz5efduSN7NHC8nQXD3sKQKZAu5mNEA==",
|
||||
"version": "7.1.5",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-7.1.5.tgz",
|
||||
"integrity": "sha512-4cKBO9wR75r0BeIWWWId9XK9Lj6La5X846Zw9dFfzMRw38IlTk2iCcUt6hsyiDRcPidc55ZParFYDXi0nXOeLQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"esbuild": "^0.25.0",
|
||||
"fdir": "^6.4.6",
|
||||
"picomatch": "^4.0.2",
|
||||
"fdir": "^6.5.0",
|
||||
"picomatch": "^4.0.3",
|
||||
"postcss": "^8.5.6",
|
||||
"rollup": "^4.40.0",
|
||||
"tinyglobby": "^0.2.14"
|
||||
"rollup": "^4.43.0",
|
||||
"tinyglobby": "^0.2.15"
|
||||
},
|
||||
"bin": {
|
||||
"vite": "bin/vite.js"
|
||||
@@ -7772,10 +7775,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/vite/node_modules/fdir": {
|
||||
"version": "6.4.6",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz",
|
||||
"integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==",
|
||||
"version": "6.5.0",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
|
||||
"integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"picomatch": "^3 || ^4"
|
||||
},
|
||||
@@ -7786,9 +7792,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/vite/node_modules/picomatch": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
|
||||
"integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
|
||||
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"react-input-mask": "^2.0.4",
|
||||
"react-router-dom": "^7.6.3",
|
||||
"uplot": "^1.6.32",
|
||||
"vite": "^7.0.4",
|
||||
"vite": "^7.1.5",
|
||||
"web-vitals": "^5.0.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -13,6 +13,8 @@ import Button from "../../Button/Button";
|
||||
interface DatePickerProps {
|
||||
date: Date | Dayjs
|
||||
format?: string
|
||||
minDate?: Date | Dayjs
|
||||
maxDate?: Date | Dayjs
|
||||
onChange: (date: string) => void
|
||||
}
|
||||
|
||||
@@ -24,6 +26,8 @@ enum CalendarTypeView {
|
||||
|
||||
const Calendar: FC<DatePickerProps> = ({
|
||||
date,
|
||||
minDate,
|
||||
maxDate,
|
||||
format = DATE_TIME_FORMAT,
|
||||
onChange,
|
||||
}) => {
|
||||
@@ -34,6 +38,8 @@ const Calendar: FC<DatePickerProps> = ({
|
||||
const today = dayjs.tz();
|
||||
const viewDateIsToday = today.format(DATE_FORMAT) === viewDate.format(DATE_FORMAT);
|
||||
const { isMobile } = useDeviceDetect();
|
||||
const min = minDate ? dayjs(minDate) : undefined;
|
||||
const max = maxDate ? dayjs(maxDate) : undefined;
|
||||
|
||||
const toggleDisplayYears = () => {
|
||||
setViewType(prev => prev === CalendarTypeView.years ? CalendarTypeView.days : CalendarTypeView.years);
|
||||
@@ -75,9 +81,13 @@ const Calendar: FC<DatePickerProps> = ({
|
||||
onChangeViewDate={handleChangeViewDate}
|
||||
toggleDisplayYears={toggleDisplayYears}
|
||||
showArrowNav={viewType === CalendarTypeView.days}
|
||||
hasPrev={viewType === CalendarTypeView.days && (!min || viewDate.startOf("month").isAfter(min))}
|
||||
hasNext={viewType === CalendarTypeView.days && (!max || viewDate.endOf("month").isBefore(max))}
|
||||
/>
|
||||
{viewType === CalendarTypeView.days && (
|
||||
<CalendarBody
|
||||
minDate={min}
|
||||
maxDate={max}
|
||||
viewDate={viewDate}
|
||||
selectDate={selectDate}
|
||||
onChangeSelectDate={handleChangeSelectDate}
|
||||
@@ -85,12 +95,16 @@ const Calendar: FC<DatePickerProps> = ({
|
||||
)}
|
||||
{viewType === CalendarTypeView.years && (
|
||||
<YearsList
|
||||
minDate={min}
|
||||
maxDate={max}
|
||||
viewDate={viewDate}
|
||||
onChangeViewDate={handleChangeViewDate}
|
||||
/>
|
||||
)}
|
||||
{viewType === CalendarTypeView.months && (
|
||||
<MonthsList
|
||||
minDate={min}
|
||||
maxDate={max}
|
||||
selectDate={selectDate}
|
||||
viewDate={viewDate}
|
||||
onChangeViewDate={handleChangeViewDate}
|
||||
|
||||
@@ -4,6 +4,8 @@ import classNames from "classnames";
|
||||
import Tooltip from "../../../Tooltip/Tooltip";
|
||||
|
||||
interface CalendarBodyProps {
|
||||
minDate?: Dayjs
|
||||
maxDate?: Dayjs
|
||||
viewDate: Dayjs
|
||||
selectDate: Dayjs
|
||||
onChangeSelectDate: (date: Dayjs) => void
|
||||
@@ -11,7 +13,7 @@ interface CalendarBodyProps {
|
||||
|
||||
const weekday = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
|
||||
|
||||
const CalendarBody: FC<CalendarBodyProps> = ({ viewDate: date, selectDate, onChangeSelectDate }) => {
|
||||
const CalendarBody: FC<CalendarBodyProps> = ({ minDate, maxDate, viewDate: date, selectDate, onChangeSelectDate }) => {
|
||||
const format = "YYYY-MM-DD";
|
||||
const today = dayjs.tz();
|
||||
const viewDate = dayjs(date.format(format));
|
||||
@@ -44,21 +46,25 @@ const CalendarBody: FC<CalendarBodyProps> = ({ viewDate: date, selectDate, onCha
|
||||
</Tooltip>
|
||||
))}
|
||||
|
||||
{days.map((d, i) => (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-calendar-body-cell": true,
|
||||
"vm-calendar-body-cell_day": true,
|
||||
"vm-calendar-body-cell_day_empty": !d,
|
||||
"vm-calendar-body-cell_day_active": (d && d.format(format)) === selectDate.format(format),
|
||||
"vm-calendar-body-cell_day_today": (d && d.format(format)) === today.format(format)
|
||||
})}
|
||||
key={d ? d.format(format) : i}
|
||||
onClick={createHandlerSelectDate(d)}
|
||||
>
|
||||
{d && d.format("D")}
|
||||
</div>
|
||||
))}
|
||||
{days.map((d, i) => {
|
||||
const isDisabled = d && ((minDate && d.isBefore(minDate)) || (maxDate && d.isAfter(maxDate)));
|
||||
return (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-calendar-body-cell": true,
|
||||
"vm-calendar-body-cell_day": true,
|
||||
"vm-calendar-body-cell_day_empty": !d,
|
||||
"vm-calendar-body-cell_day_active": (d && d.format(format)) === selectDate.format(format),
|
||||
"vm-calendar-body-cell_day_today": (d && d.format(format)) === today.format(format),
|
||||
"vm-calendar-body-cell_day_disabled": isDisabled,
|
||||
})}
|
||||
key={d ? d.format(format) : i}
|
||||
onClick={createHandlerSelectDate(d)}
|
||||
>
|
||||
{d && d.format("D")}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import { FC } from "preact/compat";
|
||||
import { Dayjs } from "dayjs";
|
||||
import { ArrowDownIcon, ArrowDropDownIcon } from "../../../Icons";
|
||||
import classNames from "classnames";
|
||||
|
||||
interface CalendarHeaderProps {
|
||||
viewDate: Dayjs
|
||||
onChangeViewDate: (date: Dayjs) => void
|
||||
showArrowNav: boolean
|
||||
toggleDisplayYears: () => void
|
||||
hasNext: boolean
|
||||
hasPrev: boolean
|
||||
}
|
||||
|
||||
const CalendarHeader: FC<CalendarHeaderProps> = ({ viewDate, showArrowNav, onChangeViewDate, toggleDisplayYears }) => {
|
||||
const CalendarHeader: FC<CalendarHeaderProps> = ({ hasPrev, hasNext, viewDate, showArrowNav, onChangeViewDate, toggleDisplayYears }) => {
|
||||
|
||||
const setPrevMonth = () => {
|
||||
onChangeViewDate(viewDate.subtract(1, "month"));
|
||||
@@ -35,14 +38,20 @@ const CalendarHeader: FC<CalendarHeaderProps> = ({ viewDate, showArrowNav, onCha
|
||||
{showArrowNav && (
|
||||
<div className="vm-calendar-header-right">
|
||||
<div
|
||||
className="vm-calendar-header-right__prev"
|
||||
onClick={setPrevMonth}
|
||||
className={classNames({
|
||||
"vm-calendar-header-right__prev": true,
|
||||
"vm-calendar-header-right_disabled": !hasPrev,
|
||||
})}
|
||||
onClick={hasPrev ? setPrevMonth : undefined}
|
||||
>
|
||||
<ArrowDownIcon/>
|
||||
</div>
|
||||
<div
|
||||
className="vm-calendar-header-right__next"
|
||||
onClick={setNextMonth}
|
||||
className={classNames({
|
||||
"vm-calendar-header-right__next": true,
|
||||
"vm-calendar-header-right_disabled": !hasNext,
|
||||
})}
|
||||
onClick={hasNext ? setNextMonth : undefined}
|
||||
>
|
||||
<ArrowDownIcon/>
|
||||
</div>
|
||||
|
||||
@@ -3,13 +3,14 @@ import dayjs, { Dayjs } from "dayjs";
|
||||
import classNames from "classnames";
|
||||
|
||||
interface CalendarMonthsProps {
|
||||
minDate?: Dayjs
|
||||
maxDate?: Dayjs
|
||||
viewDate: Dayjs,
|
||||
selectDate: Dayjs
|
||||
|
||||
onChangeViewDate: (date: Dayjs) => void
|
||||
}
|
||||
|
||||
const MonthsList: FC<CalendarMonthsProps> = ({ viewDate, selectDate, onChangeViewDate }) => {
|
||||
const MonthsList: FC<CalendarMonthsProps> = ({ minDate, maxDate, viewDate, selectDate, onChangeViewDate }) => {
|
||||
|
||||
const today = dayjs().format("MM");
|
||||
const currentMonths = useMemo(() => selectDate.format("MM"), [selectDate]);
|
||||
@@ -29,20 +30,24 @@ const MonthsList: FC<CalendarMonthsProps> = ({ viewDate, selectDate, onChangeVie
|
||||
|
||||
return (
|
||||
<div className="vm-calendar-years">
|
||||
{months.map(m => (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-calendar-years__year": true,
|
||||
"vm-calendar-years__year_selected": m.format("MM") === currentMonths,
|
||||
"vm-calendar-years__year_today": m.format("MM") === today
|
||||
})}
|
||||
id={`vm-calendar-year-${m.format("MM")}`}
|
||||
key={m.format("MM")}
|
||||
onClick={createHandlerClick(m)}
|
||||
>
|
||||
{m.format("MMMM")}
|
||||
</div>
|
||||
))}
|
||||
{months.map(m => {
|
||||
const isDisabled = m && ((minDate && m.isBefore(minDate)) || (maxDate && m.isAfter(maxDate)));
|
||||
return (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-calendar-years__year": true,
|
||||
"vm-calendar-years__year_selected": m.format("MM") === currentMonths,
|
||||
"vm-calendar-years__year_today": m.format("MM") === today,
|
||||
"vm-calendar-years__year_disabled": isDisabled,
|
||||
})}
|
||||
id={`vm-calendar-year-${m.format("MM")}`}
|
||||
key={m.format("MM")}
|
||||
onClick={isDisabled ? undefined : createHandlerClick(m)}
|
||||
>
|
||||
{m.format("MMMM")}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -3,11 +3,13 @@ import dayjs, { Dayjs } from "dayjs";
|
||||
import classNames from "classnames";
|
||||
|
||||
interface CalendarYearsProps {
|
||||
minDate?: Dayjs
|
||||
maxDate?: Dayjs
|
||||
viewDate: Dayjs
|
||||
onChangeViewDate: (date: Dayjs) => void
|
||||
}
|
||||
|
||||
const YearsList: FC<CalendarYearsProps> = ({ viewDate, onChangeViewDate }) => {
|
||||
const YearsList: FC<CalendarYearsProps> = ({ minDate, maxDate, viewDate, onChangeViewDate }) => {
|
||||
|
||||
const today = dayjs().format("YYYY");
|
||||
const currentYear = useMemo(() => viewDate.format("YYYY"), [viewDate]);
|
||||
@@ -30,20 +32,24 @@ const YearsList: FC<CalendarYearsProps> = ({ viewDate, onChangeViewDate }) => {
|
||||
|
||||
return (
|
||||
<div className="vm-calendar-years">
|
||||
{years.map(y => (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-calendar-years__year": true,
|
||||
"vm-calendar-years__year_selected": y.format("YYYY") === currentYear,
|
||||
"vm-calendar-years__year_today": y.format("YYYY") === today
|
||||
})}
|
||||
id={`vm-calendar-year-${y.format("YYYY")}`}
|
||||
key={y.format("YYYY")}
|
||||
onClick={createHandlerClick(y)}
|
||||
>
|
||||
{y.format("YYYY")}
|
||||
</div>
|
||||
))}
|
||||
{years.map(y => {
|
||||
const isDisabled = y && (minDate && y.isBefore(minDate)) || (maxDate && y.isAfter(maxDate));
|
||||
return (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-calendar-years__year": true,
|
||||
"vm-calendar-years__year_selected": y.format("YYYY") === currentYear,
|
||||
"vm-calendar-years__year_today": y.format("YYYY") === today,
|
||||
"vm-calendar-years__year_disabled": isDisabled,
|
||||
})}
|
||||
id={`vm-calendar-year-${y.format("YYYY")}`}
|
||||
key={y.format("YYYY")}
|
||||
onClick={isDisabled ? undefined : createHandlerClick(y)}
|
||||
>
|
||||
{y.format("YYYY")}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -69,6 +69,10 @@
|
||||
}
|
||||
}
|
||||
|
||||
&_disabled {
|
||||
color: $color-text-disabled;
|
||||
}
|
||||
|
||||
&__prev {
|
||||
transform: rotate(90deg);
|
||||
}
|
||||
@@ -108,7 +112,12 @@
|
||||
cursor: pointer;
|
||||
transition: color 200ms ease, background-color 300ms ease-in-out;
|
||||
|
||||
&:hover {
|
||||
&_disabled {
|
||||
cursor: unset;
|
||||
color: $color-text-disabled;
|
||||
}
|
||||
|
||||
&:not(&_disabled):hover {
|
||||
background-color: $color-hover-black;
|
||||
}
|
||||
|
||||
@@ -148,7 +157,12 @@
|
||||
cursor: pointer;
|
||||
transition: color 200ms ease, background-color 300ms ease-in-out;
|
||||
|
||||
&:hover {
|
||||
&_disabled {
|
||||
cursor: unset;
|
||||
color: $color-text-disabled;
|
||||
}
|
||||
|
||||
&:not(&_disabled):hover {
|
||||
background-color: $color-hover-black;
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,10 @@ import useEventListener from "../../../hooks/useEventListener";
|
||||
interface DatePickerProps {
|
||||
date: string | Date | Dayjs,
|
||||
targetRef: React.RefObject<HTMLElement>;
|
||||
format?: string
|
||||
label?: string
|
||||
format?: string;
|
||||
label?: string;
|
||||
minDate?: Date | Dayjs;
|
||||
maxDate?: Date | Dayjs;
|
||||
onChange: (val: string) => void
|
||||
}
|
||||
|
||||
@@ -20,7 +22,9 @@ const DatePicker = forwardRef<HTMLDivElement, DatePickerProps>(({
|
||||
targetRef,
|
||||
format = DATE_TIME_FORMAT,
|
||||
onChange,
|
||||
label
|
||||
label,
|
||||
minDate,
|
||||
maxDate
|
||||
}, ref) => {
|
||||
const dateDayjs = useMemo(() => dayjs(date).isValid() ? dayjs.tz(date) : dayjs().tz(), [date]);
|
||||
const { isMobile } = useDeviceDetect();
|
||||
@@ -56,6 +60,8 @@ const DatePicker = forwardRef<HTMLDivElement, DatePickerProps>(({
|
||||
date={dateDayjs}
|
||||
format={format}
|
||||
onChange={handleChangeDate}
|
||||
minDate={minDate}
|
||||
maxDate={maxDate}
|
||||
/>
|
||||
</div>
|
||||
</Popper>
|
||||
|
||||
@@ -3,39 +3,52 @@ import { ChangeEvent, KeyboardEvent } from "react";
|
||||
import { CalendarIcon } from "../../Icons";
|
||||
import DatePicker from "../DatePicker";
|
||||
import Button from "../../Button/Button";
|
||||
import { DATE_TIME_FORMAT } from "../../../../constants/date";
|
||||
import { DATE_ISO_FORMAT, DATE_FORMAT, DATE_TIME_FORMAT } from "../../../../constants/date";
|
||||
import InputMask from "react-input-mask";
|
||||
import dayjs from "dayjs";
|
||||
import dayjs, { Dayjs } from "dayjs";
|
||||
import classNames from "classnames";
|
||||
import "./style.scss";
|
||||
|
||||
const formatStringDate = (val: string) => {
|
||||
return dayjs(val).isValid() ? dayjs.tz(val).format(DATE_TIME_FORMAT) : val;
|
||||
const formatStringDate = (val: string, format: string) => {
|
||||
return dayjs(val).isValid() ? dayjs.tz(val).format(format) : val;
|
||||
};
|
||||
|
||||
interface DateTimeInputProps {
|
||||
value?: string;
|
||||
label: string;
|
||||
pickerLabel: string;
|
||||
dateOnly?: boolean;
|
||||
format?: string;
|
||||
pickerRef: React.RefObject<HTMLDivElement>;
|
||||
onChange: (date: string) => void;
|
||||
onEnter: () => void;
|
||||
disabled?: boolean;
|
||||
minDate?: Date | Dayjs;
|
||||
maxDate?: Date | Dayjs;
|
||||
}
|
||||
|
||||
const masks: Record<string, string> = {
|
||||
[DATE_ISO_FORMAT]: "9999-99-99T99:99:99",
|
||||
[DATE_FORMAT]: "9999-99-99",
|
||||
[DATE_TIME_FORMAT]: "9999-99-99 99:99:99"
|
||||
};
|
||||
|
||||
const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
value = "",
|
||||
dateOnly = false,
|
||||
format = DATE_TIME_FORMAT,
|
||||
minDate,
|
||||
maxDate,
|
||||
label,
|
||||
pickerLabel,
|
||||
pickerRef,
|
||||
onChange,
|
||||
onEnter
|
||||
onEnter,
|
||||
disabled
|
||||
}) => {
|
||||
const wrapperRef = useRef<HTMLDivElement>(null);
|
||||
const [inputRef, setInputRef] = useState<HTMLInputElement | null>(null);
|
||||
const mask = masks[format];
|
||||
|
||||
const [maskedValue, setMaskedValue] = useState(formatStringDate(value));
|
||||
const [maskedValue, setMaskedValue] = useState(formatStringDate(value, format));
|
||||
const [focusToTime, setFocusToTime] = useState(false);
|
||||
const [awaitChangeForEnter, setAwaitChangeForEnter] = useState(false);
|
||||
const error = dayjs(maskedValue).isValid() ? "" : "Invalid date format";
|
||||
@@ -55,16 +68,13 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
}
|
||||
};
|
||||
|
||||
const mask = dateOnly ? "9999-99-99" : "9999-99-99 99:99:99";
|
||||
const placeholder = dateOnly ? "YYYY-MM-DD" : "YYYY-MM-DD HH:mm:ss";
|
||||
|
||||
const handleChangeDate = (val: string) => {
|
||||
setMaskedValue(val);
|
||||
setFocusToTime(true);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const newValue = formatStringDate(value);
|
||||
const newValue = formatStringDate(value, format);
|
||||
if (newValue !== maskedValue) {
|
||||
setMaskedValue(newValue);
|
||||
}
|
||||
@@ -87,7 +97,8 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-date-time-input": true,
|
||||
"vm-date-time-input_error": error
|
||||
"vm-date-time-input_error": error,
|
||||
"vm-date-time-input_disabled": disabled,
|
||||
})}
|
||||
>
|
||||
<label>{label}</label>
|
||||
@@ -95,7 +106,7 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
tabIndex={1}
|
||||
inputRef={setInputRef}
|
||||
mask={mask}
|
||||
placeholder={placeholder}
|
||||
placeholder={format}
|
||||
value={maskedValue}
|
||||
autoCapitalize={"none"}
|
||||
inputMode={"numeric"}
|
||||
@@ -103,6 +114,7 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
onChange={handleMaskedChange}
|
||||
onBlur={handleBlur}
|
||||
onKeyUp={handleKeyUp}
|
||||
disabled={disabled}
|
||||
/>
|
||||
{error && (
|
||||
<span className="vm-date-time-input__error-text">{error}</span>
|
||||
@@ -117,6 +129,7 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
size="small"
|
||||
startIcon={<CalendarIcon/>}
|
||||
ariaLabel="calendar"
|
||||
disabled={disabled}
|
||||
/>
|
||||
</div>
|
||||
<DatePicker
|
||||
@@ -125,6 +138,9 @@ const DateTimeInput: FC<DateTimeInputProps> = ({
|
||||
date={maskedValue}
|
||||
onChange={handleChangeDate}
|
||||
targetRef={wrapperRef}
|
||||
minDate={minDate}
|
||||
maxDate={maxDate}
|
||||
format={format}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -23,6 +23,14 @@
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
&_disabled {
|
||||
cursor: default;
|
||||
pointer-events: none;
|
||||
* {
|
||||
color: $color-text-disabled !important;
|
||||
}
|
||||
}
|
||||
|
||||
&__icon {
|
||||
position: absolute;
|
||||
bottom: 2px;
|
||||
|
||||
@@ -46,11 +46,12 @@ const Select: FC<SelectProps> = ({
|
||||
const autocompleteAnchorEl = useRef<HTMLDivElement>(null);
|
||||
const [wrapperRef, setWrapperRef] = useState<React.RefObject<HTMLElement> | null>(null);
|
||||
const [openList, setOpenList] = useState(false);
|
||||
const resultList = [...list];
|
||||
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const isMultiple = Array.isArray(value);
|
||||
const selectedValues = Array.isArray(value) ? value.slice() : [];
|
||||
let selectedValues = Array.isArray(value) ? value.slice() : [];
|
||||
const hideInput = isMobile && isMultiple && !!selectedValues?.length;
|
||||
|
||||
const textFieldValue = useMemo(() => {
|
||||
@@ -77,7 +78,7 @@ const Select: FC<SelectProps> = ({
|
||||
};
|
||||
|
||||
const handleBlur = () => {
|
||||
list.includes(search) && onChange(search);
|
||||
resultList.includes(search) && onChange(search);
|
||||
};
|
||||
|
||||
const handleToggleList = (e: MouseEvent<HTMLDivElement>) => {
|
||||
@@ -123,8 +124,10 @@ const Select: FC<SelectProps> = ({
|
||||
useEventListener("keyup", handleKeyUp);
|
||||
useClickOutside(autocompleteAnchorEl, handleCloseList, wrapperRef);
|
||||
|
||||
includeAll && !list.includes("All") && list.push("All");
|
||||
includeAll && !selectedValues?.length && selectedValues.push("All");
|
||||
if (includeAll && !resultList.includes("All")) resultList.push("All");
|
||||
if (includeAll && (!selectedValues?.length || selectedValues?.length === resultList?.length)) {
|
||||
selectedValues = ["All"];
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
@@ -155,6 +158,7 @@ const Select: FC<SelectProps> = ({
|
||||
onInput={handleChange}
|
||||
onFocus={handleFocus}
|
||||
onBlur={handleBlur}
|
||||
disabled={disabled}
|
||||
ref={inputRef}
|
||||
readOnly={isMobile || !searchable}
|
||||
/>
|
||||
@@ -182,7 +186,7 @@ const Select: FC<SelectProps> = ({
|
||||
itemClassName={itemClassName}
|
||||
label={label}
|
||||
value={autocompleteValue}
|
||||
options={list.map(l => ({ value: l }))}
|
||||
options={resultList.map(l => ({ value: l }))}
|
||||
anchor={autocompleteAnchorEl}
|
||||
selected={selectedValues}
|
||||
minLength={1}
|
||||
|
||||
@@ -128,8 +128,14 @@
|
||||
}
|
||||
|
||||
&_disabled {
|
||||
pointer-events: none;
|
||||
* {
|
||||
cursor: not-allowed;
|
||||
color: var(--color-text-disabled);
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
input::placeholder {
|
||||
color: var(--color-text-disabled);
|
||||
}
|
||||
|
||||
.vm-select-input {
|
||||
|
||||
@@ -72,7 +72,6 @@ const TextField: FC<TextFieldProps> = ({
|
||||
"vm-text-field__input_error": error,
|
||||
"vm-text-field__input_warning": !error && warning,
|
||||
"vm-text-field__input_icon-start": startIcon,
|
||||
"vm-text-field__input_disabled": disabled,
|
||||
"vm-text-field__input_textarea": type === "textarea",
|
||||
});
|
||||
|
||||
@@ -136,7 +135,8 @@ const TextField: FC<TextFieldProps> = ({
|
||||
className={classNames({
|
||||
"vm-text-field": true,
|
||||
"vm-text-field_textarea": type === "textarea",
|
||||
"vm-text-field_dark": isDarkTheme
|
||||
"vm-text-field_dark": isDarkTheme,
|
||||
"vm-text-field_disabled": disabled
|
||||
})}
|
||||
data-replicated-value={value}
|
||||
>
|
||||
|
||||
@@ -6,6 +6,15 @@
|
||||
margin: 6px 0;
|
||||
width: 100%;
|
||||
|
||||
&_disabled {
|
||||
color: $color-text-disabled;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
&:is(&_disabled) > &__label {
|
||||
color: $color-text-disabled;
|
||||
}
|
||||
|
||||
&_textarea:after {
|
||||
content: attr(data-replicated-value) " ";
|
||||
white-space: pre-wrap;
|
||||
|
||||
@@ -155,15 +155,18 @@ const ExploreRules: FC = () => {
|
||||
[groups, types, states, searchInput]
|
||||
);
|
||||
|
||||
const selectedTypes = allTypes.size === types.length ? [] : types;
|
||||
const selectedStates = allStates.size === states.length ? [] : states;
|
||||
|
||||
return (
|
||||
<>
|
||||
{modalOpen && getModal()}
|
||||
{(!modalOpen || !!allStates?.size) && (
|
||||
<div className="vm-explore-alerts">
|
||||
<RulesHeader
|
||||
types={types}
|
||||
types={selectedTypes}
|
||||
allTypes={Array.from(allTypes)}
|
||||
states={states}
|
||||
states={selectedStates}
|
||||
allStates={Array.from(allStates)}
|
||||
onChangeTypes={handleChangeTypes}
|
||||
onChangeStates={handleChangeStates}
|
||||
|
||||
26
app/vmui/packages/vmui/src/utils/default-server-url.test.ts
Normal file
26
app/vmui/packages/vmui/src/utils/default-server-url.test.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { getDefaultURL } from "./default-server-url";
|
||||
|
||||
describe("test server urls", () => {
|
||||
describe("getDefaultURL()", () => {
|
||||
it("/select/0/vmui/", () => {
|
||||
const result = getDefaultURL("https://localhost:1111/select/0/vmui/");
|
||||
expect(result).toBe("https://localhost:1111/select/0/prometheus");
|
||||
});
|
||||
|
||||
it("/any/path/prefix/select/multitenant/vmui/#/rules?q=test", () => {
|
||||
const result = getDefaultURL("http://test/any/path/prefix/select/multitenant/vmui/#/rules?q=test");
|
||||
expect(result).toBe("http://test/any/path/prefix/select/multitenant/prometheus");
|
||||
});
|
||||
|
||||
it("/test/select/1:1/prometheus/graph/", () => {
|
||||
const result = getDefaultURL("https://domain.com/test/select/1:1/prometheus/graph/");
|
||||
expect(result).toBe("https://domain.com/test/select/1:1/prometheus");
|
||||
});
|
||||
|
||||
it("https://play.vm.com/#/rules?q=test", () => {
|
||||
const result = getDefaultURL("https://play.vm.com/#/rules?q=test");
|
||||
expect(result).toBe("https://play.vm.com");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -3,12 +3,15 @@ import { replaceTenantId } from "./tenants";
|
||||
import { APP_TYPE, AppType } from "../constants/appType";
|
||||
import { getFromStorage } from "./storage";
|
||||
|
||||
export const getDefaultURL = (u: string) => {
|
||||
return u.replace(/(\/(?:prometheus\/)?(?:graph|vmui)\/.*|\/#\/.*)/, "").replace(/(\/select\/[^/]+)$/, "$1/prometheus");
|
||||
};
|
||||
|
||||
export const getDefaultServer = (tenantId?: string): string => {
|
||||
const { serverURL } = getAppModeParams();
|
||||
const storageURL = getFromStorage("SERVER_URL") as string;
|
||||
const anomalyURL = `${window.location.origin}${window.location.pathname.replace(/^\/vmui/, "")}`;
|
||||
const baseURL = window.location.href.replace(/(\/(?:prometheus\/)?(?:graph|vmui)\/.*|\/#\/.*)/, "");
|
||||
const defaultURL = baseURL.replace(/(\/select\/[\d:]+)$/, "$1/prometheus");
|
||||
const defaultURL = getDefaultURL(window.location.href);
|
||||
const url = serverURL || storageURL || defaultURL;
|
||||
|
||||
switch (APP_TYPE) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
const regexp = /(\/select\/)(\d+|\d.+)(\/)(.+)/;
|
||||
const regexp = /(\/select\/)([^/])(\/)(.+)/;
|
||||
|
||||
export const replaceTenantId = (serverUrl: string, tenantId: string) => {
|
||||
return serverUrl.replace(regexp, `$1${tenantId}/$4`);
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
package apptest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/url"
|
||||
"slices"
|
||||
@@ -503,44 +500,3 @@ func sortTSDBStatusResponseEntries(entries []TSDBStatusResponseEntry) {
|
||||
return left.Count < right.Count
|
||||
})
|
||||
}
|
||||
|
||||
// LogsQLQueryResponse is an in-memory representation of the
|
||||
// /select/logsql/query response.
|
||||
type LogsQLQueryResponse struct {
|
||||
LogLines []string
|
||||
}
|
||||
|
||||
// NewLogsQLQueryResponse is a test helper function that creates a new
|
||||
// instance of LogsQLQueryResponse by unmarshalling a json string.
|
||||
func NewLogsQLQueryResponse(t *testing.T, s string) *LogsQLQueryResponse {
|
||||
t.Helper()
|
||||
res := &LogsQLQueryResponse{}
|
||||
if len(s) == 0 {
|
||||
return res
|
||||
}
|
||||
bs := bytes.NewBufferString(s)
|
||||
for {
|
||||
logLine, err := bs.ReadString('\n')
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
if len(logLine) > 0 {
|
||||
t.Fatalf("BUG: unexpected non-empty line=%q with io.EOF", logLine)
|
||||
}
|
||||
break
|
||||
}
|
||||
t.Fatalf("BUG: cannot read logline from buffer: %s", err)
|
||||
}
|
||||
var lv map[string]any
|
||||
if err := json.Unmarshal([]byte(logLine), &lv); err != nil {
|
||||
t.Fatalf("cannot parse log line=%q: %s", logLine, err)
|
||||
}
|
||||
delete(lv, "_stream_id")
|
||||
normalizedLine, err := json.Marshal(lv)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot marshal parsed logline=%q: %s", logLine, err)
|
||||
}
|
||||
res.LogLines = append(res.LogLines, string(normalizedLine))
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
@@ -52,6 +52,7 @@ func testSpecialQueryRegression(tc *apptest.TestCase, sut apptest.PrometheusWrit
|
||||
testTooBigLookbehindWindow(tc, sut)
|
||||
testMatchSeries(tc, sut)
|
||||
testNegativeIncrease(tc, sut)
|
||||
testInstantQueryWithOffsetUsingCache(tc, sut)
|
||||
|
||||
// graphite
|
||||
testComparisonNotInfNotNan(tc, sut)
|
||||
@@ -292,6 +293,45 @@ func testNegativeIncrease(tc *apptest.TestCase, sut apptest.PrometheusWriteQueri
|
||||
})
|
||||
}
|
||||
|
||||
func testInstantQueryWithOffsetUsingCache(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
// unexpected /api/v1/query response due to wrong applied offset to request range
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9762
|
||||
sut.PrometheusAPIV1ImportPrometheus(t, []string{
|
||||
`vm_http_requests_total 1 1758196800000`, // 2025-09-18 12:00:00
|
||||
`vm_http_requests_total 2 1758218400000`, // 2025-09-18 18:00:00
|
||||
`vm_http_requests_total 3 1758240000000`, // 2025-09-19 00:00:00
|
||||
`vm_http_requests_total 4 1758261600000`, // 2025-09-19 06:00:00
|
||||
`vm_http_requests_total 5 1758283200000`, // 2025-09-19 12:00:00
|
||||
`vm_http_requests_total 6 1758304800000`, // 2025-09-19 18:00:00
|
||||
`vm_http_requests_total 7 1758326400000`, // 2025-09-20 00:00:00
|
||||
}, apptest.QueryOpts{})
|
||||
sut.ForceFlush(t)
|
||||
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
Msg: "unexpected /api/v1/query response",
|
||||
DoNotRetry: true,
|
||||
Got: func() any {
|
||||
return sut.PrometheusAPIV1Query(t, `avg_over_time(vm_http_requests_total[1d] offset 12h)`, apptest.QueryOpts{
|
||||
Time: "2025-09-20T12:00:00.000Z",
|
||||
})
|
||||
},
|
||||
Want: &apptest.PrometheusAPIV1QueryResponse{
|
||||
Status: "success",
|
||||
Data: &apptest.QueryData{
|
||||
ResultType: "vector",
|
||||
Result: []*apptest.QueryResult{
|
||||
{
|
||||
Metric: map[string]string{},
|
||||
Sample: &apptest.Sample{Timestamp: 1758369600000, Value: 5.5},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func testComparisonNotInfNotNan(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier) {
|
||||
t := tc.T()
|
||||
|
||||
|
||||
BIN
apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/chunks/000001
vendored
Normal file
BIN
apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/chunks/000001
vendored
Normal file
Binary file not shown.
BIN
apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/index
vendored
Normal file
BIN
apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/index
vendored
Normal file
Binary file not shown.
51
apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/meta.json
vendored
Normal file
51
apptest/tests/testdata/mimir-tsdb/anonymous/01JFJBS3YP1SHZ3PJQ6HK76EC3/meta.json
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"ulid": "01JFJBS3YP1SHZ3PJQ6HK76EC3",
|
||||
"minTime": 1734709200000,
|
||||
"maxTime": 1734709320000,
|
||||
"stats": {
|
||||
"numSamples": 400,
|
||||
"numSeries": 100,
|
||||
"numChunks": 100
|
||||
},
|
||||
"compaction": {
|
||||
"level": 1,
|
||||
"sources": [
|
||||
"01JFJBS3YP1SHZ3PJQ6HK76EC3"
|
||||
],
|
||||
"parents": [
|
||||
{
|
||||
"ulid": "00000000000000000000000000",
|
||||
"minTime": 0,
|
||||
"maxTime": 0
|
||||
}
|
||||
],
|
||||
"hints": [
|
||||
"from-out-of-order"
|
||||
]
|
||||
},
|
||||
"version": 1,
|
||||
"out_of_order": false,
|
||||
"thanos": {
|
||||
"labels": {},
|
||||
"downsample": {
|
||||
"resolution": 0
|
||||
},
|
||||
"source": "receive",
|
||||
"segment_files": [
|
||||
"000001"
|
||||
],
|
||||
"files": [
|
||||
{
|
||||
"rel_path": "chunks/000001",
|
||||
"size_bytes": 4808
|
||||
},
|
||||
{
|
||||
"rel_path": "index",
|
||||
"size_bytes": 55021
|
||||
},
|
||||
{
|
||||
"rel_path": "meta.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
BIN
apptest/tests/testdata/mimir-tsdb/anonymous/bucket-index.json.gz
vendored
Normal file
BIN
apptest/tests/testdata/mimir-tsdb/anonymous/bucket-index.json.gz
vendored
Normal file
Binary file not shown.
1
apptest/tests/testdata/mimir-tsdb/expected_response.json
vendored
Normal file
1
apptest/tests/testdata/mimir-tsdb/expected_response.json
vendored
Normal file
File diff suppressed because one or more lines are too long
139
apptest/tests/vmctl_mimir_migration_test.go
Normal file
139
apptest/tests/vmctl_mimir_migration_test.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
||||
)
|
||||
|
||||
const (
|
||||
testMimirPath = "testdata/mimir-tsdb"
|
||||
expectedMimirResponseFile = "./testdata/mimir-tsdb/expected_response.json"
|
||||
)
|
||||
|
||||
func TestSingleVmctlMimirProtocol(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
vmsingleDst := tc.MustStartDefaultVmsingle()
|
||||
vmAddr := fmt.Sprintf("http://%s/", vmsingleDst.HTTPAddr())
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot get current working directory: %s", err)
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
|
||||
vmctlFlags := []string{
|
||||
`mimir`,
|
||||
`--mimir-tenant-id=anonymous`,
|
||||
`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
|
||||
`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
|
||||
`--mimir-custom-s3-endpoint=http://localhost:9000`,
|
||||
`--mimir-path=` + path,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
`--vm-concurrency=6`,
|
||||
`--mimir-concurrency=6`,
|
||||
}
|
||||
|
||||
testMimirProtocol(tc, vmsingleDst, vmctlFlags)
|
||||
}
|
||||
|
||||
func TestClusterVmctlMimirProtocol(t *testing.T) {
|
||||
fs.MustRemoveDir(t.Name())
|
||||
|
||||
tc := apptest.NewTestCase(t)
|
||||
defer tc.Stop()
|
||||
|
||||
cluster := tc.MustStartDefaultCluster()
|
||||
vmAddr := fmt.Sprintf("http://%s/", cluster.Vminsert.HTTPAddr())
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("cannot get current working directory: %s", err)
|
||||
}
|
||||
|
||||
path := fmt.Sprintf("fs://%s/%s", dir, testMimirPath)
|
||||
|
||||
vmctlFlags := []string{
|
||||
`mimir`,
|
||||
`--mimir-tenant-id=anonymous`,
|
||||
`--mimir-filter-time-start=2024-12-01T00:00:00Z`,
|
||||
`--mimir-filter-time-end=2024-12-31T23:59:59Z`,
|
||||
`--mimir-custom-s3-endpoint=http://localhost:9000`,
|
||||
`--mimir-path=` + path,
|
||||
`--vm-addr=` + vmAddr,
|
||||
`--disable-progress-bar=true`,
|
||||
`--vm-concurrency=6`,
|
||||
`--mimir-concurrency=6`,
|
||||
}
|
||||
|
||||
testMimirProtocol(tc, cluster, vmctlFlags)
|
||||
}
|
||||
|
||||
func testMimirProtocol(tc *apptest.TestCase, sut apptest.PrometheusWriteQuerier, vmctlFlags []string) {
|
||||
t := tc.T()
|
||||
t.Helper()
|
||||
|
||||
cmpOpt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
|
||||
|
||||
// test for empty data request
|
||||
got := sut.PrometheusAPIV1Query(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Step: "5m",
|
||||
Time: "2025-06-02T17:14:00Z",
|
||||
})
|
||||
|
||||
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[]}}`)
|
||||
if diff := cmp.Diff(want, got, cmpOpt); diff != "" {
|
||||
t.Errorf("unexpected response (-want, +got):\n%s", diff)
|
||||
}
|
||||
|
||||
tc.MustStartVmctl("vmctl", vmctlFlags)
|
||||
|
||||
sut.ForceFlush(t)
|
||||
|
||||
// open the expected series response file
|
||||
file, err := os.Open(expectedMimirResponseFile)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot open expected series response file: %s", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
bytes, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot read expected series response file: %s", err)
|
||||
}
|
||||
|
||||
var wantResponse apptest.PrometheusAPIV1QueryResponse
|
||||
if err := json.Unmarshal(bytes, &wantResponse); err != nil {
|
||||
t.Fatalf("cannot unmarshal expected series response file: %s", err)
|
||||
}
|
||||
wantResponse.Sort()
|
||||
|
||||
tc.Assert(&apptest.AssertOptions{
|
||||
// For cluster version, we need to wait longer for the metrics to be stored
|
||||
Retries: 300,
|
||||
Msg: `unexpected metrics stored on vmsingle via the prometheus protocol`,
|
||||
Got: func() any {
|
||||
expected := sut.PrometheusAPIV1Export(t, `{__name__=~".*"}`, apptest.QueryOpts{
|
||||
Start: "2024-12-01T15:31:10Z",
|
||||
End: "2024-12-31T15:32:20Z",
|
||||
})
|
||||
expected.Sort()
|
||||
return expected.Data.Result
|
||||
},
|
||||
Want: wantResponse.Data.Result,
|
||||
CmpOpts: []cmp.Option{
|
||||
cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -1786,4 +1786,4 @@
|
||||
"uid": "gF-lxRdVz",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1168,7 +1168,7 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99,sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by(le,controller) )",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le, controller) )",
|
||||
"legendFormat": "q.99 {{controller}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -1265,7 +1265,7 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(rest_client_requests_total{job=~\"$job\"}[$__interval])) by (method,code)",
|
||||
"expr": "sum(rate(rest_client_requests_total{job=~\"$job\"}[$__interval])) by (method, code)",
|
||||
"instant": false,
|
||||
"legendFormat": "{{method}} {{code}}",
|
||||
"range": true,
|
||||
@@ -1489,7 +1489,7 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by(job)",
|
||||
"expr": "max(histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (job, instance, le))) by (job)",
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
@@ -1588,7 +1588,7 @@
|
||||
"uid": "$ds"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.99,sum(rate(rest_client_request_duration_seconds_bucket{job=~\"$job\"})) by(le,method,api) )",
|
||||
"expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le, method, api))",
|
||||
"instant": false,
|
||||
"legendFormat": "{{method}} {{api}}",
|
||||
"range": true,
|
||||
@@ -2135,6 +2135,16 @@
|
||||
"skipUrlSync": false,
|
||||
"sort": 2,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"baseFilters": [],
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"filters": [],
|
||||
"name": "adhoc",
|
||||
"type": "adhoc"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -1950,6 +1950,16 @@
|
||||
],
|
||||
"query": "*",
|
||||
"type": "textbox"
|
||||
},
|
||||
{
|
||||
"baseFilters": [],
|
||||
"datasource": {
|
||||
"type": "victoriametrics-logs-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"filters": [],
|
||||
"name": "adhoc",
|
||||
"type": "adhoc"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1962,4 +1972,4 @@
|
||||
"title": "Query Stats (cluster)",
|
||||
"uid": "feg3od1zt1fy8e",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1787,4 +1787,4 @@
|
||||
"uid": "gF-lxRdVz_vm",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1994,7 +1994,7 @@
|
||||
"baseFilters": [],
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "PE8D8DB4BEE4E4B22"
|
||||
"uid": "$ds"
|
||||
},
|
||||
"filters": [],
|
||||
"name": "adhoc",
|
||||
|
||||
@@ -2136,6 +2136,16 @@
|
||||
"skipUrlSync": false,
|
||||
"sort": 2,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"baseFilters": [],
|
||||
"datasource": {
|
||||
"type": "victoriametrics-metrics-datasource",
|
||||
"uid": "$ds"
|
||||
},
|
||||
"filters": [],
|
||||
"name": "adhoc",
|
||||
"type": "adhoc"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -4238,4 +4238,4 @@
|
||||
"title": "VictoriaMetrics - vmalert (VM)",
|
||||
"uid": "LzldHAVnz_vm",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2652,7 +2652,7 @@
|
||||
{
|
||||
"datasource": {
|
||||
"type": "victoriametrics-datasource",
|
||||
"uid": "P38648FE0F8C5BEA2"
|
||||
"uid": "$ds"
|
||||
},
|
||||
"filters": [],
|
||||
"hide": 0,
|
||||
|
||||
@@ -7,7 +7,7 @@ ROOT_IMAGE ?= alpine:3.22.1
|
||||
ROOT_IMAGE_SCRATCH ?= scratch
|
||||
CERTS_IMAGE := alpine:3.22.1
|
||||
|
||||
GO_BUILDER_IMAGE := golang:1.25.0
|
||||
GO_BUILDER_IMAGE := golang:1.25.1
|
||||
|
||||
BUILDER_IMAGE := local/builder:2.0.0-$(shell echo $(GO_BUILDER_IMAGE) | tr :/ __)-1
|
||||
BASE_IMAGE := local/base:1.1.4-$(shell echo $(ROOT_IMAGE) | tr :/ __)-$(shell echo $(CERTS_IMAGE) | tr :/ __)
|
||||
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
# It scrapes targets defined in --promscrape.config
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.125.1
|
||||
image: victoriametrics/vmagent:v1.126.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
@@ -19,7 +19,7 @@ services:
|
||||
restart: always
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.1
|
||||
image: grafana/grafana:12.2.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
@@ -37,14 +37,14 @@ services:
|
||||
# vmstorage shards. Each shard receives 1/N of all metrics sent to vminserts,
|
||||
# where N is number of vmstorages (2 in this case).
|
||||
vmstorage-1:
|
||||
image: victoriametrics/vmstorage:v1.125.1-cluster
|
||||
image: victoriametrics/vmstorage:v1.126.0-cluster
|
||||
volumes:
|
||||
- strgdata-1:/storage
|
||||
command:
|
||||
- "--storageDataPath=/storage"
|
||||
restart: always
|
||||
vmstorage-2:
|
||||
image: victoriametrics/vmstorage:v1.125.1-cluster
|
||||
image: victoriametrics/vmstorage:v1.126.0-cluster
|
||||
volumes:
|
||||
- strgdata-2:/storage
|
||||
command:
|
||||
@@ -54,7 +54,7 @@ services:
|
||||
# vminsert is ingestion frontend. It receives metrics pushed by vmagent,
|
||||
# pre-process them and distributes across configured vmstorage shards.
|
||||
vminsert-1:
|
||||
image: victoriametrics/vminsert:v1.125.1-cluster
|
||||
image: victoriametrics/vminsert:v1.126.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -63,7 +63,7 @@ services:
|
||||
- "--storageNode=vmstorage-2:8400"
|
||||
restart: always
|
||||
vminsert-2:
|
||||
image: victoriametrics/vminsert:v1.125.1-cluster
|
||||
image: victoriametrics/vminsert:v1.126.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -75,7 +75,7 @@ services:
|
||||
# vmselect is a query fronted. It serves read queries in MetricsQL or PromQL.
|
||||
# vmselect collects results from configured `--storageNode` shards.
|
||||
vmselect-1:
|
||||
image: victoriametrics/vmselect:v1.125.1-cluster
|
||||
image: victoriametrics/vmselect:v1.126.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -85,7 +85,7 @@ services:
|
||||
- "--vmalert.proxyURL=http://vmalert:8880"
|
||||
restart: always
|
||||
vmselect-2:
|
||||
image: victoriametrics/vmselect:v1.125.1-cluster
|
||||
image: victoriametrics/vmselect:v1.126.0-cluster
|
||||
depends_on:
|
||||
- "vmstorage-1"
|
||||
- "vmstorage-2"
|
||||
@@ -100,7 +100,7 @@ services:
|
||||
# read requests from Grafana, vmui, vmalert among vmselects.
|
||||
# It can be used as an authentication proxy.
|
||||
vmauth:
|
||||
image: victoriametrics/vmauth:v1.125.1
|
||||
image: victoriametrics/vmauth:v1.126.0
|
||||
depends_on:
|
||||
- "vmselect-1"
|
||||
- "vmselect-2"
|
||||
@@ -114,7 +114,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.125.1
|
||||
image: victoriametrics/vmalert:v1.126.0
|
||||
depends_on:
|
||||
- "vmauth"
|
||||
ports:
|
||||
@@ -138,7 +138,7 @@ services:
|
||||
# alertmanager receives alerting notifications from vmalert
|
||||
# and distributes them according to --config.file.
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.0
|
||||
image: prom/alertmanager:v0.28.1
|
||||
volumes:
|
||||
- ./alertmanager.yml:/config/alertmanager.yml
|
||||
command:
|
||||
|
||||
@@ -3,7 +3,7 @@ services:
|
||||
# It scrapes targets defined in --promscrape.config
|
||||
# And forward them to --remoteWrite.url
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.125.1
|
||||
image: victoriametrics/vmagent:v1.126.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
# VictoriaMetrics instance, a single process responsible for
|
||||
# storing metrics and serve read requests.
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.125.1
|
||||
image: victoriametrics/victoria-metrics:v1.126.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
- 8089:8089
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
restart: always
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.1
|
||||
image: grafana/grafana:12.2.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -54,7 +54,7 @@ services:
|
||||
|
||||
# vmalert executes alerting and recording rules
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.125.1
|
||||
image: victoriametrics/vmalert:v1.126.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
- "alertmanager"
|
||||
@@ -79,7 +79,7 @@ services:
|
||||
# alertmanager receives alerting notifications from vmalert
|
||||
# and distributes them according to --config.file.
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.0
|
||||
image: prom/alertmanager:v0.28.1
|
||||
volumes:
|
||||
- ./alertmanager.yml:/config/alertmanager.yml
|
||||
command:
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
groups:
|
||||
- name: log-rules
|
||||
type: vlogs
|
||||
interval: 30s
|
||||
rules:
|
||||
- alert: AlwaysFiring
|
||||
expr: '* | stats count()'
|
||||
annotations:
|
||||
description: "Generated more than {{$value}} log entries in the last 1 minute"
|
||||
- alert: TooManyLogs
|
||||
expr: '* | stats by (path) count() as total | filter total:>50'
|
||||
annotations:
|
||||
description: "Path {{$labels.path}} generated more than 50 log entries in the last 1 minute: {{$value}}"
|
||||
- record: path:logs:count
|
||||
expr: '* | stats by (path) count()'
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.125.1
|
||||
image: victoriametrics/vmagent:v1.126.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
restart: always
|
||||
|
||||
victoriametrics:
|
||||
image: victoriametrics/victoria-metrics:v1.125.1
|
||||
image: victoriametrics/victoria-metrics:v1.126.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -27,7 +27,7 @@ services:
|
||||
restart: always
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.1
|
||||
image: grafana/grafana:12.2.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -40,7 +40,7 @@ services:
|
||||
restart: always
|
||||
|
||||
vmalert:
|
||||
image: victoriametrics/vmalert:v1.125.1
|
||||
image: victoriametrics/vmalert:v1.126.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -73,7 +73,7 @@ services:
|
||||
- "/config.yaml"
|
||||
- "--licenseFile=/license"
|
||||
alertmanager:
|
||||
image: prom/alertmanager:v0.28.0
|
||||
image: prom/alertmanager:v0.28.1
|
||||
volumes:
|
||||
- ./alertmanager.yml:/config/alertmanager.yml
|
||||
command:
|
||||
@@ -83,7 +83,7 @@ services:
|
||||
restart: always
|
||||
|
||||
node-exporter:
|
||||
image: quay.io/prometheus/node-exporter:v1.7.0
|
||||
image: quay.io/prometheus/node-exporter:v1.9.1
|
||||
ports:
|
||||
- 9100:9100
|
||||
pid: host
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
prepare-logs:
|
||||
cd ./source_logs && bash download.sh
|
||||
|
||||
docker-up-elk:
|
||||
docker-compose -f docker-compose.yml -f docker-compose-elk.yml up -d
|
||||
|
||||
docker-stop-elk:
|
||||
docker-compose -f docker-compose.yml -f docker-compose-elk.yml stop
|
||||
|
||||
docker-up-loki:
|
||||
docker-compose -f docker-compose.yml -f docker-compose-loki.yml up -d
|
||||
|
||||
docker-stop-loki:
|
||||
docker-compose -f docker-compose.yml -f docker-compose-loki.yml stop
|
||||
|
||||
docker-cleanup:
|
||||
docker-compose -f docker-compose.yml -f docker-compose-elk.yml -f docker-compose-loki.yml down -v --remove-orphans
|
||||
@@ -1,69 +0,0 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
filebeat-elastic:
|
||||
image: docker.elastic.co/beats/filebeat:8.8.0
|
||||
restart: on-failure
|
||||
volumes:
|
||||
- ./elk/filebeat/filebeat-elastic.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
depends_on:
|
||||
- elastic
|
||||
|
||||
filebeat-vlogs:
|
||||
image: docker.elastic.co/beats/filebeat:8.8.0
|
||||
restart: on-failure
|
||||
volumes:
|
||||
- ./elk/filebeat/filebeat-vlogs.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
depends_on:
|
||||
- vlogs
|
||||
|
||||
generator:
|
||||
image: golang:1.25.0-alpine
|
||||
restart: always
|
||||
working_dir: /go/src/app
|
||||
volumes:
|
||||
- ./generator:/go/src/app
|
||||
- ./source_logs:/go/src/source_logs
|
||||
command:
|
||||
- go
|
||||
- run
|
||||
- main.go
|
||||
- -logsPath=/go/src/source_logs/logs
|
||||
- -outputRateLimitItems=10000
|
||||
- -syslog.addr=filebeat-elastic:12345
|
||||
- -syslog.addr2=filebeat-vlogs:12345
|
||||
- -logs.randomSuffix=false
|
||||
depends_on: [filebeat-elastic, filebeat-vlogs]
|
||||
|
||||
elastic:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
|
||||
volumes:
|
||||
- ./elk/elastic/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
|
||||
- elastic:/usr/share/elasticsearch/data
|
||||
environment:
|
||||
ES_JAVA_OPTS: "-Xmx2048m"
|
||||
|
||||
kibana:
|
||||
image: docker.elastic.co/kibana/kibana:8.8.0
|
||||
volumes:
|
||||
- ./elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
|
||||
ports:
|
||||
- "5601:5601"
|
||||
depends_on: [elastic]
|
||||
|
||||
beat-exporter-elastic:
|
||||
image: trustpilot/beat-exporter:0.4.0
|
||||
command:
|
||||
- -beat.uri=http://filebeat-elastic:5066
|
||||
depends_on:
|
||||
- filebeat-elastic
|
||||
|
||||
beat-exporter-vlogs:
|
||||
image: trustpilot/beat-exporter:0.4.0
|
||||
command:
|
||||
- -beat.uri=http://filebeat-vlogs:5066
|
||||
depends_on:
|
||||
- filebeat-vlogs
|
||||
|
||||
volumes:
|
||||
elastic:
|
||||
@@ -1,51 +0,0 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
generator:
|
||||
image: golang:1.25.0-alpine
|
||||
restart: always
|
||||
working_dir: /go/src/app
|
||||
volumes:
|
||||
- ./generator:/go/src/app
|
||||
- ./source_logs:/go/src/source_logs
|
||||
command:
|
||||
- go
|
||||
- run
|
||||
- main.go
|
||||
- -logsPath=/go/src/source_logs/logs
|
||||
- -outputRateLimitItems=10000
|
||||
- -outputRateLimitPeriod=1s
|
||||
- -syslog.addr=rsyslog:514
|
||||
- -syslog.addr2=rsyslog:514
|
||||
- -logs.randomSuffix=false
|
||||
depends_on: [rsyslog]
|
||||
|
||||
loki:
|
||||
image: grafana/loki:2.9.0
|
||||
user: 0:0
|
||||
ports:
|
||||
- "3100:3100"
|
||||
command: -config.file=/etc/loki/loki-config.yaml
|
||||
volumes:
|
||||
- loki:/tmp/loki
|
||||
- ./loki/:/etc/loki/
|
||||
|
||||
promtail:
|
||||
image: grafana/promtail:2.9.0
|
||||
command: -config.file=/etc/promtail/promtail-config.yaml
|
||||
volumes:
|
||||
- ./loki/:/etc/promtail/
|
||||
depends_on:
|
||||
- loki
|
||||
- vlogs
|
||||
|
||||
rsyslog:
|
||||
build:
|
||||
dockerfile: Dockerfile
|
||||
context: rsyslog
|
||||
volumes:
|
||||
- ./rsyslog/rsyslog.conf:/etc/rsyslog.conf
|
||||
depends_on: [promtail]
|
||||
|
||||
volumes:
|
||||
loki:
|
||||
@@ -1,74 +0,0 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
vlogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
volumes:
|
||||
- vlogs:/vlogs
|
||||
ports:
|
||||
- "9428:9428"
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
|
||||
cadvisor:
|
||||
image: gcr.io/cadvisor/cadvisor:v0.47.0
|
||||
restart: unless-stopped
|
||||
privileged: true
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
|
||||
node-exporter:
|
||||
image: prom/node-exporter:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /proc:/host/proc:ro
|
||||
- /sys:/host/sys:ro
|
||||
- /:/rootfs:ro
|
||||
command:
|
||||
- "--path.procfs=/host/proc"
|
||||
- "--path.rootfs=/rootfs"
|
||||
- "--path.sysfs=/host/sys"
|
||||
- "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
|
||||
|
||||
du-exporter:
|
||||
image: ghcr.io/dundee/disk_usage_exporter/disk_usage_exporter-c4084307c537335c2ddb6f4b9b527422:latest
|
||||
restart: unless-stopped
|
||||
user: "root"
|
||||
volumes:
|
||||
- /var/lib/docker/volumes:/var/lib/docker/volumes:ro
|
||||
- ./du/config.yml:/config.yml:ro
|
||||
command:
|
||||
- "--config=/config.yml"
|
||||
|
||||
vmsingle:
|
||||
image: victoriametrics/victoria-metrics:v1.109.0
|
||||
ports:
|
||||
- "8428:8428"
|
||||
command:
|
||||
- -storageDataPath=/vmsingle
|
||||
- -promscrape.config=/promscrape.yml
|
||||
- -promscrape.maxScrapeSize=1Gb
|
||||
volumes:
|
||||
- vmsingle:/vmsingle
|
||||
- ./vmsingle/promscrape.yml:/promscrape.yml
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:12.1.1
|
||||
depends_on: [vmsingle]
|
||||
ports:
|
||||
- 3000:3000
|
||||
volumes:
|
||||
- grafanadata:/var/lib/grafana
|
||||
- ./grafana/provisioning/:/etc/grafana/provisioning/
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards/
|
||||
restart: always
|
||||
|
||||
volumes:
|
||||
vlogs:
|
||||
vmsingle:
|
||||
grafanadata: {}
|
||||
@@ -1,3 +0,0 @@
|
||||
analyzed-path: /var/lib/docker/volumes
|
||||
bind-address: 0.0.0.0:9995
|
||||
dir-level: 1
|
||||
@@ -1,4 +0,0 @@
|
||||
cluster.name: "bench"
|
||||
network.host: 0.0.0.0
|
||||
xpack.security.enabled: false
|
||||
discovery.type: single-node
|
||||
@@ -1,15 +0,0 @@
|
||||
filebeat.inputs:
|
||||
- type: syslog
|
||||
format: rfc3164
|
||||
protocol.tcp:
|
||||
host: "0.0.0.0:12345"
|
||||
|
||||
output.elasticsearch:
|
||||
hosts: [ "http://elastic:9200" ]
|
||||
worker: 5
|
||||
bulk_max_size: 1000
|
||||
|
||||
http:
|
||||
enabled: true
|
||||
host: 0.0.0.0
|
||||
port: 5066
|
||||
@@ -1,19 +0,0 @@
|
||||
filebeat.inputs:
|
||||
- type: syslog
|
||||
format: rfc3164
|
||||
protocol.tcp:
|
||||
host: "0.0.0.0:12345"
|
||||
|
||||
output.elasticsearch:
|
||||
hosts: [ "http://vlogs:9428/insert/elasticsearch/" ]
|
||||
worker: 5
|
||||
bulk_max_size: 1000
|
||||
parameters:
|
||||
_msg_field: "message"
|
||||
_time_field: "@timestamp"
|
||||
_stream_fields: "host.name,process.program,process.pid"
|
||||
|
||||
http:
|
||||
enabled: true
|
||||
host: 0.0.0.0
|
||||
port: 5066
|
||||
@@ -1,3 +0,0 @@
|
||||
server.name: kibana
|
||||
server.host: "0.0.0.0"
|
||||
elasticsearch.hosts: [ "http://elastic:9200" ]
|
||||
@@ -1,106 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/syslog"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
logsPath = flag.String("logsPath", "", "Path to logs directory")
|
||||
syslogAddr = flag.String("syslog.addr", "logstash:12345", "Addr to send logs to")
|
||||
syslogAddr2 = flag.String("syslog.addr2", "logstash:12345", "Addr to send logs to")
|
||||
randomSuffix = flag.Bool("logs.randomSuffix", false, "Whether to add a random suffix to a log line")
|
||||
|
||||
outputRateLimitItems = flag.Int("outputRateLimitItems", 100, "Number of items to send per second")
|
||||
outputRateLimitPeriod = flag.Duration("outputRateLimitPeriod", time.Second, "Period of time to send items")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
startedAt := time.Now().Unix()
|
||||
|
||||
logFiles, err := os.ReadDir(*logsPath)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("error reading directory %s:%w", *logsPath, err))
|
||||
}
|
||||
|
||||
sourceFiles := make([]string, 0)
|
||||
|
||||
for _, logFile := range logFiles {
|
||||
if strings.HasSuffix(logFile.Name(), ".log") {
|
||||
sourceFiles = append(sourceFiles, logFile.Name())
|
||||
}
|
||||
}
|
||||
log.Printf("sourceFiles: %v", sourceFiles)
|
||||
log.Printf("running with rate limit: %d items per %s", *outputRateLimitItems, *outputRateLimitPeriod)
|
||||
|
||||
limitTicker := time.NewTicker(*outputRateLimitPeriod)
|
||||
limitItems := *outputRateLimitItems
|
||||
limiter := make(chan struct{}, limitItems)
|
||||
go func() {
|
||||
for {
|
||||
<-limitTicker.C
|
||||
for i := 0; i < limitItems; i++ {
|
||||
limiter <- struct{}{}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for _, sourceFile := range sourceFiles {
|
||||
log.Printf("sourceFile: %s", sourceFile)
|
||||
f, err := os.Open(*logsPath + "/" + sourceFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
syslogTag := "logs-benchmark-" + sourceFile + "-" + strconv.FormatInt(startedAt, 10)
|
||||
|
||||
// Loki uses RFC5424 syslog format, which has a 48 character limit on the tag.
|
||||
tagLen := len(syslogTag)
|
||||
if tagLen > 48 {
|
||||
truncate := tagLen - 48
|
||||
syslogTag = syslogTag[truncate:]
|
||||
}
|
||||
logger, err := syslog.Dial("tcp", *syslogAddr, syslog.LOG_INFO, syslogTag)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("error dialing syslog: %w", err))
|
||||
}
|
||||
logger2, err := syslog.Dial("tcp", *syslogAddr2, syslog.LOG_INFO, syslogTag)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("error dialing syslog: %w", err))
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
<-limiter
|
||||
line := scanner.Text()
|
||||
if *randomSuffix {
|
||||
line = line + " " + randomString()
|
||||
}
|
||||
_ = logger.Info(line)
|
||||
_ = logger2.Info(line)
|
||||
}
|
||||
|
||||
logger.Close()
|
||||
logger2.Close()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func randomString() string {
|
||||
buf := make([]byte, 4)
|
||||
ip := rand.Uint32()
|
||||
|
||||
binary.LittleEndian.PutUint32(buf, ip)
|
||||
return net.IP(buf).String()
|
||||
}
|
||||
@@ -1,393 +0,0 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "victoria-logs"
|
||||
},
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "purple",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*elastic.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "elasticsearch"
|
||||
},
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 5,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"minVizHeight": 10,
|
||||
"minVizWidth": 0,
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"}[5m])) by (name) * 100",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "CPU Usage",
|
||||
"type": "bargauge"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "purple",
|
||||
"mode": "fixed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "victoria-logs"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*elastic.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "elasticsearch"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 5,
|
||||
"x": 5,
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"minVizHeight": 10,
|
||||
"minVizWidth": 0,
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(container_memory_rss{name=~\"$containers\"}[5m]) by (name)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Memory Usage",
|
||||
"type": "bargauge"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "purple",
|
||||
"mode": "fixed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "victoria-logs"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*elastic.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "elasticsearch"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 5,
|
||||
"x": 10,
|
||||
"y": 0
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"minVizHeight": 10,
|
||||
"minVizWidth": 0,
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true
|
||||
},
|
||||
"pluginVersion": "9.2.7",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}[5m]) by (path)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Disk space used",
|
||||
"type": "bargauge"
|
||||
}
|
||||
],
|
||||
"refresh": false,
|
||||
"schemaVersion": 37,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"hide": 2,
|
||||
"name": "containers_selector",
|
||||
"query": ".*(vlogs|elastic).*",
|
||||
"skipUrlSync": false,
|
||||
"type": "constant"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": [
|
||||
"logs-benchmark-elastic-1",
|
||||
"logs-benchmark-vlogs-1"
|
||||
],
|
||||
"value": [
|
||||
"logs-benchmark-elastic-1",
|
||||
"logs-benchmark-vlogs-1"
|
||||
]
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "",
|
||||
"multi": true,
|
||||
"name": "containers",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
|
||||
"refId": "StandardVariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": ".*vlogs|elastic.*",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-30m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Elastic vs VLogs - stats only",
|
||||
"uid": "hkm6P6_4J",
|
||||
"version": 2,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,554 +0,0 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"last"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(filebeat_libbeat_output_events{type=\"acked\"})) by (instance, type)",
|
||||
"legendFormat": "{{instance}} - {{type}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Filebeat items",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*elastic.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"min",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"})) by (name)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "CPU cores usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*elastic.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"min",
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(container_memory_rss{name=~\"$containers\"}) by (name)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Memory Usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*elastic.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"min",
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}) by (path)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Disk space used",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 37,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"hide": 2,
|
||||
"name": "containers_selector",
|
||||
"query": ".*(vlogs|elastic).*",
|
||||
"skipUrlSync": false,
|
||||
"type": "constant"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": [
|
||||
"All"
|
||||
],
|
||||
"value": [
|
||||
"$__all"
|
||||
]
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "",
|
||||
"multi": true,
|
||||
"name": "containers",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
|
||||
"refId": "StandardVariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": ".*benchmark-vlogs|benchmark-elastic.*",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Elastic vs VLogs",
|
||||
"uid": "hkm6P6_4z",
|
||||
"version": 3,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,597 +0,0 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*VictoriaLogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*Loki.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"last"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(vl_rows_ingested_total))",
|
||||
"legendFormat": "VictoriaLogs",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rate(loki_distributor_lines_received_total)",
|
||||
"hide": false,
|
||||
"legendFormat": "Loki",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Ingestion rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*loki.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 7
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"min",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$containers\"})) by (name)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "CPU cores usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*loki.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"min",
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(container_memory_rss{name=~\"$containers\"}) by (name)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Memory Usage",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*vlogs.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "yellow",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byRegexp",
|
||||
"options": ".*loki.*"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "color",
|
||||
"value": {
|
||||
"fixedColor": "green",
|
||||
"mode": "fixed"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 11,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"min",
|
||||
"max",
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(node_disk_usage_bytes{path=~\"$containers_selector\"}) by (path)",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Disk space used",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "10s",
|
||||
"schemaVersion": 37,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"hide": 2,
|
||||
"name": "containers_selector",
|
||||
"query": ".*(vlogs|loki).*",
|
||||
"skipUrlSync": false,
|
||||
"type": "constant"
|
||||
},
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": [
|
||||
"All"
|
||||
],
|
||||
"value": [
|
||||
"$__all"
|
||||
]
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "P4169E866C3094E38"
|
||||
},
|
||||
"definition": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "",
|
||||
"multi": true,
|
||||
"name": "containers",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(container_cpu_usage_seconds_total{name!=\"\",}, name)",
|
||||
"refId": "StandardVariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": ".*benchmark-vlogs|benchmark-loki.*",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Loki vs VLogs",
|
||||
"uid": "hkm6P6_4y",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -1,8 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: VictoriaMetrics
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://vmsingle:8428
|
||||
isDefault: true
|
||||
@@ -1,63 +0,0 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
instance_addr: 0.0.0.0
|
||||
path_prefix: /tmp/loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /tmp/loki/chunks
|
||||
rules_directory: /tmp/loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
query_range:
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: boltdb-shipper
|
||||
object_store: filesystem
|
||||
schema: v11
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
|
||||
# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
|
||||
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
|
||||
#
|
||||
# Statistics help us better understand how Loki is used, and they show us performance
|
||||
# levels for most users. This helps us prioritize features and documentation.
|
||||
# For more information on what's sent, look at
|
||||
# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
|
||||
# Refer to the buildReport method to see what goes into a report.
|
||||
#
|
||||
# If you would like to disable reporting, uncomment the following lines:
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
|
||||
|
||||
limits_config:
|
||||
ingestion_rate_mb: 10240
|
||||
ingestion_burst_size_mb: 10240
|
||||
max_streams_per_user: 10000000
|
||||
max_global_streams_per_user: 10000000
|
||||
|
||||
retention_period: 30d
|
||||
|
||||
per_stream_rate_limit: 10240M
|
||||
per_stream_rate_limit_burst: 10240M
|
||||
cardinality_limit: 20000000
|
||||
@@ -1,26 +0,0 @@
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
- url: http://vlogs:9428/insert/loki/api/v1/push?_stream_fields=hostname,application
|
||||
# batchwait: 5s
|
||||
# batchsize: 5242880
|
||||
|
||||
scrape_configs:
|
||||
- job_name: syslog
|
||||
syslog:
|
||||
listen_address: 0.0.0.0:5140
|
||||
idle_timeout: 12h
|
||||
use_incoming_timestamp: true
|
||||
labels:
|
||||
job: syslog
|
||||
relabel_configs:
|
||||
- source_labels: [__syslog_message_hostname]
|
||||
target_label: hostname
|
||||
- source_labels: [__syslog_message_app_name]
|
||||
target_label: application
|
||||
@@ -1,125 +0,0 @@
|
||||
# Benchmark for VictoriaLogs
|
||||
|
||||
Benchmark compares VictoriaLogs with ELK stack and Grafana Loki.
|
||||
|
||||
Benchmark is based on:
|
||||
|
||||
- Logs from this repository - [https://github.com/logpai/loghub](https://github.com/logpai/loghub)
|
||||
- [logs generator](./generator)
|
||||
|
||||
For ELK suite it uses:
|
||||
|
||||
- filebeat - [https://www.elastic.co/beats/filebeat](https://www.elastic.co/beats/filebeat)
|
||||
- elastic + kibana
|
||||
|
||||
For Grafana Loki suite it uses:
|
||||
|
||||
- [Promtail](https://grafana.com/docs/loki/latest/send-data/promtail/)
|
||||
- rsyslog - required to push logs in RFC5424 format to Promtail
|
||||
- [Loki](https://grafana.com/oss/loki/)
|
||||
|
||||
## How it works
|
||||
|
||||
[docker-compose.yml](./docker-compose.yml) contains common configurations for all suites:
|
||||
|
||||
- VictoriaLogs instance
|
||||
- vmsingle - port forwarded to `localhost:8428` to see UI
|
||||
- exporters for system metrics
|
||||
|
||||
ELK suite uses [docker-compose-elk.yml](./docker-compose-elk.yml) with the following services:
|
||||
|
||||
- [logs generator](./generator) which generates logs and sends them to filebeat instances via syslog
|
||||
- 2 filebeat instances - one for elastic and one for VictoriaLogs.
|
||||
- elastic instance
|
||||
- kibana instance - port forwarded to `localhost:5601` to see UI
|
||||
|
||||
Loki suite uses [docker-compose-loki.yml](./docker-compose-loki.yml) with the following services:
|
||||
|
||||
- [logs generator](./generator) which generates logs and sends them rsyslog
|
||||
- rsyslog instance - sends logs to Promtail
|
||||
- Promtail instance - sends logs to Loki and VictoriaLogs
|
||||
- Loki instance
|
||||
|
||||
[Logs generator](./generator) generates logs based on logs located at `./source_logs/logs` and sends them to filebeat
|
||||
instances via syslog.
|
||||
Logs are generated by reading files line by line, adding randomized suffix to each line and sending them to filebeat via
|
||||
syslog.
|
||||
By default, generator will exit once all files are read. `docker-compose` will restart it and files will be read again
|
||||
generating new logs.
|
||||
|
||||
Each filebeat than writes logs to elastic and VictoriaLogs via elasticsearch-compatible API.
|
||||
|
||||
## How to run
|
||||
|
||||
1. Download and unarchive logs by running:
|
||||
|
||||
```shell
|
||||
cd source_logs
|
||||
bash download.sh
|
||||
```
|
||||
|
||||
Note that with logs listed in `download.sh` it will require 49GB of free space:
|
||||
|
||||
- 3GB for archives
|
||||
- 46GB for unarchived logs
|
||||
|
||||
If it is needed to minimize disk footprint, you can download only some of them by commenting out lines in `download.sh`.
|
||||
Unarchived logs size per file for reference:
|
||||
|
||||
```shell
|
||||
2.3M Linux.log
|
||||
73M SSH.log
|
||||
32G Thunderbird.log
|
||||
5.1M Apache.log
|
||||
13G hadoop-*.log
|
||||
```
|
||||
|
||||
1. (optional) If needed, adjust amount of logs sent by generator by modifying `-outputRateLimitItems` and
|
||||
`outputRateLimitPeriod` parameters in [docker-compose.yml](./docker-compose.yml). By default, it is configured to
|
||||
send 10000 logs per second.
|
||||
|
||||
1. (optional) Build victoria-logs image and adjust `image` parameter in [docker-compose.yml](./docker-compose.yml):
|
||||
|
||||
```shell
|
||||
make package-victoria-logs
|
||||
```
|
||||
|
||||
Image name should be replaced at `vlogs` service in [docker-compose.yml](./docker-compose.yml).
|
||||
|
||||
It is also possible to configure filebeat to send logs to VictoriaLogs running on local machine.
|
||||
To do this modify [filebeat config for vlogs](./elk/filebeat/filebeat-vlogs.yml) and replace `vlogs` address
|
||||
with address of local VictoriaLogs instance:
|
||||
|
||||
```yaml
|
||||
output.elasticsearch:
|
||||
hosts: [ "http://vlogs:9428/insert/elasticsearch/" ]
|
||||
```
|
||||
|
||||
1. Choose a suite to run.
|
||||
|
||||
In order to run ELK suite use the following command:
|
||||
|
||||
```sh
|
||||
make docker-up-elk
|
||||
```
|
||||
|
||||
In order to run Loki suite use the following command:
|
||||
|
||||
```sh
|
||||
make docker-up-loki
|
||||
```
|
||||
|
||||
1. Navigate to `http://localhost:3000/` to see Grafana dashboards with resource usage
|
||||
comparison.
|
||||
|
||||
Navigate to `http://localhost:3000/d/hkm6P6_4z/elastic-vs-vlogs` to see ELK suite results.
|
||||
|
||||
Navigate to `http://localhost:3000/d/hkm6P6_4y/loki-vs-vlogs` to see Loki suite results.
|
||||
|
||||
Example results vs ELK:
|
||||
|
||||

|
||||
|
||||
Example results vs Loki:
|
||||
|
||||

|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 157 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 176 KiB |
@@ -1,9 +0,0 @@
|
||||
FROM alpine:3
|
||||
RUN apk add --no-cache rsyslog rsyslog-tls tzdata
|
||||
|
||||
COPY rsyslog.conf /etc/rsyslog.conf
|
||||
|
||||
VOLUME /var/run/rsyslog/dev
|
||||
EXPOSE 514 10514
|
||||
|
||||
CMD ["/usr/sbin/rsyslogd", "-n"]
|
||||
@@ -1,12 +0,0 @@
|
||||
module(load="imudp")
|
||||
input(type="imudp" port="514")
|
||||
|
||||
module(load="imtcp")
|
||||
input(type="imtcp" port="514")
|
||||
|
||||
*.* action(type="omfwd"
|
||||
protocol="tcp" target="promtail" port="5140"
|
||||
Template="RSYSLOG_SyslogProtocol23Format"
|
||||
TCP_Framing="octet-counted" KeepAlive="on"
|
||||
action.resumeRetryCount="-1"
|
||||
queue.type="linkedlist" queue.size="1000000")
|
||||
@@ -1,2 +0,0 @@
|
||||
*.log
|
||||
*.tar.gz
|
||||
@@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
# Unarchived size: 5.1M Apache.log
|
||||
if [ ! -f Apache.tar.gz ]; then
|
||||
curl -o Apache.tar.gz -L -C - https://zenodo.org/records/3227177/files/Apache.tar.gz?download=1
|
||||
fi
|
||||
|
||||
# Unarchived size: 13G hadoop-*.log
|
||||
if [ ! -f HDFS_2.tar.gz ]; then
|
||||
curl -o HDFS_2.tar.gz -L -C - https://zenodo.org/records/3227177/files/HDFS_2.tar.gz?download=1
|
||||
fi
|
||||
|
||||
# Unarchived size: 2.3M Linux.log
|
||||
if [ ! -f Linux.tar.gz ]; then
|
||||
curl -o Linux.tar.gz -L -C - https://zenodo.org/records/3227177/files/Linux.tar.gz?download=1
|
||||
fi
|
||||
|
||||
# Unarchived size: 32G Thunderbird.log
|
||||
if [ ! -f Thunderbird.tar.gz ]; then
|
||||
curl -o Thunderbird.tar.gz -L -C - https://zenodo.org/records/3227177/files/Thunderbird.tar.gz?download=1
|
||||
fi
|
||||
|
||||
# Unarchived size: 73M SSH.log
|
||||
if [ ! -f SSH.tar.gz ]; then
|
||||
curl -o SSH.tar.gz -L -C - https://zenodo.org/records/3227177/files/SSH.tar.gz?download=1
|
||||
fi
|
||||
|
||||
mkdir -p logs
|
||||
|
||||
for file in *.tar.gz; do
|
||||
tar -xzf $file -C logs
|
||||
done
|
||||
@@ -1,30 +0,0 @@
|
||||
scrape_configs:
|
||||
- job_name: "filebeat"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- beat-exporter-elastic:9479
|
||||
- beat-exporter-vlogs:9479
|
||||
- job_name: "victoria-logs"
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- targets:
|
||||
- vlogs:9428
|
||||
- job_name: "cadvisor"
|
||||
scrape_interval: 30s
|
||||
metric_relabel_configs:
|
||||
- action: labeldrop
|
||||
regex: "container_label_.*"
|
||||
static_configs:
|
||||
- targets:
|
||||
- cadvisor:8080
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
- targets: ['node-exporter:9100']
|
||||
- targets: ['du-exporter:9995']
|
||||
- job_name: 'loki'
|
||||
static_configs:
|
||||
- targets: ['loki:3100']
|
||||
- job_name: 'promtail'
|
||||
static_configs:
|
||||
- targets: ['promtail:9080']
|
||||
@@ -10,9 +10,9 @@ sitemap:
|
||||
|
||||
- To use *vmanomaly*, part of the enterprise package, a license key is required. Obtain your key [here](https://victoriametrics.com/products/enterprise/trial/) for this tutorial or for enterprise use.
|
||||
- In the tutorial, we'll be using the following VictoriaMetrics components:
|
||||
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.125.1)
|
||||
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.125.1)
|
||||
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.125.1)
|
||||
- [VictoriaMetrics Single-Node](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) (v1.126.0)
|
||||
- [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/) (v1.126.0)
|
||||
- [vmagent](https://docs.victoriametrics.com/victoriametrics/vmagent/) (v1.126.0)
|
||||
- [Grafana](https://grafana.com/) (v.10.2.1)
|
||||
- [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/)
|
||||
- [Node exporter](https://github.com/prometheus/node_exporter#node-exporter) (v1.7.0) and [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (v0.27.0)
|
||||
@@ -323,7 +323,7 @@ Let's wrap it all up together into the `docker-compose.yml` file.
|
||||
services:
|
||||
vmagent:
|
||||
container_name: vmagent
|
||||
image: victoriametrics/vmagent:v1.125.1
|
||||
image: victoriametrics/vmagent:v1.126.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
@@ -340,7 +340,7 @@ services:
|
||||
|
||||
victoriametrics:
|
||||
container_name: victoriametrics
|
||||
image: victoriametrics/victoria-metrics:v1.125.1
|
||||
image: victoriametrics/victoria-metrics:v1.126.0
|
||||
ports:
|
||||
- 8428:8428
|
||||
volumes:
|
||||
@@ -373,7 +373,7 @@ services:
|
||||
|
||||
vmalert:
|
||||
container_name: vmalert
|
||||
image: victoriametrics/vmalert:v1.125.1
|
||||
image: victoriametrics/vmalert:v1.126.0
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
0
docs/changelog/CHANGELOG.md
Normal file
0
docs/changelog/CHANGELOG.md
Normal file
@@ -249,27 +249,27 @@ services:
|
||||
- grafana_data:/var/lib/grafana/
|
||||
|
||||
vmsingle:
|
||||
image: victoriametrics/victoria-metrics:v1.125.1
|
||||
image: victoriametrics/victoria-metrics:v1.126.0
|
||||
command:
|
||||
- -httpListenAddr=0.0.0.0:8429
|
||||
|
||||
vmstorage:
|
||||
image: victoriametrics/vmstorage:v1.125.1-cluster
|
||||
image: victoriametrics/vmstorage:v1.126.0-cluster
|
||||
|
||||
vminsert:
|
||||
image: victoriametrics/vminsert:v1.125.1-cluster
|
||||
image: victoriametrics/vminsert:v1.126.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8400
|
||||
- -httpListenAddr=0.0.0.0:8480
|
||||
|
||||
vmselect:
|
||||
image: victoriametrics/vmselect:v1.125.1-cluster
|
||||
image: victoriametrics/vmselect:v1.126.0-cluster
|
||||
command:
|
||||
- -storageNode=vmstorage:8401
|
||||
- -httpListenAddr=0.0.0.0:8481
|
||||
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.125.1
|
||||
image: victoriametrics/vmagent:v1.126.0
|
||||
volumes:
|
||||
- ./scrape.yaml:/etc/vmagent/config.yaml
|
||||
command:
|
||||
@@ -278,7 +278,7 @@ services:
|
||||
- -remoteWrite.url=http://vmsingle:8429/api/v1/write
|
||||
|
||||
vmgateway-cluster:
|
||||
image: victoriametrics/vmgateway:v1.125.1-enterprise
|
||||
image: victoriametrics/vmgateway:v1.126.0-enterprise
|
||||
ports:
|
||||
- 8431:8431
|
||||
volumes:
|
||||
@@ -294,7 +294,7 @@ services:
|
||||
- -auth.oidcDiscoveryEndpoints=http://keycloak:8080/realms/master/.well-known/openid-configuration
|
||||
|
||||
vmgateway-single:
|
||||
image: victoriametrics/vmgateway:v1.125.1-enterprise
|
||||
image: victoriametrics/vmgateway:v1.126.0-enterprise
|
||||
ports:
|
||||
- 8432:8431
|
||||
volumes:
|
||||
@@ -405,7 +405,7 @@ Once iDP configuration is done, vmagent configuration needs to be updated to use
|
||||
|
||||
```yaml
|
||||
vmagent:
|
||||
image: victoriametrics/vmagent:v1.125.1
|
||||
image: victoriametrics/vmagent:v1.126.0
|
||||
volumes:
|
||||
- ./scrape.yaml:/etc/vmagent/config.yaml
|
||||
- ./vmagent-client-secret:/etc/vmagent/oauth2-client-secret
|
||||
|
||||
@@ -125,7 +125,7 @@ As a reference, see resource consumption of VictoriaMetrics cluster on our [play
|
||||
The Retention Period is the number of days or months for storing data. It affects the disk space usage.
|
||||
The formula for calculating required disk space is the following:
|
||||
```
|
||||
Bytes Per Sample * Ingestion rate * Replication Factor * (Retention Period in Seconds +1 Retention Cycle(day or month)) * 1.2 (recommended 20% of dree space for merges )
|
||||
Bytes Per Sample * Ingestion rate * Replication Factor * (Retention Period in Seconds +1 Retention Cycle(day or month)) * 1.2 (recommended 20% of free space for merges )
|
||||
```
|
||||
|
||||
The **Retention Cycle** is one **day** or one **month**. If the retention period is higher than 30 days cycle is a month; otherwise day.
|
||||
|
||||
@@ -69,7 +69,6 @@ See also [case studies](https://docs.victoriametrics.com/victoriametrics/casestu
|
||||
* [Solving Metrics at scale with VictoriaMetrics](https://www.youtube.com/watch?v=QgLMztnj7-8)
|
||||
* [Monitoring as Code на базе VictoriaMetrics и Grafana](https://habr.com/ru/post/568090/)
|
||||
* [Push Prometheus metrics to VictoriaMetrics or other exporters](https://github.com/gistart/prometheus-push-client)
|
||||
* [Install and configure VictoriaMetrics on Debian](https://www.vultr.com/docs/install-and-configure-victoriametrics-on-debian)
|
||||
* [Superset BI with Victoria Metrics](https://cer6erus.medium.com/superset-bi-with-victoria-metrics-a109d3e91bc6)
|
||||
* [VictoriaMetrics Source Code Analysis - Bloom filter](https://www.sobyte.net/post/2022-05/victoriametrics-bloomfilter/)
|
||||
* [How we tried using VictoriaMetrics and Thanos at the same time](https://medium.com/@uburro/how-we-tried-using-victoriametrics-and-thanos-at-the-same-time-48803d2a638b)
|
||||
|
||||
@@ -77,8 +77,9 @@ Pull requests requirements:
|
||||
1. Avoid modifying code in the `/vendor` folder manually, even when the vendored package originates are from the VictoriaMetrics GitHub organization.
|
||||
For instance, VictoriaLogs vendors packages under the `/lib` folder from VictoriaMetrics, and VictoriaTraces vendors the `/lib/logstorage` package from VictoriaLogs.
|
||||
Submit a pull request to the upstream repository first. Afterward, a separate pull request can be opened to update the version of the vendored folder in downstream repository.
|
||||
The update of vendored package can be done with: run `go get` with the **tag** (avoid using the commit hash),
|
||||
and then run `go mod tidy` and `go mod vendor` to update the `go.mod`, `go.sum` and `/vendor`.
|
||||
* For common packages, the vendored package can be updated with this command: `go get <dependency>@vX.Y.Z`.
|
||||
* For VictoriaMetrics packages, use `go get <dependency>@canonical_commit_hash`.
|
||||
Finally, run `go mod tidy` and `go mod vendor` to update `go.mod`, `go.sum`, and `/vendor`.
|
||||
1. Ping reviewers who you think have the best expertise on the matter.
|
||||
|
||||
See good example of a [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6487).
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user