Compare commits

..

1 Commits

Author SHA1 Message Date
f41gh7
b5e4499c29 CHANGELOG.md: cut v1.108.0 release 2024-12-13 17:06:34 +01:00
239 changed files with 4532 additions and 7168 deletions

View File

@@ -14,7 +14,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/promql"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
@@ -40,20 +39,9 @@ var (
"The saved data survives unclean shutdowns such as OOM crash, hardware reset, SIGKILL, etc. "+
"Bigger intervals may help increase the lifetime of flash storage with limited write cycles (e.g. Raspberry PI). "+
"Smaller intervals increase disk IO load. Minimum supported value is 1s")
maxIngestionRate = flag.Int("maxIngestionRate", 0, "The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded. "+
"By default there are no limits on samples ingestion rate.")
)
func main() {
// VictoriaMetrics is optimized for reduced memory allocations,
// so it can run with the reduced GOGC in order to reduce the used memory,
// while keeping CPU usage spent in GC at low levels.
//
// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
// exceeds 0.05 for extended periods of time.
cgroup.SetGOGC(30)
// Write flags and help message to stdout, since it is easier to grep or pipe.
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage
@@ -88,7 +76,6 @@ func main() {
storage.SetDataFlushInterval(*inmemoryDataFlushInterval)
vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
vmselect.Init()
vminsertcommon.StartIngestionRateLimiter(*maxIngestionRate)
vminsert.Init()
startSelfScraper()
@@ -110,7 +97,6 @@ func main() {
}
logger.Infof("successfully shut down the webservice in %.3f seconds", time.Since(startTime).Seconds())
vminsert.Stop()
vminsertcommon.StopIngestionRateLimiter()
vmstorage.Stop()
vmselect.Stop()

View File

@@ -14,7 +14,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutils"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
@@ -22,11 +21,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
)
var (
datadogStreamFields = flagutil.NewArrayString("datadog.streamFields", "Datadog tags to be used as stream fields.")
datadogIgnoreFields = flagutil.NewArrayString("datadog.ignoreFields", "Datadog tags to ignore.")
)
var parserPool fastjson.ParserPool
// RequestHandler processes Datadog insert requests
@@ -85,13 +79,6 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
return true
}
if len(cp.StreamFields) == 0 {
cp.StreamFields = *datadogStreamFields
}
if len(cp.IgnoreFields) == 0 {
cp.IgnoreFields = *datadogIgnoreFields
}
if err := vlstorage.CanWriteData(); err != nil {
httpserver.Errorf(w, r, "%s", err)
return true
@@ -118,70 +105,6 @@ var (
v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
)
// datadog message field has two formats:
// - regular log message with string text
// - nested json format for serverless plugins
// which has folowing format:
// {"message": {"message": "text","lamdba": {"arn": "string","requestID": "string"}, "timestamp": int64} }
//
// See https://github.com/DataDog/datadog-lambda-extension/blob/28b90c7e4e985b72d60b5f5a5147c69c7ac693c4/bottlecap/src/logs/lambda/mod.rs#L24
func appendMsgFields(fields []logstorage.Field, v *fastjson.Value) ([]logstorage.Field, error) {
switch v.Type() {
case fastjson.TypeString:
val := v.GetStringBytes()
fields = append(fields, logstorage.Field{
Name: "_msg",
Value: bytesutil.ToUnsafeString(val),
})
case fastjson.TypeObject:
var firstErr error
v.GetObject().Visit(func(k []byte, v *fastjson.Value) {
if firstErr != nil {
return
}
switch bytesutil.ToUnsafeString(k) {
case "message":
val := v.GetStringBytes()
fields = append(fields, logstorage.Field{
Name: "_msg",
Value: bytesutil.ToUnsafeString(val),
})
case "status":
val := v.GetStringBytes()
fields = append(fields, logstorage.Field{
Name: "status",
Value: bytesutil.ToUnsafeString(val),
})
case "lamdba":
obj, err := v.Object()
if err != nil {
firstErr = err
firstErr = fmt.Errorf("unexpected lambda value type for %q:%q; want object", k, v)
return
}
obj.Visit(func(k []byte, v *fastjson.Value) {
if firstErr != nil {
return
}
val, err := v.StringBytes()
if err != nil {
firstErr = fmt.Errorf("unexpected lambda label value type for %q:%q; want string", k, v)
return
}
fields = append(fields, logstorage.Field{
Name: bytesutil.ToUnsafeString(k),
Value: bytesutil.ToUnsafeString(val),
})
})
}
})
default:
return fields, fmt.Errorf("unsupported message type %q", v.Type().String())
}
return fields, nil
}
// readLogsRequest parses data according to DataDog logs format
// https://docs.datadoghq.com/api/latest/logs/#send-logs
func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor) error {
@@ -206,27 +129,19 @@ func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor)
if err != nil {
return
}
switch bytesutil.ToUnsafeString(k) {
val, e := v.StringBytes()
if e != nil {
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
return
}
switch string(k) {
case "message":
fields, err = appendMsgFields(fields, v)
if err != nil {
return
}
case "timestamp":
val, e := v.Int64()
if e != nil {
err = fmt.Errorf("failed to parse timestamp for %q:%q", k, v)
}
if val > 0 {
ts = val * 1e6
}
fields = append(fields, logstorage.Field{
Name: "_msg",
Value: bytesutil.ToUnsafeString(val),
})
case "ddtags":
// https://docs.datadoghq.com/getting_started/tagging/
val, e := v.StringBytes()
if e != nil {
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
return
}
var pair []byte
idx := 0
for idx >= 0 {
@@ -253,20 +168,12 @@ func readLogsRequest(ts int64, data []byte, lmp insertutils.LogMessageProcessor)
}
}
default:
val, e := v.StringBytes()
if e != nil {
err = fmt.Errorf("unexpected label value type for %q:%q; want string", k, v)
return
}
fields = append(fields, logstorage.Field{
Name: bytesutil.ToUnsafeString(k),
Value: bytesutil.ToUnsafeString(val),
})
}
})
if err != nil {
return err
}
lmp.AddRow(ts, fields, nil)
fields = fields[:0]
}

View File

@@ -54,12 +54,6 @@ func TestReadLogsRequestSuccess(t *testing.T) {
"hostname":"127.0.0.1",
"message":"bar",
"service":"test"
}, {
"ddsource":"nginx",
"ddtags":"tag1:value1,tag2:value2",
"hostname":"127.0.0.1",
"message":{"message": "nested"},
"service":"test"
}, {
"ddsource":"nginx",
"ddtags":"tag1:value1,tag2:value2",
@@ -92,9 +86,8 @@ func TestReadLogsRequestSuccess(t *testing.T) {
"service":"test"
}
]`
rowsExpected := 7
rowsExpected := 6
resultExpected := `{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"bar","service":"test"}
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"nested","service":"test"}
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"foobar","service":"test"}
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"baz","service":"test"}
{"ddsource":"nginx","tag1":"value1","tag2":"value2","hostname":"127.0.0.1","_msg":"xyz","service":"test"}

View File

@@ -1,7 +1,6 @@
package vlinsert
import (
"fmt"
"net/http"
"strings"
@@ -35,15 +34,9 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
path = strings.TrimPrefix(path, "/insert")
path = strings.ReplaceAll(path, "//", "/")
switch path {
case "/jsonline":
if path == "/jsonline" {
jsonline.RequestHandler(w, r)
return true
case "/ready":
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
fmt.Fprintf(w, `{"status":"ok"}`)
return true
}
switch {
case strings.HasPrefix(path, "/elasticsearch/"):

View File

@@ -5,7 +5,6 @@ import (
"fmt"
"math"
"net/http"
"regexp"
"slices"
"sort"
"strconv"
@@ -14,7 +13,6 @@ import (
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/valyala/fastjson"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
@@ -50,10 +48,9 @@ func ProcessFacetsRequest(ctx context.Context, w http.ResponseWriter, r *http.Re
httpserver.Errorf(w, r, "%s", err)
return
}
keepConstFields := httputils.GetBool(r, "keep_const_fields")
q.DropAllPipes()
q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen, keepConstFields)
q.AddFacetsPipe(limit, maxValuesPerField, maxValueLen)
var mLock sync.Mutex
m := make(map[string][]facetEntry)
@@ -1095,20 +1092,18 @@ func parseCommonArgs(r *http.Request) (*logstorage.Query, []logstorage.TenantID,
}
// Parse optional extra_filters
extraFiltersStr := r.FormValue("extra_filters")
extraFilters, err := parseExtraFilters(extraFiltersStr)
extraFilters, err := getExtraFilters(r, "extra_filters")
if err != nil {
return nil, nil, err
}
q.AddExtraFilters(extraFilters)
// Parse optional extra_stream_filters
extraStreamFiltersStr := r.FormValue("extra_stream_filters")
extraStreamFilters, err := parseExtraStreamFilters(extraStreamFiltersStr)
extraStreamFilters, err := getExtraFilters(r, "extra_stream_filters")
if err != nil {
return nil, nil, err
}
q.AddExtraFilters(extraStreamFilters)
q.AddExtraStreamFilters(extraStreamFilters)
return q, tenantIDs, nil
}
@@ -1126,114 +1121,15 @@ func getTimeNsec(r *http.Request, argName string) (int64, bool, error) {
return nsecs, true, nil
}
func parseExtraFilters(s string) (*logstorage.Filter, error) {
func getExtraFilters(r *http.Request, argName string) ([]logstorage.Field, error) {
s := r.FormValue(argName)
if s == "" {
return nil, nil
}
if !strings.HasPrefix(s, `{"`) {
return logstorage.ParseFilter(s)
}
// Extra filters in the form {"field":"value",...}.
kvs, err := parseExtraFiltersJSON(s)
if err != nil {
return nil, err
var p logstorage.JSONParser
if err := p.ParseLogMessage([]byte(s)); err != nil {
return nil, fmt.Errorf("cannot parse %s: %w", argName, err)
}
filters := make([]string, len(kvs))
for i, kv := range kvs {
if len(kv.values) == 1 {
filters[i] = fmt.Sprintf("%q:=%q", kv.key, kv.values[0])
} else {
orValues := make([]string, len(kv.values))
for j, v := range kv.values {
orValues[j] = fmt.Sprintf("%q", v)
}
filters[i] = fmt.Sprintf("%q:in(%s)", kv.key, strings.Join(orValues, ","))
}
}
s = strings.Join(filters, " ")
return logstorage.ParseFilter(s)
}
func parseExtraStreamFilters(s string) (*logstorage.Filter, error) {
if s == "" {
return nil, nil
}
if !strings.HasPrefix(s, `{"`) {
return logstorage.ParseFilter(s)
}
// Extra stream filters in the form {"field":"value",...}.
kvs, err := parseExtraFiltersJSON(s)
if err != nil {
return nil, err
}
filters := make([]string, len(kvs))
for i, kv := range kvs {
if len(kv.values) == 1 {
filters[i] = fmt.Sprintf("%q=%q", kv.key, kv.values[0])
} else {
orValues := make([]string, len(kv.values))
for j, v := range kv.values {
orValues[j] = regexp.QuoteMeta(v)
}
filters[i] = fmt.Sprintf("%q=~%q", kv.key, strings.Join(orValues, "|"))
}
}
s = "{" + strings.Join(filters, ",") + "}"
return logstorage.ParseFilter(s)
}
type extraFilter struct {
key string
values []string
}
func parseExtraFiltersJSON(s string) ([]extraFilter, error) {
v, err := fastjson.Parse(s)
if err != nil {
return nil, err
}
o := v.GetObject()
var errOuter error
var filters []extraFilter
o.Visit(func(k []byte, v *fastjson.Value) {
if errOuter != nil {
return
}
switch v.Type() {
case fastjson.TypeString:
filters = append(filters, extraFilter{
key: string(k),
values: []string{string(v.GetStringBytes())},
})
case fastjson.TypeArray:
a := v.GetArray()
if len(a) == 0 {
return
}
orValues := make([]string, len(a))
for i, av := range a {
ov, err := av.StringBytes()
if err != nil {
errOuter = fmt.Errorf("cannot obtain string item at the array for key %q; item: %s", k, av)
return
}
orValues[i] = string(ov)
}
filters = append(filters, extraFilter{
key: string(k),
values: orValues,
})
default:
errOuter = fmt.Errorf("unexpected type of value for key %q: %s; value: %s", k, v.Type(), v)
}
})
if errOuter != nil {
return nil, errOuter
}
return filters, nil
return p.Fields, nil
}

View File

@@ -1,103 +0,0 @@
package logsql
import (
"testing"
)
func TestParseExtraFilters_Success(t *testing.T) {
f := func(s, resultExpected string) {
t.Helper()
f, err := parseExtraFilters(s)
if err != nil {
t.Fatalf("unexpected error in parseExtraFilters: %s", err)
}
result := f.String()
if result != resultExpected {
t.Fatalf("unexpected result\ngot\n%s\nwant\n%s", result, resultExpected)
}
}
f("", "")
// JSON string
f(`{"foo":"bar"}`, `foo:=bar`)
f(`{"foo":["bar","baz"]}`, `foo:in(bar,baz)`)
f(`{"z":"=b ","c":["d","e,"],"a":[],"_msg":"x"}`, `z:="=b " c:in(d,"e,") =x`)
// LogsQL filter
f(`foobar`, `foobar`)
f(`foo:bar`, `foo:bar`)
f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
}
func TestParseExtraFilters_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, err := parseExtraFilters(s)
if err == nil {
t.Fatalf("expecting non-nil error")
}
}
// Invalid JSON
f(`{"foo"}`)
f(`[1,2]`)
f(`{"foo":[1]}`)
// Invliad LogsQL filter
f(`foo:(bar`)
// excess pipe
f(`foo | count()`)
}
func TestParseExtraStreamFilters_Success(t *testing.T) {
f := func(s, resultExpected string) {
t.Helper()
f, err := parseExtraStreamFilters(s)
if err != nil {
t.Fatalf("unexpected error in parseExtraStreamFilters: %s", err)
}
result := f.String()
if result != resultExpected {
t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", result, resultExpected)
}
}
f("", "")
// JSON string
f(`{"foo":"bar"}`, `{foo="bar"}`)
f(`{"foo":["bar","baz"]}`, `{foo=~"bar|baz"}`)
f(`{"z":"b","c":["d","e|\""],"a":[],"_msg":"x"}`, `{z="b",c=~"d|e\\|\"",_msg="x"}`)
// LogsQL filter
f(`foobar`, `foobar`)
f(`foo:bar`, `foo:bar`)
f(`foo:(bar or baz) error _time:5m {"foo"=bar,baz="z"}`, `(foo:bar or foo:baz) error _time:5m {foo="bar",baz="z"}`)
}
func TestParseExtraStreamFilters_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, err := parseExtraStreamFilters(s)
if err == nil {
t.Fatalf("expecting non-nil error")
}
}
// Invalid JSON
f(`{"foo"}`)
f(`[1,2]`)
f(`{"foo":[1]}`)
// Invliad LogsQL filter
f(`foo:(bar`)
// excess pipe
f(`foo | count()`)
}

View File

@@ -1,13 +1,13 @@
{
"files": {
"main.css": "./static/css/main.fa83344e.css",
"main.js": "./static/js/main.8ad2bc1f.js",
"main.css": "./static/css/main.d05122da.css",
"main.js": "./static/js/main.6082e5a5.js",
"static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
"static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.fa83344e.css",
"static/js/main.8ad2bc1f.js"
"static/css/main.d05122da.css",
"static/js/main.6082e5a5.js"
]
}

View File

@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.8ad2bc1f.js"></script><link href="./static/css/main.fa83344e.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore your log data with VictoriaLogs UI"/><link rel="manifest" href="./manifest.json"/><title>UI for VictoriaLogs</title><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaLogs"><meta name="twitter:site" content="@https://victoriametrics.com/products/victorialogs/"><meta name="twitter:description" content="Explore your log data with VictoriaLogs UI"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaLogs"><meta property="og:url" content="https://victoriametrics.com/products/victorialogs/"><meta property="og:description" content="Explore your log data with VictoriaLogs UI"><script defer="defer" src="./static/js/main.6082e5a5.js"></script><link href="./static/css/main.d05122da.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -30,7 +30,6 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
@@ -98,15 +97,6 @@ var (
)
func main() {
// vmagent is optimized for reduced memory allocations,
// so it can run with the reduced GOGC in order to reduce the used memory,
// while keeping CPU usage spent in GC at low levels.
//
// Some workloads may need increased GOGC values. Then such values can be set via GOGC environment variable.
// It is recommended increasing GOGC if go_memstats_gc_cpu_fraction metric exposed at /metrics page
// exceeds 0.05 for extended periods of time.
cgroup.SetGOGC(30)
// Write flags and help message to stdout, since it is easier to grep or pipe.
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage = usage

View File

@@ -213,7 +213,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
missingRouteRequests.Inc()
var di string
if ui.DumpRequestOnErrors {
di = debugInfo(u, r)
di = debugInfo(u, r.Header)
}
httpserver.Errorf(w, r, "missing route for %q%s", u.String(), di)
return
@@ -668,13 +668,13 @@ func (rtb *readTrackingBody) Close() error {
return nil
}
func debugInfo(u *url.URL, r *http.Request) string {
func debugInfo(u *url.URL, h http.Header) string {
s := &strings.Builder{}
fmt.Fprintf(s, " (host: %q; ", r.Host)
fmt.Fprintf(s, " (host: %q; ", u.Host)
fmt.Fprintf(s, "path: %q; ", u.Path)
fmt.Fprintf(s, "args: %q; ", u.Query().Encode())
fmt.Fprint(s, "headers:")
_ = r.Header.WriteSubset(s, nil)
_ = h.WriteSubset(s, nil)
fmt.Fprint(s, ")")
return s.String()
}

View File

@@ -4,48 +4,16 @@ import (
"fmt"
"net/http"
"github.com/VictoriaMetrics/metrics"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/ratelimiter"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
)
// StartIngestionRateLimiter starts ingestion rate limiter.
//
// Ingestion rate limiter must be started before Init() call.
//
// StopIngestionRateLimiter must be called before Stop() call in order to unblock all the callers
// to ingestion rate limiter. Otherwise deadlock may occur at Stop() call.
func StartIngestionRateLimiter(maxIngestionRate int) {
if maxIngestionRate <= 0 {
return
}
ingestionRateLimitReached := metrics.NewCounter(`vm_max_ingestion_rate_limit_reached_total`)
ingestionRateLimiterStopCh = make(chan struct{})
ingestionRateLimiter = ratelimiter.New(int64(maxIngestionRate), ingestionRateLimitReached, ingestionRateLimiterStopCh)
}
// StopIngestionRateLimiter stops ingestion rate limiter.
func StopIngestionRateLimiter() {
if ingestionRateLimiterStopCh == nil {
return
}
close(ingestionRateLimiterStopCh)
ingestionRateLimiterStopCh = nil
}
var (
ingestionRateLimiter *ratelimiter.RateLimiter
ingestionRateLimiterStopCh chan struct{}
)
// InsertCtx contains common bits for data points insertion.
type InsertCtx struct {
Labels sortedLabels
@@ -105,15 +73,24 @@ func (ctx *InsertCtx) TryPrepareLabels(hasRelabeling bool) bool {
if timeserieslimits.Enabled() && timeserieslimits.IsExceeding(ctx.Labels) {
return false
}
ctx.sortLabelsIfNeeded()
ctx.SortLabelsIfNeeded()
return true
}
// WriteDataPoint writes (timestamp, value) with the given prefix and labels into ctx buffer.
func (ctx *InsertCtx) WriteDataPoint(prefix []byte, hasRelabeling bool, labels []prompbmarshal.Label, timestamp int64, value float64) error {
if !ctx.TryPrepareLabels(hasRelabeling) {
return nil
}
metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
return ctx.addRow(metricNameRaw, timestamp, value)
}
// WriteDataPointUnchecked writes (timestamp, value) with the given prefix and labels into ctx buffer.
//
// caller should invoke TryPrepareLabels before using this function if needed
func (ctx *InsertCtx) WriteDataPoint(prefix []byte, labels []prompbmarshal.Label, timestamp int64, value float64) error {
func (ctx *InsertCtx) WriteDataPointUnchecked(prefix []byte, labels []prompbmarshal.Label, timestamp int64, value float64) error {
metricNameRaw := ctx.marshalMetricNameRaw(prefix, labels)
return ctx.addRow(metricNameRaw, timestamp, value)
}
@@ -204,12 +181,9 @@ func (ctx *InsertCtx) FlushBufs() error {
}
matchIdxsPool.Put(matchIdxs)
}
ingestionRateLimiter.Register(len(ctx.mrs))
// There is no need in limiting the number of concurrent calls to vmstorage.AddRows() here,
// since the number of concurrent FlushBufs() calls should be already limited via writeconcurrencylimiter
// used at every stream.Parse() call under lib/protoparser/*
err := vmstorage.AddRows(ctx.mrs)
ctx.Reset(0)
if err == nil {

View File

@@ -12,8 +12,8 @@ var sortLabels = flag.Bool("sortLabels", false, `Whether to sort labels for inco
`For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}. `+
`Enabled sorting for labels can slow down ingestion performance a bit`)
// sortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set
func (ctx *InsertCtx) sortLabelsIfNeeded() {
// SortLabelsIfNeeded sorts labels if -sortLabels command-line flag is set
func (ctx *InsertCtx) SortLabelsIfNeeded() {
if *sortLabels {
sort.Sort(&ctx.Labels)
}

View File

@@ -267,7 +267,7 @@ func pushAggregateSeries(tss []prompbmarshal.TimeSeries) {
ctx.AddLabel(name, label.Value)
}
value := ts.Samples[0].Value
if err := ctx.WriteDataPoint(nil, ctx.Labels, currentTimestamp, value); err != nil {
if err := ctx.WriteDataPointUnchecked(nil, ctx.Labels, currentTimestamp, value); err != nil {
logger.Errorf("cannot store aggregate series: %s", err)
// Do not continue pushing the remaining samples, since it is likely they will return the same error.
return

View File

@@ -46,10 +46,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
label := &extraLabels[j]
ctx.AddLabel(label.Name, label.Value)
}
if !ctx.TryPrepareLabels(hasRelabeling) {
continue
}
if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
if err := ctx.WriteDataPoint(nil, hasRelabeling, ctx.Labels, r.Timestamp, r.Value); err != nil {
return err
}
}

View File

@@ -36,10 +36,7 @@ func insertRows(rows []parser.Row) error {
tag := &r.Tags[j]
ctx.AddLabel(tag.Key, tag.Value)
}
if !ctx.TryPrepareLabels(hasRelabeling) {
continue
}
if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
if err := ctx.WriteDataPoint(nil, hasRelabeling, ctx.Labels, r.Timestamp, r.Value); err != nil {
return err
}
}

View File

@@ -14,7 +14,6 @@ import (
parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/influx/stream"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeserieslimits"
"github.com/VictoriaMetrics/metrics"
)
@@ -70,7 +69,6 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
ic.Reset(rowsLen)
rowsTotal := 0
hasRelabeling := relabel.HasRelabeling()
hasLimitsEnabled := timeserieslimits.Enabled()
for i := range rows {
r := &rows[i]
rowsTotal += len(r.Fields)
@@ -110,17 +108,12 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
ic.Labels = append(ic.Labels[:0], ctx.originLabels...)
ic.AddLabel("", metricGroup)
if !ic.TryPrepareLabels(true) {
continue
}
if err := ic.WriteDataPoint(nil, ic.Labels, r.Timestamp, f.Value); err != nil {
if err := ic.WriteDataPoint(nil, true, ic.Labels, r.Timestamp, f.Value); err != nil {
return err
}
}
} else {
if !ic.TryPrepareLabels(false) {
continue
}
ic.SortLabelsIfNeeded()
ctx.metricNameBuf = storage.MarshalMetricNameRaw(ctx.metricNameBuf[:0], ic.Labels)
labelsLen := len(ic.Labels)
for j := range r.Fields {
@@ -131,14 +124,7 @@ func insertRows(db string, rows []parser.Row, extraLabels []prompbmarshal.Label)
metricGroup := bytesutil.ToUnsafeString(ctx.metricGroupBuf)
ic.Labels = ic.Labels[:labelsLen]
ic.AddLabel("", metricGroup)
if hasLimitsEnabled {
// special case for optimisation above
// check only __name__ label value limits
if timeserieslimits.IsExceeding(ic.Labels[len(ic.Labels)-1:]) {
continue
}
}
if err := ic.WriteDataPoint(ctx.metricNameBuf, ic.Labels[len(ic.Labels)-1:], r.Timestamp, f.Value); err != nil {
if err := ic.WriteDataPoint(ctx.metricNameBuf, false, ic.Labels[len(ic.Labels)-1:], r.Timestamp, f.Value); err != nil {
return err
}
}

View File

@@ -68,7 +68,7 @@ func insertRows(block *stream.Block, extraLabels []prompbmarshal.Label) error {
timestamp := timestamps[j]
// TODO: @f41gh7 looks like it's better to use WriteDataPointExt
// since metricName never changes inside insertRows call
if err := ic.WriteDataPoint(ctx.metricNameBuf, ic.Labels, timestamp, value); err != nil {
if err := ic.WriteDataPointUnchecked(ctx.metricNameBuf, ic.Labels, timestamp, value); err != nil {
return err
}
}

View File

@@ -58,10 +58,7 @@ func insertRows(rows []newrelic.Row, extraLabels []prompbmarshal.Label) error {
label := &extraLabels[k]
ctx.AddLabel(label.Name, label.Value)
}
if !ctx.TryPrepareLabels(hasRelabeling) {
continue
}
if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, s.Value); err != nil {
if err := ctx.WriteDataPoint(nil, hasRelabeling, ctx.Labels, r.Timestamp, s.Value); err != nil {
return err
}
}

View File

@@ -36,10 +36,7 @@ func insertRows(rows []parser.Row) error {
tag := &r.Tags[j]
ctx.AddLabel(tag.Key, tag.Value)
}
if !ctx.TryPrepareLabels(hasRelabeling) {
continue
}
if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
if err := ctx.WriteDataPoint(nil, hasRelabeling, ctx.Labels, r.Timestamp, r.Value); err != nil {
return err
}
}

View File

@@ -54,10 +54,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
label := &extraLabels[j]
ctx.AddLabel(label.Name, label.Value)
}
if !ctx.TryPrepareLabels(hasRelabeling) {
continue
}
if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
if err := ctx.WriteDataPoint(nil, hasRelabeling, ctx.Labels, r.Timestamp, r.Value); err != nil {
return err
}
}

View File

@@ -54,10 +54,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
label := &extraLabels[j]
ctx.AddLabel(label.Name, label.Value)
}
if !ctx.TryPrepareLabels(hasRelabeling) {
continue
}
if err := ctx.WriteDataPoint(nil, ctx.Labels, r.Timestamp, r.Value); err != nil {
if err := ctx.WriteDataPoint(nil, hasRelabeling, ctx.Labels, r.Timestamp, r.Value); err != nil {
return err
}
}

View File

@@ -69,7 +69,7 @@ func insertRows(rows []parser.Row, extraLabels []prompbmarshal.Label) error {
}
for j, value := range values {
timestamp := timestamps[j]
if err := ic.WriteDataPoint(ctx.metricNameBuf, nil, timestamp, value); err != nil {
if err := ic.WriteDataPointUnchecked(ctx.metricNameBuf, nil, timestamp, value); err != nil {
return err
}
}

View File

@@ -699,13 +699,8 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
// Extend dstValues in order to remove mallocs below.
dstValues = decimal.ExtendFloat64sCapacity(dstValues, len(rc.Timestamps))
// Use step as the scrape interval for instant queries (when start == end).
maxPrevInterval := rc.Step
if rc.Start < rc.End {
scrapeInterval := getScrapeInterval(timestamps, rc.Step)
maxPrevInterval = getMaxPrevInterval(scrapeInterval)
}
scrapeInterval := getScrapeInterval(timestamps, rc.Step)
maxPrevInterval := getMaxPrevInterval(scrapeInterval)
if rc.LookbackDelta > 0 && maxPrevInterval > rc.LookbackDelta {
maxPrevInterval = rc.LookbackDelta
}

View File

@@ -1,13 +1,13 @@
{
"files": {
"main.css": "./static/css/main.876c56b7.css",
"main.js": "./static/js/main.caf36c39.js",
"main.js": "./static/js/main.59602c15.js",
"static/js/685.f772060c.chunk.js": "./static/js/685.f772060c.chunk.js",
"static/media/MetricsQL.md": "./static/media/MetricsQL.a00044c91d9781cf8557.md",
"index.html": "./index.html"
},
"entrypoints": [
"static/css/main.876c56b7.css",
"static/js/main.caf36c39.js"
"static/js/main.59602c15.js"
]
}

View File

@@ -1 +1 @@
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.caf36c39.js"></script><link href="./static/css/main.876c56b7.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>
<!doctype html><html lang="en"><head><meta charset="utf-8"/><link rel="icon" href="./favicon.svg"/><link rel="apple-touch-icon" href="./favicon.svg"/><link rel="mask-icon" href="./favicon.svg" color="#000000"><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=5"/><meta name="theme-color" content="#000000"/><meta name="description" content="Explore and troubleshoot your VictoriaMetrics data"/><link rel="manifest" href="./manifest.json"/><title>vmui</title><script src="./dashboards/index.js" type="module"></script><meta name="twitter:card" content="summary"><meta name="twitter:title" content="UI for VictoriaMetrics"><meta name="twitter:site" content="@https://victoriametrics.com/"><meta name="twitter:description" content="Explore and troubleshoot your VictoriaMetrics data"><meta name="twitter:image" content="./preview.jpg"><meta property="og:type" content="website"><meta property="og:title" content="UI for VictoriaMetrics"><meta property="og:url" content="https://victoriametrics.com/"><meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data"><script defer="defer" src="./static/js/main.59602c15.js"></script><link href="./static/css/main.876c56b7.css" rel="stylesheet"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -48,11 +48,3 @@ export interface LogHits {
[key: string]: string;
};
}
export interface ReportMetaData {
id: number;
title: string;
endpoint: string;
comment: string;
params: Record<string, string>;
}

View File

@@ -43,8 +43,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
const { isMobile } = useDeviceDetect();
const [openAutocomplete, setOpenAutocomplete] = useState(false);
const [caretPositionAutocomplete, setCaretPositionAutocomplete] = useState<[number, number]>([0, 0]);
const [caretPositionInput, setCaretPositionInput] = useState<[number, number]>([0, 0]);
const [caretPosition, setCaretPosition] = useState<[number, number]>([0, 0]);
const autocompleteAnchorEl = useRef<HTMLInputElement>(null);
const [showAutocomplete, setShowAutocomplete] = useState(autocomplete);
@@ -67,7 +66,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
const handleSelect = (val: string, caretPosition: number) => {
onChange(val);
setCaretPositionInput([caretPosition, caretPosition]);
setCaretPosition([caretPosition, caretPosition]);
};
const handleKeyDown = (e: KeyboardEvent) => {
@@ -109,7 +108,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
};
const handleChangeCaret = (val: [number, number]) => {
setCaretPositionAutocomplete(prev => prev[0] === val[0] && prev[1] === val[1] ? prev : val);
setCaretPosition(prev => prev[0] === val[0] && prev[1] === val[1] ? prev : val);
};
useEffect(() => {
@@ -119,7 +118,7 @@ const QueryEditor: FC<QueryEditorProps> = ({
useEffect(() => {
setShowAutocomplete(false);
debouncedSetShowAutocomplete(true);
}, [caretPositionAutocomplete]);
}, [caretPosition]);
return (
<div
@@ -138,13 +137,13 @@ const QueryEditor: FC<QueryEditorProps> = ({
onChangeCaret={handleChangeCaret}
disabled={disabled}
inputmode={"search"}
caretPosition={caretPositionInput}
caretPosition={caretPosition}
/>
{showAutocomplete && autocomplete && (
<QueryEditorAutocomplete
value={value}
anchorEl={autocompleteAnchorEl}
caretPosition={caretPositionAutocomplete}
caretPosition={caretPosition}
hasHelperText={Boolean(warning || error)}
includeFunctions={includeFunctions}
onSelect={handleSelect}

View File

@@ -38,10 +38,6 @@
align-items: flex-start;
gap: $padding-small;
ul {
list-style-position: inside;
}
button {
color: inherit;
min-height: 29px;

View File

@@ -570,14 +570,3 @@ export const SpinnerIcon = () => (
</path>
</svg>
);
export const CommentIcon = () => (
<svg
viewBox="0 0 24 24"
fill="currentColor"
>
<path
d="M21.99 4c0-1.1-.89-2-1.99-2H4c-1.1 0-2 .9-2 2v12c0 1.1.9 2 2 2h14l4 4zM18 14H6v-2h12zm0-3H6V9h12zm0-3H6V6h12z"
></path>
</svg>
);

View File

@@ -1,62 +0,0 @@
import React, { FC } from "preact/compat";
import useBoolean from "../../../hooks/useBoolean";
import classNames from "classnames";
import TextField from "../TextField/TextField";
import "./style.scss";
import { marked } from "marked";
interface Props {
value: string;
onChange: (value: string) => void;
}
const tabs = [
{ title: "Write", value: false },
{ title: "Preview", value: true },
];
const MarkdownEditor: FC<Props> = ({ value, onChange }) => {
const {
value: markdownPreview,
setTrue: setMarkdownPreviewTrue,
setFalse: setMarkdownPreviewFalse,
} = useBoolean(false);
return (
<div className="vm-markdown-editor">
<div className="vm-markdown-editor-header">
<div className="vm-markdown-editor-header-tabs">
{tabs.map(({ title, value }) => (
<div
key={title}
className={classNames({
"vm-markdown-editor-header-tabs__tab": true,
"vm-markdown-editor-header-tabs__tab_active": markdownPreview === value,
})}
onClick={value ? setMarkdownPreviewTrue : setMarkdownPreviewFalse}
>
{title}
</div>
))}
</div>
<span className="vm-markdown-editor-header__info">
Markdown is supported
</span>
</div>
{markdownPreview ? (
<div
className="vm-markdown-editor-preview vm-markdown"
dangerouslySetInnerHTML={{ __html: marked(value) as string }}
/>
) : (
<TextField
type="textarea"
value={value}
onChange={onChange}
/>
)}
</div>
);
};
export default MarkdownEditor;

View File

@@ -1,75 +0,0 @@
@use "src/styles/variables" as *;
.vm-markdown-editor {
margin-top: 6px;
padding: 0 6px;
border-radius: $border-radius-small;
border: $border-divider;
overflow: hidden;
&-header {
display: flex;
align-items: center;
background-color: $color-hover-black;
padding-right: $padding-global;
border-bottom: $border-divider;
margin: -1px -7px 6px;
&-tabs {
display: flex;
&__tab {
display: flex;
align-items: center;
justify-content: center;
margin-bottom: -1px;
padding: $padding-small $padding-large;
min-height: 40px;
color: $color-text-secondary;
transition: color 0.3s;
cursor: pointer;
&:hover {
color: $color-text;
}
&_active {
position: relative;
color: $color-text;
background-color: $color-background-body;
border-top-right-radius: $border-radius-small;
border-top-left-radius: $border-radius-small;
z-index: 1;
&:first-child {
border-right: $border-divider;
}
&:last-child {
border-right: $border-divider;
border-left: $border-divider;
}
}
}
}
&__info {
margin-left: auto;
margin-right: 0;
color: $color-text-secondary;
font-size: $font-size-small;
font-weight: 500;
}
}
&-preview {
padding: $padding-small;
margin-bottom: 6px;
}
&-preview,
textarea {
min-height: 200px;
resize: vertical;
}
}

View File

@@ -1,6 +1,7 @@
import React, {
FC,
useEffect,
useState,
useRef,
useMemo,
FormEvent,
@@ -64,6 +65,7 @@ const TextField: FC<TextFieldProps> = ({
const inputRef = useRef<HTMLInputElement>(null);
const textareaRef = useRef<HTMLTextAreaElement>(null);
const fieldRef = useMemo(() => type === "textarea" ? textareaRef : inputRef, [type]);
const [selectionPos, setSelectionPos] = useState<[start: number, end: number]>([0, 0]);
const inputClasses = classNames({
"vm-text-field__input": true,
@@ -75,9 +77,8 @@ const TextField: FC<TextFieldProps> = ({
});
const updateCaretPosition = (target: HTMLInputElement | HTMLTextAreaElement) => {
if (!onChangeCaret) return;
const { selectionStart, selectionEnd } = target;
onChangeCaret && onChangeCaret([selectionStart || 0, selectionEnd || 0]);
setSelectionPos([selectionStart || 0, selectionEnd || 0]);
};
const handleMouseUp = (e: MouseEvent<HTMLInputElement | HTMLTextAreaElement>) => {
@@ -126,6 +127,14 @@ const TextField: FC<TextFieldProps> = ({
fieldRef?.current?.focus && fieldRef.current.focus();
}, [fieldRef, autofocus]);
useEffect(() => {
onChangeCaret && onChangeCaret(selectionPos);
}, [selectionPos]);
useEffect(() => {
setSelectionRange(selectionPos);
}, [value]);
useEffect(() => {
caretPosition && setSelectionRange(caretPosition);
}, [caretPosition]);

View File

@@ -16,20 +16,17 @@ const UploadJsonButtons: FC<Props> = ({ onOpenModal, onChange }) => (
>
Paste JSON
</Button>
<div className="vm-upload-json-buttons__upload">
<Button>
Upload Files
</Button>
<Button>
Upload Files
<input
id="json"
name="json"
type="file"
accept="application/json"
multiple
title=" "
onChange={onChange}
/>
</div>
</Button>
</div>
);

View File

@@ -6,8 +6,4 @@
gap: $padding-global;
align-items: center;
justify-content: center;
&__upload {
position: relative;
}
}

View File

@@ -1,4 +1,4 @@
import React, { FC, useCallback, useEffect, useRef, useState } from "preact/compat";
import React, { FC, useCallback, useEffect, useMemo, useRef, useState } from "preact/compat";
import { DownloadIcon } from "../../../components/Main/Icons";
import Button from "../../../components/Main/Button/Button";
import Tooltip from "../../../components/Main/Tooltip/Tooltip";
@@ -12,65 +12,32 @@ import TextField from "../../../components/Main/TextField/TextField";
import { useQueryState } from "../../../state/query/QueryStateContext";
import { ErrorTypes } from "../../../types";
import Alert from "../../../components/Main/Alert/Alert";
import qs from "qs";
import Popper from "../../../components/Main/Popper/Popper";
import helperText from "./helperText";
import { Link } from "react-router-dom";
import router from "../../../router";
import { parseLineToJSON } from "../../../utils/json";
import { ExportMetricResult, ReportMetaData } from "../../../api/types";
import { getApiEndpoint } from "../../../utils/url";
import MarkdownEditor from "../../../components/Main/MarkdownEditor/MarkdownEditor";
export enum ReportType {
QUERY_DATA,
RAW_DATA,
}
type Props = {
fetchUrl?: string[];
reportType?: ReportType
}
type MetaData = {
id: number;
url: URL;
title: string;
comment: string;
}
const getDefaultReportName = () => `vmui_report_${dayjs().utc().format(DATE_FILENAME_FORMAT)}`;
const getDefaultTitle = (type: ReportType) => {
switch (type) {
case ReportType.RAW_DATA:
return "Raw report";
default:
return "Report";
}
};
const getDefaultFilename = (title: string) => {
const timestamp = dayjs().utc().format(DATE_FILENAME_FORMAT);
return `vmui_${title.toLowerCase().replace(/ /g, "_")}_${timestamp}`;
};
const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DATA }) => {
const DownloadReport: FC<Props> = ({ fetchUrl }) => {
const { query } = useQueryState();
const defaultTitle = getDefaultTitle(reportType);
const defaultFilename = getDefaultFilename(defaultTitle);
const [title, setTitle] = useState(defaultTitle);
const [filename, setFilename] = useState(defaultFilename);
const [filename, setFilename] = useState(getDefaultReportName());
const [comment, setComment] = useState("");
const [trace, setTrace] = useState(reportType === ReportType.QUERY_DATA);
const [trace, setTrace] = useState(true);
const [error, setError] = useState<ErrorTypes | string>();
const [isLoading, setIsLoading] = useState(false);
const titleRef = useRef<HTMLDivElement>(null);
const filenameRef = useRef<HTMLDivElement>(null);
const commentRef = useRef<HTMLDivElement>(null);
const traceRef = useRef<HTMLDivElement>(null);
const generateRef = useRef<HTMLDivElement>(null);
const helperRefs = [filenameRef, titleRef, commentRef, traceRef, generateRef];
const helperRefs = [filenameRef, commentRef, traceRef, generateRef];
const [stepHelper, setStepHelper] = useState(0);
const {
@@ -85,17 +52,13 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
setFalse: handleCloseHelper,
} = useBoolean(false);
const getFetchUrlReport = useCallback(() => {
const fetchUrlReport = useMemo(() => {
if (!fetchUrl) return;
try {
return fetchUrl.map((str, i) => {
const url = new URL(str);
trace ? url.searchParams.set("trace", "1") : url.searchParams.delete("trace");
return { id: i, url: url };
});
} catch (e) {
setError(String(e));
}
return fetchUrl.map((str, i) => {
const url = new URL(str);
trace ? url.searchParams.set("trace", "1") : url.searchParams.delete("trace");
return { id: i, url: url };
});
}, [fetchUrl, trace]);
const generateFile = useCallback((data: unknown) => {
@@ -105,7 +68,7 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
const link = document.createElement("a");
link.href = href;
link.download = `${filename || defaultFilename}.json`;
link.download = `${filename || getDefaultReportName()}.json`;
document.body.appendChild(link);
link.click();
@@ -114,63 +77,9 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
handleClose();
}, [filename]);
const getMetaData = ({ id, url, comment, title }: MetaData): ReportMetaData => {
return {
id,
title: title || defaultTitle,
comment,
endpoint: getApiEndpoint(url.pathname) || "",
params: Object.fromEntries(url.searchParams)
};
};
const processJsonLineResponse = async (response: Response, metaData: MetaData) => {
const result: { metric: { [p: string]: string }, values: number[][] }[] = [];
const text = await response.text();
if (response.ok) {
const lines = text.split("\n").filter(line => line);
lines.forEach((line: string) => {
const jsonLine = parseLineToJSON(line) as (ExportMetricResult | null);
if (!jsonLine) return;
result.push({
metric: jsonLine.metric,
values: jsonLine.values.map((value, index) => [(jsonLine.timestamps[index] / 1000), value]),
});
});
} else {
setError(String(text));
}
return { data: { result, resultType: "matrix" }, vmui: getMetaData(metaData) };
};
const processJsonResponse = async (response: Response, metaData: MetaData) => {
const resp = await response.json();
if (response.ok) {
resp.vmui = getMetaData(metaData);
return resp;
} else {
const errorType = resp.errorType ? `${resp.errorType}\r\n` : "";
setError(`${errorType}${resp?.error || resp?.message || "unknown error"}`);
}
};
const processResponse = async (response: Response, metaData: MetaData) => {
switch (reportType) {
case ReportType.RAW_DATA:
return await processJsonLineResponse(response, metaData);
default:
return await processJsonResponse(response, metaData);
}
};
const handleGenerateReport = useCallback(async () => {
const fetchUrlReport = getFetchUrlReport();
if (!fetchUrlReport) {
setError(prev => !prev ? ErrorTypes.validQuery : prev);
setError(ErrorTypes.validQuery);
return;
}
@@ -179,12 +88,20 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
try {
const result = [];
for await (const fetchOps of fetchUrlReport) {
if (!fetchOps) continue;
const { url, id } = fetchOps;
for await (const { url, id } of fetchUrlReport) {
const response = await fetch(url);
const data = await processResponse(response, { id, url, comment, title });
result.push(data);
const resp = await response.json();
if (response.ok) {
resp.vmui = {
id,
comment,
params: qs.parse(new URL(url).search.replace(/^\?/, ""))
};
result.push(resp);
} else {
const errorType = resp.errorType ? `${resp.errorType}\r\n` : "";
setError(`${errorType}${resp?.error || resp?.message || "unknown error"}`);
}
}
result.length && generateFile(result);
} catch (e) {
@@ -194,20 +111,15 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
} finally {
setIsLoading(false);
}
}, [getFetchUrlReport, comment, generateFile, query, title]);
}, [fetchUrlReport, comment, generateFile, query]);
const handleChangeHelp = (step: number) => () => {
const findNextRef = (index: number): number => {
const nextIndex = index + step;
if (helperRefs[nextIndex]?.current) return nextIndex;
return findNextRef(nextIndex);
};
setStepHelper(findNextRef);
setStepHelper(prevStep => prevStep + step);
};
useEffect(() => {
setError("");
setFilename(defaultFilename);
setFilename(getDefaultReportName());
setComment("");
}, [openModal]);
@@ -243,41 +155,31 @@ const DownloadReport: FC<Props> = ({ fetchUrl, reportType = ReportType.QUERY_DAT
<div className="vm-download-report">
<div className="vm-download-report-settings">
<div ref={filenameRef}>
<div className="vm-download-report-settings__title">Filename</div>
<TextField
label="Filename"
value={filename}
onChange={setFilename}
/>
</div>
<div ref={titleRef}>
<div className="vm-download-report-settings__title">Report title</div>
<TextField
value={title}
onChange={setTitle}
/>
</div>
<div ref={commentRef}>
<div className="vm-download-report-settings__title">Comment</div>
<MarkdownEditor
<TextField
type="textarea"
label="Comment"
value={comment}
onChange={setComment}
/>
</div>
{reportType === ReportType.QUERY_DATA && (
<>
<div ref={traceRef}>
<Checkbox
checked={trace}
onChange={setTrace}
label={"Include query trace"}
/>
</div>
<Alert variant="info">
If confused with the query results,
try viewing the raw samples for selected series in <RawQueryLink/> tab.
</Alert>
</>
)}
<div ref={traceRef}>
<Checkbox
checked={trace}
onChange={setTrace}
label={"Include query trace"}
/>
</div>
<Alert variant="info">
If confused with the query results,
try viewing the raw samples for selected series in <RawQueryLink/> tab.
</Alert>
</div>
{error && <Alert variant="error">{error}</Alert>}
<div className="vm-download-report__buttons">

View File

@@ -11,18 +11,6 @@ const filename = (
</>
);
const tittle = (
<>
<p>Title - specify the title that will be displayed on the <Link
to={router.queryAnalyzer}
target="_blank"
rel="noreferrer"
className="vm-link vm-link_underlined"
>{routerOptions[router.queryAnalyzer].title}</Link> page.</p>
<p>This helps identify your report in the interface.</p>
</>
);
const comment = (
<>
<p>Comment (optional) - add a comment to your report.</p>
@@ -51,7 +39,6 @@ const generate = (
export default [
filename,
tittle,
comment,
trace,
generate,

View File

@@ -9,15 +9,10 @@
&-settings {
display: grid;
gap: $padding-large;
gap: $padding-global;
&__title {
display: flex;
align-items: center;
margin-right: $padding-global;
font-size: $font-size;
font-weight: 600;
white-space: nowrap;
textarea {
min-height: 200px;
}
}
@@ -39,7 +34,7 @@
line-height: 1.3;
p {
margin-bottom: calc($padding-small / 2);
margin-bottom: calc($padding-small/2);
}
}

View File

@@ -1,20 +1,13 @@
import React, { FC, useMemo } from "preact/compat";
import { DataAnalyzerType } from "../index";
import {
ClockIcon,
CommentIcon,
InfoIcon,
TimelineIcon
} from "../../../components/Main/Icons";
import Button from "../../../components/Main/Button/Button";
import { ClockIcon, InfoIcon, TimelineIcon } from "../../../components/Main/Icons";
import useBoolean from "../../../hooks/useBoolean";
import Modal from "../../../components/Main/Modal/Modal";
import { TimeParams } from "../../../types";
import "./style.scss";
import dayjs from "dayjs";
import { DATE_TIME_FORMAT } from "../../../constants/date";
import useBoolean from "../../../hooks/useBoolean";
import Modal from "../../../components/Main/Modal/Modal";
import { marked } from "marked";
import Button from "../../../components/Main/Button/Button";
import get from "lodash.get";
type Props = {
data: DataAnalyzerType[];
@@ -22,23 +15,8 @@ type Props = {
}
const QueryAnalyzerInfo: FC<Props> = ({ data, period }) => {
const dataWithStats = useMemo(() => data.filter(d => d.vmui || d.stats), [data]);
const title = dataWithStats.find(d => d?.vmui?.title)?.vmui?.title || "Report";
const comment = dataWithStats.find(d => d?.vmui?.comment)?.vmui?.comment;
const table = useMemo(() => {
return [
"vmui.endpoint",
...new Set(dataWithStats.flatMap(d => [
...Object.keys(d.vmui?.params || []).map(key => `vmui.params.${key}`),
...Object.keys(d.stats || []).map(key => `stats.${key}`),
"isPartial"
]))
].map(key => ({
column: key.split(".").pop(),
values: dataWithStats.map(data => get(data, key, "-"))
})).filter(({ values }) => values.length && values.every(v => v !== "-"));
}, [dataWithStats]);
const dataWithStats = useMemo(() => data.filter(d => d.stats && d.data.resultType === "matrix"), [data]);
const comment = useMemo(() => data.find(d => d?.vmui?.comment)?.vmui?.comment, [data]);
const timeRange = useMemo(() => {
if (!period) return "";
@@ -56,80 +34,59 @@ const QueryAnalyzerInfo: FC<Props> = ({ data, period }) => {
return (
<>
<div className="vm-query-analyzer-info-header">
<h1 className="vm-query-analyzer-info-header__title">{title}</h1>
{timeRange && (
<div className="vm-query-analyzer-info-header__timerange">
<ClockIcon/> {timeRange}
</div>
)}
{period?.step && (
<div className="vm-query-analyzer-info-header__timerange">
<TimelineIcon/> step {period.step}
</div>
)}
{(comment || !!table.length) && (
<div className="vm-query-analyzer-info-header__info">
<Button
startIcon={<InfoIcon/>}
variant="outlined"
color="warning"
onClick={handleOpenModal}
>
Show stats{comment && " & comments"}
</Button>
</div>
<Button
startIcon={<InfoIcon/>}
variant="outlined"
color="warning"
onClick={handleOpenModal}
>
Show report info
</Button>
{period && (
<>
<div className="vm-query-analyzer-info-header__period">
<TimelineIcon/> step: {period.step}
</div>
<div className="vm-query-analyzer-info-header__period">
<ClockIcon/> {timeRange}
</div>
</>
)}
</div>
{openModal && (
<Modal
title={title}
title="Report info"
onClose={handleCloseModal}
>
<div className="vm-query-analyzer-info__modal">
{!!table.length && (
<div className="vm-query-analyzer-info-stats">
<div className="vm-query-analyzer-info-comment-header">
<InfoIcon/>
Stats
</div>
<table>
<thead>
<tr>
{table.map(({ column }) => (
<th key={column}>
{column}
</th>
))}
</tr>
</thead>
<tbody>
{table[0]?.values.map((_, rowIndex) => (
<tr key={rowIndex}>
{table.map(({ values }, j) => (
<td key={j}>
{values[rowIndex]}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
)}
<div className="vm-query-analyzer-info">
{comment && (
<div className="vm-query-analyzer-info-comment">
<div className="vm-query-analyzer-info-comment-header">
<CommentIcon/>
Comments
</div>
<div
className="vm-query-analyzer-info-comment-body vm-markdown"
dangerouslySetInnerHTML={{ __html: (marked(comment) as string) || comment }}
/>
<div className="vm-query-analyzer-info-item vm-query-analyzer-info-item_comment">
<div className="vm-query-analyzer-info-item__title">Comment:</div>
<div className="vm-query-analyzer-info-item__text">{comment}</div>
</div>
)}
{dataWithStats.map((d, i) => (
<div
className="vm-query-analyzer-info-item"
key={i}
>
<div className="vm-query-analyzer-info-item__title">
{dataWithStats.length > 1 ? `Query ${i + 1}:` : "Stats:"}
</div>
<div className="vm-query-analyzer-info-item__text">
{Object.entries(d.stats || {}).map(([key, value]) => (
<div key={key}>
{key}: {value ?? "-"}
</div>
))}
isPartial: {String(d.isPartial ?? "-")}
</div>
</div>
))}
<div className="vm-query-analyzer-info-type">
{dataWithStats[0]?.vmui?.params ? "The report was created using vmui" : "The report was created manually"}
</div>
</div>
</Modal>
)}

View File

@@ -1,115 +1,47 @@
@use "src/styles/variables" as *;
.vm-query-analyzer-info {
.vm-query-analyzer-info-header {
display: flex;
gap: $padding-global;
&-header {
&__period {
display: flex;
align-items: center;
width: 100%;
height: 100%;
gap: $padding-small;
font-size: $font-size-small;
background-color: $color-background-body;
z-index: 1;
&__title {
font-size: $font-size-large;
font-weight: 500;
}
&__timerange {
display: flex;
align-items: center;
gap: calc($padding-small / 2);
border: $border-divider;
border-radius: $border-radius-small;
padding: calc($padding-small / 2) $padding-small;
font-size: $font-size-small;
svg {
width: calc($font-size-small + 1px);
color: $color-primary;
}
}
&__info {
margin-left: auto;
margin-right: 0;
}
}
&__modal {
width: min(800px, 90vw);
}
&-comment {
position: relative;
max-width: 800px;
border-radius: $border-radius-medium;
border: $border-divider;
font-size: $font-size-small;
border-radius: $border-radius-small;
padding: 6px $padding-global;
&-header {
display: grid;
grid-template-columns: 16px 1fr;
align-items: center;
gap: $padding-small;
padding: $padding-small;
border-bottom: $border-divider;
background-color: $color-hover-black;
font-weight: 500;
z-index: 1;
svg {
color: $color-primary;
}
}
&-body {
padding: $padding-small;
max-height: 60vh;
overflow: auto;
}
}
&-stats {
border-radius: $border-radius-medium;
border: $border-divider;
font-size: $font-size-small;
margin-bottom: $padding-global;
overflow: hidden;
table {
width: 100%;
}
td, th {
padding: $padding-small;
text-align: left;
}
tr {
border-bottom: $border-divider;
}
thead {
th {
font-weight: 500;
}
}
tbody {
tr {
transition: background-color 0.3s;
&:hover {
background-color: $color-hover-black;
}
&:last-child {
border-bottom: none;
}
}
svg {
width: calc($font-size-small + 1px);
color: $color-primary;
}
}
}
.vm-query-analyzer-info {
display: grid;
gap: $padding-large;
min-width: 300px;
&-type {
text-align: center;
font-style: italic;
color: $color-text-secondary;
}
&-item {
display: grid;
padding-bottom: $padding-large;
border-bottom: $border-divider;
line-height: 130%;
&__title {
font-weight: bold;
}
&__text {
white-space: pre-wrap;
}
}
}

View File

@@ -9,9 +9,8 @@ import useBoolean from "../../hooks/useBoolean";
import UploadJsonButtons from "../../components/UploadJsonButtons/UploadJsonButtons";
import JsonForm from "./JsonForm/JsonForm";
import "../TracePage/style.scss";
import "./style.scss";
import QueryAnalyzerView from "./QueryAnalyzerView/QueryAnalyzerView";
import { InstantMetricResult, MetricResult, ReportMetaData, TracingData } from "../../api/types";
import { InstantMetricResult, MetricResult, TracingData } from "../../api/types";
import QueryAnalyzerInfo from "./QueryAnalyzerInfo/QueryAnalyzerInfo";
import { TimeParams } from "../../types";
import { dateFromSeconds, formatDateToUTC, humanizeSeconds } from "../../utils/time";
@@ -22,8 +21,15 @@ export type DataAnalyzerType = {
resultType: "vector" | "matrix";
result: MetricResult[] | InstantMetricResult[]
};
stats?: Record<string, string>;
vmui?: ReportMetaData;
stats?: {
seriesFetched?: string;
executionTimeMsec?: number
};
vmui?: {
id: number;
comment: string;
params: Record<string, string>;
};
status: string;
trace?: TracingData;
isPartial?: boolean;
@@ -86,12 +92,10 @@ const QueryAnalyzer: FC = () => {
setData(response);
} else {
setError("Invalid structure - JSON does not match the expected format");
setData([]);
}
} catch (e) {
if (e instanceof Error) {
setError(`${e.name}: ${e.message}`);
setData([]);
}
}
};
@@ -125,16 +129,33 @@ const QueryAnalyzer: FC = () => {
}, [files]);
return (
<div className="vm-query-analyzer">
<div className="vm-trace-page">
{hasData && (
<div className="vm-query-analyzer-header">
<QueryAnalyzerInfo
data={data}
period={period}
/>
<UploadJsonButtons
onOpenModal={handleOpenModal}
onChange={handleChange}
<div className="vm-trace-page-header">
<div className="vm-trace-page-header-errors">
<QueryAnalyzerInfo
data={data}
period={period}
/>
</div>
<div>
<UploadJsonButtons
onOpenModal={handleOpenModal}
onChange={handleChange}
/>
</div>
</div>
)}
{error && (
<div className="vm-trace-page-header-errors-item vm-trace-page-header-errors-item_margin-bottom">
<Alert variant="error">{error}</Alert>
<Button
className="vm-trace-page-header-errors-item__close"
startIcon={<CloseIcon/>}
variant="text"
color="error"
onClick={handleCloseError}
/>
</div>
)}
@@ -164,19 +185,6 @@ const QueryAnalyzer: FC = () => {
</div>
)}
{error && (
<div className="vm-query-analyzer-error">
<Alert variant="error">{error}</Alert>
<Button
className="vm-query-analyzer-error__close"
startIcon={<CloseIcon/>}
variant="text"
color="error"
onClick={handleCloseError}
/>
</div>
)}
{openModal && (
<Modal
title="Paste JSON"

View File

@@ -1,29 +0,0 @@
@use "src/styles/variables" as *;
.vm-query-analyzer {
display: flex;
flex-direction: column;
@media (max-width: 768px) {
padding: $padding-medium 0;
}
&-header {
display: grid;
grid-template-columns: 1fr auto;
gap: $padding-global;
margin-bottom: $padding-global;
}
&-error {
position: relative;
margin: $padding-global 0;
&__close {
position: absolute;
top: $padding-small;
right: $padding-small;
z-index: 2;
}
}
}

View File

@@ -7,7 +7,6 @@ import { useAppState } from "../../../state/common/StateContext";
import { useCustomPanelState } from "../../../state/customPanel/CustomPanelStateContext";
import { isValidHttpUrl } from "../../../utils/url";
import { getExportDataUrl } from "../../../api/query-range";
import { parseLineToJSON } from "../../../utils/json";
interface FetchQueryParams {
hideQuery?: number[];
@@ -25,6 +24,14 @@ interface FetchQueryReturn {
abortFetch: () => void
}
const parseLineToJSON = (line: string): ExportMetricResult | null => {
try {
return JSON.parse(line);
} catch (e) {
return null;
}
};
export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams): FetchQueryReturn => {
const { query } = useQueryState();
const { period } = useTimeState();
@@ -55,7 +62,7 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
}
}, [serverUrl, period, hideQuery, reduceMemUsage]);
const fetchData = useCallback(async ({ fetchUrl, stateSeriesLimits, showAllSeries }: {
const fetchData = useCallback(async ( { fetchUrl, stateSeriesLimits, showAllSeries }: {
fetchUrl: string[];
stateSeriesLimits: SeriesLimits;
showAllSeries?: boolean;
@@ -92,12 +99,12 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
const lines = text.split("\n").filter(line => line);
const lineLimited = lines.slice(0, freeTempSize).sort();
lineLimited.forEach((line: string) => {
const jsonLine = parseLineToJSON(line) as (ExportMetricResult | null);
const jsonLine = parseLineToJSON(line);
if (!jsonLine) return;
tempData.push({
group: counter,
metric: jsonLine.metric,
values: jsonLine.values.map((value, index) => [(jsonLine.timestamps[index] / 1000), value]),
values: jsonLine.values.map((value, index) => [(jsonLine.timestamps[index]/1000), value]),
} as MetricBase);
});
totalLength += lines.length;
@@ -112,7 +119,7 @@ export const useFetchExport = ({ hideQuery, showAllSeries }: FetchQueryParams):
} catch (e) {
setIsLoading(false);
if (e instanceof Error && e.name !== "AbortError") {
setError(String(e));
setError(error);
console.error(e);
}
}

View File

@@ -17,7 +17,6 @@ import { DisplayType } from "../../types";
import Hyperlink from "../../components/Main/Hyperlink/Hyperlink";
import { CloseIcon } from "../../components/Main/Icons";
import Button from "../../components/Main/Button/Button";
import DownloadReport, { ReportType } from "../CustomPanel/DownloadReport/DownloadReport";
const RawSamplesLink = () => (
<Hyperlink
@@ -66,7 +65,6 @@ const RawQueryPage: FC = () => {
queryErrors,
setQueryErrors,
abortFetch,
fetchUrl,
} = useFetchExport({ hideQuery, showAllSeries });
const controlsRef = useRef<HTMLDivElement>(null);
@@ -108,22 +106,12 @@ const RawQueryPage: FC = () => {
{showPageDescription && (
<Alert variant="info">
<div className="vm-explore-metrics-header-description">
<ul>
<li>
This page provides a dedicated view for querying and displaying <RawSamplesLink/> from VictoriaMetrics.
</li>
<li>
It expects only <TimeSeriesSelectorLink/> as a query argument.
</li>
<li>
Deduplication can only be disabled if it was previously enabled on the server
(<code>-dedup.minScrapeInterval</code>).
</li>
<li>
Users often assume that the <QueryDataLink/> returns data exactly as stored,
but data samples and timestamps may be modified by the API.
</li>
</ul>
<p>
This page provides a dedicated view for querying and displaying <RawSamplesLink/> from VictoriaMetrics.
It expects only <TimeSeriesSelectorLink/> as a query argument.
Users often assume that the <QueryDataLink/> returns data exactly as stored,
but data samples and timestamps may be modified by the API.
</p>
<Button
variant="text"
size="small"
@@ -158,12 +146,6 @@ const RawQueryPage: FC = () => {
<div className="vm-custom-panel-body-header__tabs">
<DisplayTypeSwitch tabFilter={(tab) => (tab.value !== DisplayType.table)}/>
</div>
{data && (
<DownloadReport
fetchUrl={fetchUrl}
reportType={ReportType.RAW_DATA}
/>
)}
</div>
<CustomPanelTabs
graphData={data}

View File

@@ -61,8 +61,7 @@
display: flex;
flex-direction: column;
align-items: center;
justify-content: flex-end;
min-height: calc(($vh * 50) - var(--scrollbar-height));
justify-content: center;
&__text {
margin-bottom: $padding-global;

View File

@@ -1,139 +0,0 @@
@use "src/styles/variables" as *;
.vm-markdown {
display: block;
line-height: 1.5;
pre,
code {
font-size: 1em;
font-family: $font-family-monospace
}
pre {
padding: .5em;
line-height: 1.25;
overflow-x: scroll;
}
a,
a:visited {
color: $color-primary;
text-decoration: underline;
cursor: pointer;
}
body {
line-height: 1.85;
}
p {
font-size: 1em;
margin-bottom: 1.3em;
}
h1, h2, h3, h4, h5, h6 {
margin: .5em 0 0.25em;
font-weight: inherit;
line-height: 1.42;
}
h1 {
font-size: 1.8em;
}
h2 {
font-size: 1.6em;
}
h3 {
font-size: 1.4em;
}
h4 {
font-size: 1.2em;
}
h5 {
font-size: 1em;
}
h6 {
font-size: 0.8em;
}
small {
font-size: 0.8em;
}
img,
canvas,
iframe,
video,
svg,
select,
textarea {
max-width: 100%;
}
pre {
background-color: $color-hover-black;
}
blockquote {
border-left: 3px solid rgba($color-black, 0.2);
padding-left: 1em;
opacity: 0.7;
}
ul, ol {
margin: 0.3em 0;
list-style-position: inside;
li {
margin-bottom: 0.3em;
ul, ol {
margin-left: 1em;
}
}
}
input[type="checkbox"] {
display: none;
}
th,
td {
padding: 0.2em 0.4em;
}
hr {
border-top: $border-divider;
}
strong {
font-weight: bold;
}
em {
font-style: italic;
}
del {
text-decoration: line-through;
}
code:not(pre code) {
display: inline;
vertical-align: middle;
background: $color-hover-black;
border-radius: $border-radius-small;
border: $border-divider;
tab-size: 4;
font-variant-ligatures: none;
padding: 0.12em 0.4em;
font-size: 0.9em;
white-space: nowrap;
}
}

View File

@@ -50,6 +50,10 @@ button {
background: none;
}
strong {
letter-spacing: 1px;
}
input[type='file'] {
opacity: 0;
cursor: pointer;

View File

@@ -10,7 +10,6 @@
@forward "./components/table";
@forward "./components/link";
@forward "./components/dynamic-number";
@forward "./components/markdown";
:root {
/* base palette */

View File

@@ -1,7 +0,0 @@
export const parseLineToJSON = (line: string) => {
try {
return JSON.parse(line);
} catch (e) {
return null;
}
};

View File

@@ -25,13 +25,3 @@ export const isEqualURLSearchParams = (params1: URLSearchParams, params2: URLSea
return true;
};
export const getApiEndpoint = (url: string): string | null => {
try {
const match = url.match(/\/api\/v1\/[^?]+/);
return match ? match[0] : null;
} catch (error) {
console.error("Invalid URL:", error);
return null;
}
};

View File

@@ -128,7 +128,7 @@ func (app *ServesMetrics) GetMetric(t *testing.T, metricName string) float64 {
return res
}
}
t.Fatalf("metric not found: %s", metricName)
t.Fatalf("metic not found: %s", metricName)
return 0
}

View File

@@ -133,7 +133,7 @@ func NewSample(t *testing.T, timeStr string, value float64) *Sample {
if err != nil {
t.Fatalf("could not parse RFC3339 time %q: %v", timeStr, err)
}
return &Sample{parsedTime.UnixMilli(), value}
return &Sample{parsedTime.Unix(), value}
}
// UnmarshalJSON populates the sample fields from a JSON string.
@@ -149,7 +149,7 @@ func (s *Sample) UnmarshalJSON(b []byte) error {
if got, want := len(raw), 2; got != want {
return fmt.Errorf("unexpected number of fields: got %d, want %d (raw sample: %s)", got, want, string(b))
}
s.Timestamp = int64(ts * 1000)
s.Timestamp = int64(ts)
var err error
s.Value, err = strconv.ParseFloat(v, 64)
if err != nil {

View File

@@ -134,21 +134,8 @@ func (c *vmcluster) ForceFlush(t *testing.T) {
}
}
// MustStartDefaultCluster starts a typical cluster configuration with default
// flags.
func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {
tc.t.Helper()
return tc.MustStartCluster(&ClusterOptions{
Vmstorage1Instance: "vmstorage1",
Vmstorage2Instance: "vmstorage2",
VminsertInstance: "vminsert",
VmselectInstance: "vmselect",
})
}
// ClusterOptions holds the params for simple cluster configuration suitable for
// most tests.
// MustStartDefaultCluster is a typical cluster configuration suitable for most
// tests.
//
// The cluster consists of two vmstorages, one vminsert and one vmselect, no
// data replication.
@@ -158,42 +145,23 @@ func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {
// vmselect) but instead just need a typical cluster configuration to verify
// some business logic (such as API surface, or MetricsQL). Such cluster
// tests usually come paired with corresponding vmsingle tests.
type ClusterOptions struct {
Vmstorage1Instance string
Vmstorage1Flags []string
Vmstorage2Instance string
Vmstorage2Flags []string
VminsertInstance string
VminsertFlags []string
VmselectInstance string
VmselectFlags []string
}
// MustStartCluster starts a typical cluster configuration with custom flags.
func (tc *TestCase) MustStartCluster(opts *ClusterOptions) PrometheusWriteQuerier {
func (tc *TestCase) MustStartDefaultCluster() PrometheusWriteQuerier {
tc.t.Helper()
opts.Vmstorage1Flags = append(opts.Vmstorage1Flags, []string{
"-storageDataPath=" + tc.Dir() + "/" + opts.Vmstorage1Instance,
vmstorage1 := tc.MustStartVmstorage("vmstorage-1", []string{
"-storageDataPath=" + tc.Dir() + "/vmstorage-1",
"-retentionPeriod=100y",
}...)
vmstorage1 := tc.MustStartVmstorage(opts.Vmstorage1Instance, opts.Vmstorage1Flags)
opts.Vmstorage2Flags = append(opts.Vmstorage2Flags, []string{
"-storageDataPath=" + tc.Dir() + "/" + opts.Vmstorage2Instance,
})
vmstorage2 := tc.MustStartVmstorage("vmstorage-2", []string{
"-storageDataPath=" + tc.Dir() + "/vmstorage-2",
"-retentionPeriod=100y",
}...)
vmstorage2 := tc.MustStartVmstorage(opts.Vmstorage2Instance, opts.Vmstorage2Flags)
opts.VminsertFlags = append(opts.VminsertFlags, []string{
})
vminsert := tc.MustStartVminsert("vminsert", []string{
"-storageNode=" + vmstorage1.VminsertAddr() + "," + vmstorage2.VminsertAddr(),
}...)
vminsert := tc.MustStartVminsert(opts.VminsertInstance, opts.VminsertFlags)
opts.VmselectFlags = append(opts.VmselectFlags, []string{
})
vmselect := tc.MustStartVmselect("vmselect", []string{
"-storageNode=" + vmstorage1.VmselectAddr() + "," + vmstorage2.VmselectAddr(),
}...)
vmselect := tc.MustStartVmselect(opts.VmselectInstance, opts.VmselectFlags)
})
return &vmcluster{vminsert, vmselect, []*Vmstorage{vmstorage1, vmstorage2}}
}

View File

@@ -3,10 +3,9 @@ package tests
import (
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
// Data used in examples in
@@ -31,7 +30,7 @@ var docData = []string{
// TestSingleKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
// for vm-single.
func TestSingleKeyConceptsQuery(t *testing.T) {
tc := at.NewTestCase(t)
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartDefaultVmsingle()
@@ -42,7 +41,7 @@ func TestSingleKeyConceptsQuery(t *testing.T) {
// TestClusterKeyConceptsQueryData verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
// for vm-cluster.
func TestClusterKeyConceptsQueryData(t *testing.T) {
tc := at.NewTestCase(t)
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartDefaultCluster()
@@ -51,10 +50,10 @@ func TestClusterKeyConceptsQueryData(t *testing.T) {
}
// testClusterKeyConceptsQuery verifies cases from https://docs.victoriametrics.com/keyconcepts/#query-data
func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
func testKeyConceptsQueryData(t *testing.T, sut apptest.PrometheusWriteQuerier) {
// Insert example data from documentation.
sut.PrometheusAPIV1ImportPrometheus(t, docData, at.QueryOpts{})
sut.PrometheusAPIV1ImportPrometheus(t, docData, apptest.QueryOpts{})
sut.ForceFlush(t)
testInstantQuery(t, sut)
@@ -65,14 +64,14 @@ func testKeyConceptsQueryData(t *testing.T, sut at.PrometheusWriteQuerier) {
// testInstantQuery verifies the statements made in the `Instant query` section
// of the VictoriaMetrics documentation. See:
// https://docs.victoriametrics.com/keyconcepts/#instant-query
func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
func testInstantQuery(t *testing.T, q apptest.PrometheusQuerier) {
// Get the value of the foo_bar time series at 2022-05-10T08:03:00Z with the
// step of 5m and timeout 5s. There is no sample at exactly this timestamp.
// Therefore, VictoriaMetrics will search for the nearest sample within the
// [time-5m..time] interval.
got := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
got := q.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{Time: "2022-05-10T08:03:00.000Z", Step: "5m"})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data":{"result":[{"metric":{"__name__":"foo_bar"},"value":[1652169780,"3"]}]}}`)
opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
if diff := cmp.Diff(want, got, opt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -82,7 +81,7 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
// Therefore, VictoriaMetrics will search for the nearest sample within the
// [time-1m..time] interval. Since the nearest sample is 2m away and the
// step is 1m, then the VictoriaMetrics must return empty response.
got = q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
got = q.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{Time: "2022-05-10T08:18:00.000Z", Step: "1m"})
if len(got.Data.Result) > 0 {
t.Errorf("unexpected response: got non-empty result, want empty result:\n%v", got)
}
@@ -91,14 +90,14 @@ func testInstantQuery(t *testing.T, q at.PrometheusQuerier) {
// testRangeQuery verifies the statements made in the `Range query` section of
// the VictoriaMetrics documentation. See:
// https://docs.victoriametrics.com/keyconcepts/#range-query
func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
f := func(start, end, step string, wantSamples []*at.Sample) {
func testRangeQuery(t *testing.T, q apptest.PrometheusQuerier) {
f := func(start, end, step string, wantSamples []*apptest.Sample) {
t.Helper()
got := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{Start: start, End: end, Step: step})
want := at.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
got := q.PrometheusAPIV1QueryRange(t, "foo_bar", apptest.QueryOpts{Start: start, End: end, Step: step})
want := apptest.NewPrometheusAPIV1QueryResponse(t, `{"data": {"result": [{"metric": {"__name__": "foo_bar"}, "values": []}]}}`)
want.Data.Result[0].Samples = wantSamples
opt := cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
opt := cmpopts.IgnoreFields(apptest.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType")
if diff := cmp.Diff(want, got, opt); diff != "" {
t.Errorf("unexpected response (-want, +got):\n%s", diff)
}
@@ -107,68 +106,66 @@ func testRangeQuery(t *testing.T, q at.PrometheusQuerier) {
// Verify the statement that the query result for
// [2022-05-10T07:59:00Z..2022-05-10T08:17:00Z] time range and 1m step will
// contain 17 points.
f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "1m", []*at.Sample{
f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "1m", []*apptest.Sample{
// Sample for 2022-05-10T07:59:00Z is missing because the time series has
// samples only starting from 8:00.
at.NewSample(t, "2022-05-10T08:00:00Z", 1),
at.NewSample(t, "2022-05-10T08:01:00Z", 2),
at.NewSample(t, "2022-05-10T08:02:00Z", 3),
at.NewSample(t, "2022-05-10T08:03:00Z", 3),
at.NewSample(t, "2022-05-10T08:04:00Z", 5),
at.NewSample(t, "2022-05-10T08:05:00Z", 5),
at.NewSample(t, "2022-05-10T08:06:00Z", 5.5),
at.NewSample(t, "2022-05-10T08:07:00Z", 5.5),
at.NewSample(t, "2022-05-10T08:08:00Z", 4),
at.NewSample(t, "2022-05-10T08:09:00Z", 4),
apptest.NewSample(t, "2022-05-10T08:00:00Z", 1),
apptest.NewSample(t, "2022-05-10T08:01:00Z", 2),
apptest.NewSample(t, "2022-05-10T08:02:00Z", 3),
apptest.NewSample(t, "2022-05-10T08:03:00Z", 3),
apptest.NewSample(t, "2022-05-10T08:04:00Z", 5),
apptest.NewSample(t, "2022-05-10T08:05:00Z", 5),
apptest.NewSample(t, "2022-05-10T08:06:00Z", 5.5),
apptest.NewSample(t, "2022-05-10T08:07:00Z", 5.5),
apptest.NewSample(t, "2022-05-10T08:08:00Z", 4),
apptest.NewSample(t, "2022-05-10T08:09:00Z", 4),
// Sample for 2022-05-10T08:10:00Z is missing because there is no sample
// within the [8:10 - 1m .. 8:10] interval.
at.NewSample(t, "2022-05-10T08:11:00Z", 3.5),
at.NewSample(t, "2022-05-10T08:12:00Z", 3.25),
at.NewSample(t, "2022-05-10T08:13:00Z", 3),
at.NewSample(t, "2022-05-10T08:14:00Z", 2),
at.NewSample(t, "2022-05-10T08:15:00Z", 1),
at.NewSample(t, "2022-05-10T08:16:00Z", 4),
at.NewSample(t, "2022-05-10T08:17:00Z", 4),
apptest.NewSample(t, "2022-05-10T08:11:00Z", 3.5),
apptest.NewSample(t, "2022-05-10T08:12:00Z", 3.25),
apptest.NewSample(t, "2022-05-10T08:13:00Z", 3),
apptest.NewSample(t, "2022-05-10T08:14:00Z", 2),
apptest.NewSample(t, "2022-05-10T08:15:00Z", 1),
apptest.NewSample(t, "2022-05-10T08:16:00Z", 4),
apptest.NewSample(t, "2022-05-10T08:17:00Z", 4),
})
// Verify the statement that a query is executed at start, start+step,
// start+2*step, …, step+N*step timestamps, where N is the whole number
// of steps that fit between start and end.
f("2022-05-10T08:00:01.000Z", "2022-05-10T08:02:00.000Z", "1m", []*at.Sample{
at.NewSample(t, "2022-05-10T08:00:01Z", 1),
at.NewSample(t, "2022-05-10T08:01:01Z", 2),
f("2022-05-10T08:00:01.000Z", "2022-05-10T08:02:00.000Z", "1m", []*apptest.Sample{
apptest.NewSample(t, "2022-05-10T08:00:01Z", 1),
apptest.NewSample(t, "2022-05-10T08:01:01Z", 2),
})
// Verify the statement that a query is executed at start, start+step,
// start+2*step, …, end timestamps, when end = start + N*step.
f("2022-05-10T08:00:00.000Z", "2022-05-10T08:02:00.000Z", "1m", []*at.Sample{
at.NewSample(t, "2022-05-10T08:00:00Z", 1),
at.NewSample(t, "2022-05-10T08:01:00Z", 2),
at.NewSample(t, "2022-05-10T08:02:00Z", 3),
f("2022-05-10T08:00:00.000Z", "2022-05-10T08:02:00.000Z", "1m", []*apptest.Sample{
apptest.NewSample(t, "2022-05-10T08:00:00Z", 1),
apptest.NewSample(t, "2022-05-10T08:01:00Z", 2),
apptest.NewSample(t, "2022-05-10T08:02:00Z", 3),
})
// If the step isnt set, then it defaults to 5m (5 minutes).
f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "", []*at.Sample{
f("2022-05-10T07:59:00.000Z", "2022-05-10T08:17:00.000Z", "", []*apptest.Sample{
// Sample for 2022-05-10T07:59:00Z is missing because the time series has
// samples only starting from 8:00.
at.NewSample(t, "2022-05-10T08:04:00Z", 5),
at.NewSample(t, "2022-05-10T08:09:00Z", 4),
at.NewSample(t, "2022-05-10T08:14:00Z", 2),
apptest.NewSample(t, "2022-05-10T08:04:00Z", 5),
apptest.NewSample(t, "2022-05-10T08:09:00Z", 4),
apptest.NewSample(t, "2022-05-10T08:14:00Z", 2),
})
}
// testRangeQueryIsEquivalentToManyInstantQueries verifies the statement made in
// the `Range query` section of the VictoriaMetrics documentation that a range
// query is actually an instant query executed 1 + (start-end)/step times on the
// time range from start to end. The only difference is that instant queries
// will not procude ephemeral points.
//
// See: https://docs.victoriametrics.com/keyconcepts/#range-query
func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.PrometheusQuerier) {
f := func(timestamp string, want *at.Sample) {
// time range from start to end. See:
// https://docs.victoriametrics.com/keyconcepts/#range-query
func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q apptest.PrometheusQuerier) {
f := func(timestamp string, want *apptest.Sample) {
t.Helper()
gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", at.QueryOpts{Time: timestamp, Step: "1m"})
gotInstant := q.PrometheusAPIV1Query(t, "foo_bar", apptest.QueryOpts{Time: timestamp, Step: "1m"})
if want == nil {
if got, want := len(gotInstant.Data.Result), 0; got != want {
t.Errorf("unexpected instant result size: got %d, want %d", got, want)
@@ -181,7 +178,7 @@ func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.Prometheu
}
}
rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", at.QueryOpts{
rangeRes := q.PrometheusAPIV1QueryRange(t, "foo_bar", apptest.QueryOpts{
Start: "2022-05-10T07:59:00.000Z",
End: "2022-05-10T08:17:00.000Z",
Step: "1m",
@@ -192,13 +189,13 @@ func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.Prometheu
f("2022-05-10T08:00:00.000Z", rangeSamples[0])
f("2022-05-10T08:01:00.000Z", rangeSamples[1])
f("2022-05-10T08:02:00.000Z", rangeSamples[2])
f("2022-05-10T08:03:00.000Z", nil)
f("2022-05-10T08:03:00.000Z", rangeSamples[3])
f("2022-05-10T08:04:00.000Z", rangeSamples[4])
f("2022-05-10T08:05:00.000Z", nil)
f("2022-05-10T08:05:00.000Z", rangeSamples[5])
f("2022-05-10T08:06:00.000Z", rangeSamples[6])
f("2022-05-10T08:07:00.000Z", rangeSamples[7])
f("2022-05-10T08:08:00.000Z", rangeSamples[8])
f("2022-05-10T08:09:00.000Z", nil)
f("2022-05-10T08:09:00.000Z", rangeSamples[9])
f("2022-05-10T08:10:00.000Z", nil)
f("2022-05-10T08:11:00.000Z", rangeSamples[10])
f("2022-05-10T08:12:00.000Z", rangeSamples[11])
@@ -206,164 +203,5 @@ func testRangeQueryIsEquivalentToManyInstantQueries(t *testing.T, q at.Prometheu
f("2022-05-10T08:14:00.000Z", rangeSamples[13])
f("2022-05-10T08:15:00.000Z", rangeSamples[14])
f("2022-05-10T08:16:00.000Z", rangeSamples[15])
f("2022-05-10T08:17:00.000Z", nil)
}
func TestSingleMillisecondPrecisionInInstantQueries(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartDefaultVmsingle()
testMillisecondPrecisionInInstantQueries(tc, sut)
}
func TestClusterMillisecondPrecisionInInstantQueries(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartDefaultCluster()
testMillisecondPrecisionInInstantQueries(tc, sut)
}
func testMillisecondPrecisionInInstantQueries(tc *at.TestCase, sut at.PrometheusWriteQuerier) {
t := tc.T()
type opts struct {
query string
qtime string
step string
wantMetric map[string]string
wantSample *at.Sample
wantSamples []*at.Sample
}
f := func(sut at.PrometheusQuerier, opts *opts) {
t.Helper()
wantResult := []*at.QueryResult{}
if opts.wantMetric != nil && (opts.wantSample != nil || len(opts.wantSamples) > 0) {
wantResult = append(wantResult, &at.QueryResult{
Metric: opts.wantMetric,
Sample: opts.wantSample,
Samples: opts.wantSamples,
})
}
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
Time: opts.qtime,
Step: opts.step,
})
},
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
sut.PrometheusAPIV1ImportPrometheus(t, []string{
`series1{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`series1{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{})
sut.ForceFlush(t)
// Verify that both points were created correctly. Fetch both points by
// passing a duration into the instant query: the difference between the two
// timestamps, plus 1ms (to include both points), so 101ms:
f(sut, &opts{
query: "series1[101ms]",
qtime: "1707123456800", // 2024-02-05T08:57:36.800Z
wantMetric: map[string]string{"__name__": "series1", "label": "foo"},
wantSamples: []*at.Sample{
{Timestamp: 1707123456700, Value: 10},
{Timestamp: 1707123456800, Value: 20},
},
})
// Search the first point at its own timestamp with step 1ms.
f(sut, &opts{
query: "series1",
qtime: "1707123456700", // 2024-02-05T08:57:36.700Z
step: "1ms",
wantMetric: map[string]string{"__name__": "series1", "label": "foo"},
wantSample: &at.Sample{Timestamp: 1707123456700, Value: 10},
})
// Search the first point at 1ms past its own timestamp.
// Expect empty result because the search interval (qtime-step, qtime]
// excludes the timestamp of the first point.
f(sut, &opts{
query: "series1",
qtime: "1707123456701", // 2024-02-05T08:57:36.701Z
step: "1ms",
})
// Fetch the last point at its timestamp with step 1ms.
f(sut, &opts{
query: "series1",
qtime: "1707123456800", // 2024-02-05T08:57:36.800Z
step: "1ms",
wantMetric: map[string]string{"__name__": "series1", "label": "foo"},
wantSample: &at.Sample{Timestamp: 1707123456800, Value: 20},
})
// Fetch the last point at its timestamp with step 1ms.
// Expect empty result because the search interval (qtime-step, qtime]
// excludes the timestamp of the first point.
f(sut, &opts{
query: "series1",
qtime: "1707123456801", // 2024-02-05T08:57:36.801Z
step: "1ms",
// wantMetric: map[string]string{"__name__": "series1", "label": "foo"},
// wantSample: &at.Sample{Timestamp: 1707123456801, Value: 20},
})
// Insert samples with different dates. The difference in ms between the two
// timestamps is 4236579304.
sut.PrometheusAPIV1ImportPrometheus(t, []string{
`series2{label="foo"} 10 1638564958042`, // 2021-12-03T20:55:58.042Z
`series2{label="foo"} 20 1642801537346`, // 2022-01-21T21:45:37.346Z
}, at.QueryOpts{})
sut.ForceFlush(t)
// Both Prometheus and VictoriaMetrics exclude the leftmost millisecond,
// thus the following queries must return only one sample.
f(sut, &opts{
query: "series2[4236579304ms]",
qtime: "1642801537346",
step: "1ms",
wantMetric: map[string]string{"__name__": "series2", "label": "foo"},
wantSamples: []*at.Sample{
{Timestamp: 1642801537346, Value: 20},
},
})
f(sut, &opts{
query: "count_over_time(series2[4236579304ms])",
qtime: "1642801537346", // 2022-01-21T21:45:37.346Z
step: "1ms",
wantMetric: map[string]string{"label": "foo"},
wantSample: &at.Sample{Timestamp: 1642801537346, Value: 1},
})
// Adding 1ms to the duration (4236579305ms) causes queries to return 2
// samples.
f(sut, &opts{
query: "series2[4236579305ms]",
qtime: "1642801537346",
step: "1ms",
wantMetric: map[string]string{"__name__": "series2", "label": "foo"},
wantSamples: []*at.Sample{
{Timestamp: 1638564958042, Value: 10}, // 2021-12-03T20:55:58.042Z
{Timestamp: 1642801537346, Value: 20},
},
})
f(sut, &opts{
query: "count_over_time(series2[4236579305ms])",
qtime: "1642801537346", // 2022-01-21T21:45:37.346Z
step: "1ms",
wantMetric: map[string]string{"label": "foo"},
wantSample: &at.Sample{Timestamp: 1642801537346, Value: 2},
})
f("2022-05-10T08:17:00.000Z", rangeSamples[16])
}

View File

@@ -1,33 +0,0 @@
package tests
import (
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/apptest"
)
// Data used in tests
var testData = []string{
"foo_bar 1.00",
"foo_bar 2.00",
}
func TestSingleMaxIngestionRateIncrementsMetric(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=1"})
sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
if got := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"); got <= 0 {
t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
}
}
func TestSingleMaxIngestionRateDoesNotIncrementMetric(t *testing.T) {
tc := apptest.NewTestCase(t)
defer tc.Stop()
sut := tc.MustStartVmsingle("vmsingle", []string{"-maxIngestionRate=15"})
sut.PrometheusAPIV1ImportPrometheus(t, testData, apptest.QueryOpts{})
if got, want := sut.GetMetric(t, "vm_max_ingestion_rate_limit_reached_total"), 0.0; got != want {
t.Fatalf("Unexpected vm_max_ingestion_rate_limit_reached_total: got %f, want >0", got)
}
}

View File

@@ -1,204 +0,0 @@
package tests
import (
"fmt"
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
at "github.com/VictoriaMetrics/VictoriaMetrics/apptest"
pb "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
)
func TestSingleIngestionWithRelabeling(t *testing.T) {
tc := at.NewTestCase(t)
defer tc.Stop()
const relabelFileName = "relabel_config.yaml"
relabelingRules := `
# add 4 labels in order to call memory allocation
- replacement: value1
target_label: label1
- replacement: value2
target_label: label2
- replacement: value3
target_label: label3
- replacement: value4
target_label: label4
# drop specific timeseries by name prefix
- action: drop
if: '{__name__=~"^must_drop.+"}'
# strip prefix from metric name
# and write it into special label
- source_labels: [__name__]
regex: '^(.+)_(.+)'
replacement: $1
target_label: ingestion_protocol
- source_labels: [__name__]
regex: '^(.+)_(.+)'
replacement: $2
target_label: __name__
`
relabelFilePath := fmt.Sprintf("%s/%s", t.TempDir(), relabelFileName)
if err := os.WriteFile(relabelFilePath, []byte(relabelingRules), os.ModePerm); err != nil {
t.Fatalf("cannot create file=%q: %s", relabelFilePath, err)
}
sut := tc.MustStartVmsingle("relabeling-ingest",
[]string{fmt.Sprintf(`-relabelConfig=%s`, relabelFilePath),
`-retentionPeriod=100y`})
type opts struct {
query string
qtime string
step string
wantMetrics []map[string]string
wantSamples []*at.Sample
}
f := func(sut at.PrometheusQuerier, opts *opts) {
t.Helper()
wantResult := []*at.QueryResult{}
for idx, wm := range opts.wantMetrics {
wantResult = append(wantResult, &at.QueryResult{
Metric: wm,
Samples: []*at.Sample{opts.wantSamples[idx]},
})
}
tc.Assert(&at.AssertOptions{
Msg: "unexpected /api/v1/query response",
Got: func() any {
return sut.PrometheusAPIV1Query(t, opts.query, at.QueryOpts{
Time: opts.qtime,
Step: opts.step,
})
},
Want: &at.PrometheusAPIV1QueryResponse{Data: &at.QueryData{Result: wantResult}},
CmpOpts: []cmp.Option{
cmpopts.IgnoreFields(at.PrometheusAPIV1QueryResponse{}, "Status", "Data.ResultType"),
},
})
}
sut.PrometheusAPIV1ImportPrometheus(t, []string{
`importprometheus_series{label="foo"} 10 1707123456700`, // 2024-02-05T08:57:36.700Z
`must_drop_series{label="foo"} 20 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{})
sut.ForceFlush(t)
f(sut, &opts{
query: `{label="foo"}[120ms]`,
qtime: "1707123456800", // 2024-02-05T08:57:36.800Z
wantMetrics: []map[string]string{
{
"__name__": "series",
"label": "foo",
"label1": "value1",
"label2": "value2",
"label3": "value3",
"label4": "value4",
"ingestion_protocol": "importprometheus",
},
},
wantSamples: []*at.Sample{
{Timestamp: 1707123456700, Value: 10},
},
})
// write influx with multi field set series1 and series2 in order to test
// memory optimisation at vminsert side
sut.InfluxWrite(t, []string{
`influxline,label=foo1 series1=10,series2=30 1707123456700`, // 2024-02-05T08:57:36.700Z
`must_drop,label=foo1 series1=20,series2=40 1707123456800`, // 2024-02-05T08:57:36.800Z
}, at.QueryOpts{})
sut.ForceFlush(t)
f(sut, &opts{
query: `{label="foo1"}[120ms]`,
qtime: "1707123456800", // 2024-02-05T08:57:36.800Z
wantMetrics: []map[string]string{
{
"__name__": "series1",
"label": "foo1",
"label1": "value1",
"label2": "value2",
"label3": "value3",
"label4": "value4",
"ingestion_protocol": "influxline",
},
{
"__name__": "series2",
"label": "foo1",
"label1": "value1",
"label2": "value2",
"label3": "value3",
"label4": "value4",
"ingestion_protocol": "influxline"},
},
wantSamples: []*at.Sample{
{Timestamp: 1707123456700, Value: 10},
{Timestamp: 1707123456700, Value: 30},
},
})
pbData := []pb.TimeSeries{
{
Labels: []pb.Label{
{
Name: "__name__",
Value: "prometheusrw_series",
},
{
Name: "label",
Value: "foo2",
},
},
Samples: []pb.Sample{
{
Value: 10,
Timestamp: 1707123456700, // 2024-02-05T08:57:36.700Z
},
},
},
{
Labels: []pb.Label{
{
Name: "__name__",
Value: "must_drop_series",
},
{
Name: "label",
Value: "foo2",
},
},
Samples: []pb.Sample{
{
Value: 20,
Timestamp: 1707123456800, // 2024-02-05T08:57:36.800Z
},
},
},
}
sut.PrometheusAPIV1Write(t, pbData, at.QueryOpts{})
sut.ForceFlush(t)
f(sut, &opts{
query: `{label="foo2"}[120ms]`,
qtime: "1707123456800", // 2024-02-05T08:57:36.800Z
wantMetrics: []map[string]string{
{
"__name__": "series",
"label": "foo2",
"label1": "value1",
"label2": "value2",
"label3": "value3",
"label4": "value4",
"ingestion_protocol": "prometheusrw",
},
},
wantSamples: []*at.Sample{
{Timestamp: 1707123456700, Value: 10}, // 2024-02-05T08:57:36.700Z
},
})
}

View File

@@ -26,7 +26,6 @@ type Vmsingle struct {
forceFlushURL string
// vminsert URLs.
influxLineWriteURL string
prometheusAPIV1ImportPrometheusURL string
prometheusAPIV1WriteURL string
@@ -65,7 +64,6 @@ func StartVmsingle(instance string, flags []string, cli *Client) (*Vmsingle, err
httpListenAddr: stderrExtracts[1],
forceFlushURL: fmt.Sprintf("http://%s/internal/force_flush", stderrExtracts[1]),
influxLineWriteURL: fmt.Sprintf("http://%s/influx/write", stderrExtracts[1]),
prometheusAPIV1ImportPrometheusURL: fmt.Sprintf("http://%s/prometheus/api/v1/import/prometheus", stderrExtracts[1]),
prometheusAPIV1WriteURL: fmt.Sprintf("http://%s/prometheus/api/v1/write", stderrExtracts[1]),
prometheusAPIV1ExportURL: fmt.Sprintf("http://%s/prometheus/api/v1/export", stderrExtracts[1]),
@@ -83,18 +81,6 @@ func (app *Vmsingle) ForceFlush(t *testing.T) {
app.cli.Get(t, app.forceFlushURL, http.StatusOK)
}
// InfluxWrite is a test helper function that inserts a
// collection of records in Influx line format by sending a HTTP
// POST request to /influx/write vmsingle endpoint.
//
// See https://docs.victoriametrics.com/url-examples/#influxwrite
func (app *Vmsingle) InfluxWrite(t *testing.T, records []string, _ QueryOpts) {
t.Helper()
data := []byte(strings.Join(records, "\n"))
app.cli.Post(t, app.influxLineWriteURL, "text/plain", data, http.StatusNoContent)
}
// PrometheusAPIV1Write is a test helper function that inserts a
// collection of records in Prometheus remote-write format by sending a HTTP
// POST request to /prometheus/api/v1/write vmsingle endpoint.

View File

@@ -210,4 +210,3 @@ Please see more examples on integration of VictoriaLogs with other log shippers
* [opentelemetry-collector](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/opentelemetry-collector)
* [telegraf](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/telegraf)
* [fluentd]((https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/fluentd)
* [datadog-serverless](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/datadog-serverless)

View File

@@ -4,7 +4,7 @@ services:
# And forward them to --remoteWrite.url
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.108.1
image: victoriametrics/vmagent:v1.107.0
depends_on:
- "vminsert"
ports:
@@ -39,7 +39,7 @@ services:
# where N is number of vmstorages (2 in this case).
vmstorage-1:
container_name: vmstorage-1
image: victoriametrics/vmstorage:v1.108.1-cluster
image: victoriametrics/vmstorage:v1.107.0-cluster
ports:
- 8482
- 8400
@@ -51,7 +51,7 @@ services:
restart: always
vmstorage-2:
container_name: vmstorage-2
image: victoriametrics/vmstorage:v1.108.1-cluster
image: victoriametrics/vmstorage:v1.107.0-cluster
ports:
- 8482
- 8400
@@ -66,7 +66,7 @@ services:
# pre-process them and distributes across configured vmstorage shards.
vminsert:
container_name: vminsert
image: victoriametrics/vminsert:v1.108.1-cluster
image: victoriametrics/vminsert:v1.107.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -81,7 +81,7 @@ services:
# vmselect collects results from configured `--storageNode` shards.
vmselect-1:
container_name: vmselect-1
image: victoriametrics/vmselect:v1.108.1-cluster
image: victoriametrics/vmselect:v1.107.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -94,7 +94,7 @@ services:
restart: always
vmselect-2:
container_name: vmselect-2
image: victoriametrics/vmselect:v1.108.1-cluster
image: victoriametrics/vmselect:v1.107.0-cluster
depends_on:
- "vmstorage-1"
- "vmstorage-2"
@@ -112,7 +112,7 @@ services:
# It can be used as an authentication proxy.
vmauth:
container_name: vmauth
image: victoriametrics/vmauth:v1.108.1
image: victoriametrics/vmauth:v1.107.0
depends_on:
- "vmselect-1"
- "vmselect-2"
@@ -127,7 +127,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.108.1
image: victoriametrics/vmalert:v1.107.0
depends_on:
- "vmauth"
ports:

View File

@@ -10,14 +10,14 @@ services:
- 3000:3000
volumes:
- grafanadata:/var/lib/grafana
- ./provisioning/datasources/victoriametrics-logs-datasource:/etc/grafana/provisioning/datasources
- ./provisioning/datasources/victorialogs-datasource:/etc/grafana/provisioning/datasources
- ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
- ./provisioning/plugins/:/var/lib/grafana/plugins
- ./../../dashboards/victoriametrics.json:/var/lib/grafana/dashboards/vm.json
- ./../../dashboards/victorialogs.json:/var/lib/grafana/dashboards/vl.json
environment:
- "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.13.0/victoriametrics-logs-datasource-v0.13.0.zip;victoriametrics-logs-datasource"
- "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-logs-datasource"
- "GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.11.1/victorialogs-datasource-v0.11.1.zip;victorialogs-datasource"
- "GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victorialogs-datasource"
networks:
- vm_net
restart: always
@@ -45,7 +45,7 @@ services:
# storing logs and serving read queries.
victorialogs:
container_name: victorialogs
image: victoriametrics/victoria-logs:v1.4.0-victorialogs
image: victoriametrics/victoria-logs:v1.3.2-victorialogs
command:
- "--storageDataPath=/vlogs"
- "--httpListenAddr=:9428"
@@ -60,7 +60,7 @@ services:
# scraping, storing metrics and serve read requests.
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.108.1
image: victoriametrics/victoria-metrics:v1.107.0
ports:
- 8428:8428
volumes:
@@ -79,7 +79,7 @@ services:
# depending on the requested path.
vmauth:
container_name: vmauth
image: victoriametrics/vmauth:v1.108.1
image: victoriametrics/vmauth:v1.107.0
depends_on:
- "victoriametrics"
- "victorialogs"
@@ -96,7 +96,7 @@ services:
# vmalert executes alerting and recording rules according to given rule type.
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.108.1
image: victoriametrics/vmalert:v1.107.0
depends_on:
- "vmauth"
- "alertmanager"

View File

@@ -4,7 +4,7 @@ services:
# And forward them to --remoteWrite.url
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.108.1
image: victoriametrics/vmagent:v1.107.0
depends_on:
- "victoriametrics"
ports:
@@ -22,7 +22,7 @@ services:
# storing metrics and serve read requests.
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.108.1
image: victoriametrics/victoria-metrics:v1.107.0
ports:
- 8428:8428
- 8089:8089
@@ -65,7 +65,7 @@ services:
# vmalert executes alerting and recording rules
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.108.1
image: victoriametrics/vmalert:v1.107.0
depends_on:
- "victoriametrics"
- "alertmanager"
@@ -79,7 +79,7 @@ services:
command:
- "--datasource.url=http://victoriametrics:8428/"
- "--remoteRead.url=http://victoriametrics:8428/"
- "--remoteWrite.url=http://vmagent:8429/"
- "--remoteWrite.url=http://victoriametrics:8428/"
- "--notifier.url=http://alertmanager:9093/"
- "--rule=/etc/alerts/*.yml"
# display source of alerts in grafana

View File

@@ -2,11 +2,11 @@ apiVersion: 1
datasources:
- name: VictoriaLogs
type: victoriametrics-logs-datasource
type: victorialogs-datasource
access: proxy
url: http://victorialogs:9428
- name: VictoriaMetrics
type: prometheus
access: proxy
url: http://victoriametrics:8428
url: http://victoriametrics:8428

View File

@@ -30,18 +30,6 @@ groups:
summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5m"
# default value of 900 Should be changed to the scrape_interval for pull metrics. For push metrics this should be the lowest fit_every or infer_every in your vmanomaly config.
- alert: NoSelfMonitoringMetrics
expr: >
lag(vmanomaly_start_time_seconds{job="vmanomaly"}[24h]) > 900
for: 5m
labels:
severity: critical
annotations:
summary: "Metrics have not been seen from \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for {{ $value }} seconds"
description: >
The missing metric may incidate that vmanomaly is not running or is inaccessable from vmagent or the remotewrite endpoint.
- alert: ProcessNearFDLimits
expr: (process_max_fds{job=~".*vmanomaly.*"} - process_open_fds{job=~".*vmanomaly.*"}) < 100
for: 5m

View File

@@ -1,12 +1,11 @@
services:
# meta service will be ignored by compose
.victorialogs:
image: docker.io/victoriametrics/victoria-logs:v1.4.0-victorialogs
image: docker.io/victoriametrics/victoria-logs:v1.3.2-victorialogs
command:
- -storageDataPath=/vlogs
- -loggerFormat=json
- -syslog.listenAddr.tcp=0.0.0.0:8094
- -datadog.streamFields=service,hostname,ddsource
- -journald.streamFields=_HOSTNAME,_SYSTEMD_UNIT,_PID
- -journald.ignoreFields=MESSAGE_ID,INVOCATION_ID,USER_INVOCATION_ID,
- -journald.ignoreFields=_BOOT_ID,_MACHINE_ID,_SYSTEMD_INVOCATION_ID,_STREAM_ID,_UID
@@ -18,8 +17,8 @@ services:
timeout: 1s
retries: 10
dd-proxy:
image: docker.io/victoriametrics/vmauth:v1.108.1
dd-logs:
image: docker.io/victoriametrics/vmauth:v1.107.0
restart: on-failure
volumes:
- ./:/etc/vmauth

View File

@@ -11,8 +11,8 @@ services:
- /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
environment:
DD_API_KEY: test
DD_URL: http://dd-proxy:8427
DD_LOGS_CONFIG_LOGS_DD_URL: http://dd-proxy:8427
DD_URL: http://victoriametrics:8428/datadog
DD_LOGS_CONFIG_LOGS_DD_URL: http://dd-logs:8427
DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: true
DD_LOGS_ENABLED: true
DD_LOGS_CONFIG_USE_HTTP: true

View File

@@ -1 +0,0 @@
**/logs

View File

@@ -1,31 +0,0 @@
# Docker compose Serverless with DataDog extension integration with VictoriaLogs
The folder contains examples of [DataDog serverless](https://docs.datadoghq.com/serverless) integration with VictoriaLogs for:
* [AWS Lambda](./aws)
* [GCP Cloud Run](./gcp)
To spin-up environment `cd` to any of listed above directories run the following command:
```
docker compose up -d
```
To shut down the docker-compose environment run the following command:
```
docker compose down
docker compose rm -f
```
The docker compose file contains the following components:
* dd-proxy - VMAuth proxy, with path-based routing to `victoriametrics` and `victorialogs`
* lambda - Serverless application with Datadog logs collection extension, which is configured to collect and write data to `victorialogs` and `victoriametrics` via `dd-proxy`
* victorialogs - VictoriaLogs log database, which accepts the data from `datadog`
* victoriametrics - VictoriaMetrics metrics database, which collects metrics from `victorialogs` and `datadog`
Querying the data
* [vmui](https://docs.victoriametrics.com/victorialogs/querying/#vmui) - a web UI is accessible by `http://localhost:9428/select/vmui`
* for querying the data via command-line please check [these docs](https://docs.victoriametrics.com/victorialogs/querying/#command-line)
Please, note that `_stream_fields` parameter must follow recommended [best practices](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) to achieve better performance.

View File

@@ -1,32 +0,0 @@
FROM golang:1.23-bullseye as aws-lambda-rie
# Install custom aws-lambda-rie till Telemetry API support is not merged
# https://github.com/aws/aws-lambda-runtime-interface-emulator/pull/137
RUN \
git clone https://github.com/VictoriaMetrics/aws-lambda-runtime-interface-emulator -b added-telemetry-api-support /tmp/aws-lambda-rie && \
cd /tmp/aws-lambda-rie && \
CGO_ENABLED=0 go build -buildvcs=false -ldflags "-s -w" -o /aws-lambda-rie ./cmd/aws-lambda-rie
FROM python:3.12-bullseye
RUN \
apt update && \
apt install -y \
curl \
g++ \
make \
cmake \
unzip \
libcurl4-openssl-dev && \
mkdir -p /var/task && \
pip install \
--target /var/task awslambdaric datadog-lambda
WORKDIR /var/task
COPY --from=aws-lambda-rie /aws-lambda-rie /var/task/aws-lambda-rie
COPY main.py /var/task/
COPY --from=public.ecr.aws/datadog/lambda-extension:67 /opt/. /opt/
ENTRYPOINT ["/var/task/aws-lambda-rie"]
CMD ["/usr/local/bin/python", "-m", "awslambdaric", "main.lambda_handler"]

View File

@@ -1,25 +0,0 @@
name: datadog-serverless-aws
include:
- ../../compose-base.yml
services:
lambda:
build: .
restart: on-failure
ports:
- 8080:8080
environment:
DD_LOG_LEVEL: trace
DD_LOGS_ENABLED: true
DD_SOURCE: test
DD_API_KEY: test
DD_DD_URL: http://dd-proxy:8427
DD_EXTENSION_VERSION: compatibility
DD_PROFILING_ENABLED: false
DD_ENHANCED_METRICS: false
DD_LOGS_CONFIG_LOGS_DD_URL: http://dd-proxy:8427
DD_SERVERLESS_FLUSH_STRATEGY: periodically,100
depends_on:
victorialogs:
condition: service_healthy
victoriametrics:
condition: service_healthy

View File

@@ -1,15 +0,0 @@
from datadog_lambda.metric import lambda_metric
def lambda_handler(event, context):
lambda_metric(
metric_name='coffee_house.order_value',
value=12.45,
tags=['product:latte', 'order:online']
)
print('Hello, world!')
return {
'statusCode': 200,
'body': 'Hello from serverless!'
}

View File

@@ -1,14 +0,0 @@
FROM python:3.12-bullseye
COPY --from=datadog/serverless-init:1 /datadog-init /app/datadog-init
ENV DD_SERVICE=datadog-demo-run-go
ENV DD_ENV=datadog-demo
ENV DD_VERSION=1
RUN pip install Flask gunicorn datadog
WORKDIR /var/task
COPY main.py /var/task/
ENTRYPOINT ["/app/datadog-init"]
CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app

View File

@@ -1,24 +0,0 @@
name: datadog-serverless-gcp
include:
- ../../compose-base.yml
services:
cloud-run:
build: .
restart: on-failure
ports:
- 8080:8080
environment:
PORT: 8080
DD_LOG_LEVEL: trace
DD_LOGS_ENABLED: true
DD_SOURCE: test
DD_API_KEY: test
DD_DD_URL: http://dd-proxy:8427
DD_PROFILING_ENABLED: false
DD_ENHANCED_METRICS: false
DD_LOGS_CONFIG_LOGS_DD_URL: http://dd-proxy:8427
depends_on:
victorialogs:
condition: service_healthy
victoriametrics:
condition: service_healthy

View File

@@ -1,23 +0,0 @@
from datadog import initialize, statsd
import os
from flask import Flask
options = {
"statsd_host": "127.0.0.1",
"statsd_port": 8125,
}
initialize(**options)
app = Flask(__name__)
@app.route('/')
def hello_world():
statsd.gauge('active.connections', 1001, tags=["protocol:http"])
target = os.environ.get('TARGET', 'World')
return 'Hello {}!\n'.format(target)
if __name__ == "__main__":
app.run(debug=True,host='0.0.0.0',port=int(os.environ.get('PORT', 8080)))

View File

@@ -4,12 +4,3 @@ unauthorized_user:
- "/api/v2/logs"
- "/api/v1/validate"
url_prefix: "http://victorialogs:9428/insert/datadog/"
- src_paths:
- "/api/v1/series"
- "/api/v2/series"
- "/api/beta/sketches"
- "/api/v1/validate"
- "/api/v1/check_run"
- "/intake"
- "/api/v1/metadata"
url_prefix: "http://victoriametrics:8428/datadog/"

View File

@@ -1,7 +1,7 @@
services:
vmagent:
container_name: vmagent
image: victoriametrics/vmagent:v1.108.1
image: victoriametrics/vmagent:v1.107.0
depends_on:
- "victoriametrics"
ports:
@@ -18,7 +18,7 @@ services:
victoriametrics:
container_name: victoriametrics
image: victoriametrics/victoria-metrics:v1.108.1
image: victoriametrics/victoria-metrics:v1.107.0
ports:
- 8428:8428
volumes:
@@ -50,7 +50,7 @@ services:
vmalert:
container_name: vmalert
image: victoriametrics/vmalert:v1.108.1
image: victoriametrics/vmalert:v1.107.0
depends_on:
- "victoriametrics"
ports:

View File

@@ -3,7 +3,7 @@ version: "3"
services:
# Run `make package-victoria-logs` to build victoria-logs image
vlogs:
image: docker.io/victoriametrics/victoria-logs:v1.4.0-victorialogs
image: docker.io/victoriametrics/victoria-logs:v1.3.2-victorialogs
volumes:
- vlogs:/vlogs
ports:
@@ -46,7 +46,7 @@ services:
- "--config=/config.yml"
vmsingle:
image: victoriametrics/victoria-metrics:v1.108.1
image: victoriametrics/victoria-metrics:v1.107.0
ports:
- "8428:8428"
command:

View File

@@ -41,7 +41,7 @@ We use [labels](https://docs.github.com/en/issues/using-labels-and-milestones-to
to categorize GitHub issues. We have the following labels:
1. A component label: vmalert, vmagent, etc. Add this label to the issue if it is related to a specific component.
1. An issue type: `bug`, `enhancement`, `question`.
1. `enterprise`, assigned to issues related to ENT features
1. `enterprize`, assigned to issues related to ENT features
1. `need more info`, assigned to issues which require elaboration from the issue creator.
For example, if we weren't able to reproduce the reported bug based on the ticket description then we ask additional
questions which could help to reproduce the issue and add `need more info` label. This label helps other maintainers
@@ -57,8 +57,7 @@ Implementing a bugfix or enhancement requires sending a pull request to the [cor
A pull request should contain the following attributes:
1. Don't use `master` branch for making PRs, as it makes it impossible for reviewers to modify the change.
1. All commits need to be [signed](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits).
1. A clear and concise description of what was done and for what purpose. Use the imperative, present tense: "change" not "changed" nor "changes".
Read your commit message as "This commit will ..", don't capitalize the first letter
1. A clear and concise description of what was done and for what purpose.
1. A link to the issue related to this change, if any.
1. Tests proving that the change is effective. See [this style guide](https://itnext.io/f-tests-as-a-replacement-for-table-driven-tests-in-go-8814a8b19e9e) for tests.
To run tests and code checks locally execute commands `make tests-full` and `make check-all`.
@@ -67,7 +66,7 @@ A pull request should contain the following attributes:
requires reflecting these changes in the documentation. For new features add `{{%/* available_from "#" */%}}` shortcode
to the documentation. It will be later automatically replaced with an actual release version.
1. A line in the [changelog](https://docs.victoriametrics.com/changelog/#tip) mentioning the change and related issue in a way
that would be clear to other readers even if they don't have the full context. Use the same guidelines as for commit message.
that would be clear to other readers even if they don't have the full context.
1. Reviewers who you think have the best expertise on the matter.
See good example of pull request [here](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6487).

View File

@@ -793,12 +793,6 @@ Some workloads may need fine-grained resource usage limits. In these cases the f
when the database contains big number of unique time series because of [high churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate).
In this case it might be useful to set the `-search.maxLabelsAPIDuration` to quite low value in order to limit CPU and memory usage.
See also `-search.maxLabelsAPISeries` and `-search.ignoreExtraFiltersAtLabelsAPI`.
- `-search.maxFederateSeries` at `vmselect` limits maximum number of time series, which can be returned via [/federate API](https://docs.victoriametrics.com#federation).
The duration of the `/federate` queries is limited via `-search.maxQueryDuration` flag. This option allows limiting memory usage.
- `-search.maxExportSeries` at `vmselect` limits maximum number of time series, which can be returned from [/api/v1/export* APIs](https://docs.victoriametrics.com#how-to-export-data-in-json-line-format).
The duration of the export queries is limited via `-search.maxExportDuration` flag. This option allows limiting memory usage.
- `-search.maxTSDBStatusSeries` at `vmselect` limits maximum number of time series, which can be processed during the call to [/api/v1/status/tsdb](https://docs.victoriametrics.com#tsdb-stats).
The duration of the status queries is limited via `-search.maxStatusRequestDuration` flag. This option allows limiting memory usage.
- `-storage.maxDailySeries` at `vmstorage` can be used for limiting the number of time series seen per day aka
[time series churn rate](https://docs.victoriametrics.com/faq/#what-is-high-churn-rate). See [cardinality limiter docs](#cardinality-limiter).
- `-storage.maxHourlySeries` at `vmstorage` can be used for limiting the number of [active time series](https://docs.victoriametrics.com/faq/#what-is-an-active-time-series).

View File

@@ -22,5 +22,5 @@ to [the latest available releases](https://docs.victoriametrics.com/changelog/).
## Currently supported LTS release lines
- v1.102.x - the latest one is [v1.102.9 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.9)
- v1.97.x - the latest one is [v1.97.14 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.14)
- v1.102.x - the latest one is [v1.102.8 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.102.8)
- v1.97.x - the latest one is [v1.97.13 LTS release](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.97.13)

View File

@@ -1208,8 +1208,10 @@ In this case [forced merge](#forced-merge) may help freeing up storage space.
It is recommended verifying which metrics will be deleted with the call to `http://<victoria-metrics-addr>:8428/api/v1/series?match[]=<timeseries_selector_for_delete>`
before actually deleting the metrics. By default, this query will only scan series in the past 5 minutes, so you may need to
adjust `start` and `end` to a suitable range to achieve match hits. Also, if the number of returned time series is
rather big you will need to set `-search.maxDeleteSeries` flag (see [Resource usage limits](#resource-usage-limits)).
adjust `start` and `end` to a suitable range to achieve match hits. Also, if the
number of returned time series is rather big you will need to set
`-search.maxDeleteSeries` flag (see
[Resource usage limits](#resource-usage-limits)).
The `/api/v1/admin/tsdb/delete_series` handler may be protected with `authKey` if `-deleteAuthKey` command-line flag is set.
Note that handler accepts any HTTP method, so sending a `GET` request to `/api/v1/admin/tsdb/delete_series` will result in deletion of time series.
@@ -1715,7 +1717,6 @@ See also [resource usage limits docs](#resource-usage-limits).
By default, VictoriaMetrics is tuned for an optimal resource usage under typical workloads. Some workloads may need fine-grained resource usage limits. In these cases the following command-line flags may be useful:
- `-maxIngestionRate` limits samples/second ingested. This may be useful when CPU resources are limited or overloaded.
- `-memory.allowedPercent` and `-memory.allowedBytes` limit the amounts of memory, which may be used for various internal caches at VictoriaMetrics.
Note that VictoriaMetrics may use more memory, since these flags don't limit additional memory, which may be needed on a per-query basis.
- `-search.maxMemoryPerQuery` limits the amounts of memory, which can be used for processing a single query. Queries, which need more memory, are rejected.
@@ -1789,12 +1790,6 @@ By default, VictoriaMetrics is tuned for an optimal resource usage under typical
In this case it might be useful to set the `-search.maxLabelsAPIDuration` to quite low value in order to limit CPU and memory usage.
See also `-search.maxLabelsAPISeries` and `-search.ignoreExtraFiltersAtLabelsAPI`.
- `-search.maxTagValueSuffixesPerSearch` limits the number of entries, which may be returned from `/metrics/find` endpoint. See [Graphite Metrics API usage docs](#graphite-metrics-api-usage).
- `-search.maxFederateSeries` limits maximum number of time series, which can be returned via [/federate API](#federation).
The duration of the `/federate` queries is limited via `-search.maxQueryDuration` flag. This option allows limiting memory usage.
- `-search.maxExportSeries` limits maximum number of time series, which can be returned from [/api/v1/export* APIs](#how-to-export-data-in-json-line-format).
The duration of the export queries is limited via `-search.maxExportDuration` flag. This option allows limiting memory usage.
- `-search.maxTSDBStatusSeries` limits maximum number of time series, which can be processed during the call to [/api/v1/status/tsdb](#tsdb-stats).
The duration of the status queries is limited via `-search.maxStatusRequestDuration` flag. This option allows limiting memory usage.
See also [resource usage limits at VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/#resource-usage-limits),
[cardinality limiter](#cardinality-limiter) and [capacity planning docs](#capacity-planning).
@@ -2970,9 +2965,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit
-maxConcurrentInserts int
The maximum number of concurrent insert requests. Set higher value when clients send data over slow networks. Default value depends on the number of available CPU cores. It should work fine in most cases since it minimizes resource usage. See also -insert.maxQueueDuration (default 32)
-maxIngestionRate int
The maximum number of samples vmsingle can receive per second. Data ingestion is paused when the limit is exceeded
By default there are no limits on samples ingestion rate.
-maxInsertRequestSize size
The maximum size in bytes of a single Prometheus remote_write API request
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432)

View File

@@ -133,8 +133,6 @@ The helm chart repository [https://github.com/VictoriaMetrics/helm-charts/](http
### Bump the version of images
> Note that helm charts versioning uses its own versioning scheme. The version of the charts not tied to the version of VictoriaMetrics components.
Bump `tag` field in `values.yaml` with new release version.
Bump `appVersion` field in `Chart.yaml` with new release version.
Add new line to "Next release" section in `CHANGELOG.md` about version update (the line must always start with "`-`"). Do **NOT** change headers in `CHANGELOG.md`.
@@ -159,8 +157,6 @@ Once updated, run the following commands:
## Ansible Roles
> Note that ansible playbooks versioning uses its own versioning scheme. The version of the playbooks is not tied to the version of VictoriaMetrics components.
1. Update the version of VictoriaMetrics components at [https://github.com/VictoriaMetrics/ansible-playbooks](https://github.com/VictoriaMetrics/ansible-playbooks).
1. Commit changes.
1. Create a new tag with `git tag -sm <TAG> <TAG>`.

View File

@@ -136,7 +136,7 @@ If you see unexpected or unreliable query results from VictoriaMetrics, then try
Note that responses returned from [/api/v1/query](https://docs.victoriametrics.com/keyconcepts/#instant-query)
and from [/api/v1/query_range](https://docs.victoriametrics.com/keyconcepts/#range-query) contain **evaluated** data
instead of raw samples stored in VictoriaMetrics. See [these docs](https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness)
for details. The raw samples can be also viewed in [vmui](https://docs.victoriametrics.com/#vmui) in `Raw Query` tab and shared via `export` button.
for details. The raw samples can be also viewed in [vmui](https://docs.victoriametrics.com/#vmui) in `Raw Query` tab.
If you migrate from InfluxDB, then pass `-search.setLookbackToStep` command-line flag to single-node VictoriaMetrics
or to `vmselect` in VictoriaMetrics cluster. See also [how to migrate from InfluxDB to VictoriaMetrics](https://docs.victoriametrics.com/guides/migrate-from-influx.html).

View File

@@ -16,29 +16,9 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
## tip
* FEATURE: [Datadog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/): added `-datadog.streamFields` and `-datadog.ignoreFields` flags to configured default stream and ignore fields. Useful for Datadog serverless plugin, which doesn't allow to provide extra headers of query args.
* BUGFIX: [Datadog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/datadog-agent/): accepts `message` field as both string and object type to fix compatibility with Datadog serverless extension, which sends logs data in format, which is not documented. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7761).
* BUGFIX: [vlinsert](https://docs.victoriametrics.com/victorialogs/): order of VL-Msg-Field values now defines a priority of these fields and it's now obvious for a user which field will be picked if multiple msg_field values exist in a row.
## [v1.4.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.4.0-victorialogs)
Released at 2024-12-22
* FEATURE: [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe): allow non-numeric field values at [`median`](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats) and [`quantile`](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats) stats functions.
* FEATURE: improve performance of [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) and [`top` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#top-pipe) by up to 2x when these pipes are applied to logs with millions of unique `by (...)` groups.
* FEATURE: [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe): add [`count_uniq_hash`](https://docs.victoriametrics.com/victorialogs/logsql/#count_uniq_hash-stats) function, which counts the number of unique value hashes. This number is usually a good approximation to the number of unique values, so the `count_uniq_hash` can be used as a faster alternative to [`count_uniq`](https://docs.victoriametrics.com/victorialogs/logsql/#count_uniq-stats).
* FEATURE: [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe): improve performance of [`count_uniq`](https://docs.victoriametrics.com/victorialogs/logsql/#count_uniq-stats) and [`uniq_values`](https://docs.victoriametrics.com/victorialogs/logsql/#uniq_values-stats) functions when they are applied to fields with big number of unique values.
* FEATURE: [`facets` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#facets-pipe): add an ability to return log fields with the same values across all the selected logs by adding `keep_const_fields` option. Such log fields aren't interesting in most cases, so they aren't returned by default.
* FEATURE: [`in` filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter): improve performance for `in(<query>)` when the `<query>` returns big number of values.
* FEATURE: [HTTP querying APIs](https://docs.victoriametrics.com/victorialogs/querying/#http-api): allow passing arbitrary [LogsQL filters](https://docs.victoriametrics.com/victorialogs/logsql/#filters) to `extra_filters` and `extra_stream_filters` query args. See [these docs](https://docs.victoriametrics.com/victorialogs/querying/#extra-filters) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5542) for details.
* FEATURE: [Grafana Loki data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/): add support of Loki healthcheck `/insert/ready` endpoint. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7824).
* FEATURE: [`stream_context` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stream_context-pipe): return an error as soon as too many logs and/or [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) are passed to this pipe. This prevents from excess resource usage by the `stream_context` pipe when it is improperly used. It is expected that the results of this pipe are investigated by humans, who cannot inspect surrounding logs for millions of the logs passed to `stream_context`. This change addresses [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7903) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7766) issues.
* BUGFIX: [syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): correctly parse rows with multiple consecutive spaces between fields. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7776).
* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix cursor reset in query input field. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7288).
* BUGFIX: [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe): fix improper sorting of numeric fields in some cases.
* BUGFIX: properly return an empty minimum value from [`min` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#min-stats).
## [v1.3.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.3.2-victorialogs)

View File

@@ -1715,13 +1715,6 @@ For example, the following query returns most frequently values for all the log
_time:1h error | facets max_value_len 100
```
Be default `facets` pipe doesn't return log fields, which contain a single constant value across all the selected logs, since such facets aren't interesting in most cases.
Add `keep_const_fields` suffix to the `facets` pipe in order to get such fields:
```logsql
_time:1h error | facets keep_const_fields
```
See also:
- [`top`](#top-pipe)
@@ -2668,7 +2661,7 @@ See also:
with the maximum number of matching log entries.
For example, the following query returns top 7 [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
with the maximum number of log entries over the last 5 minutes. The number of entries are returned in the `hits` field:
with the maximum number of log entries over the last 5 minutes:
```logsql
_time:5m | top 7 by (_stream)
@@ -2689,12 +2682,6 @@ For example, the following query is equivalent to the previous one:
_time:5m | fields ip | top
```
It is possible to give another name for the `hits` field via `hits as <new_name>` syntax. For example, the following query returns top per-`path` hits in the `visits` field:
```logsql
_time:5m | top by (path) hits as visits
```
It is possible to set `rank` field per each returned entry for `top` pipe by adding `with rank`. For example, the following query sets the `rank` field per each returned `ip`:
```logsql
@@ -3060,9 +3047,9 @@ LogsQL supports the following functions for [`stats` pipe](#stats-pipe):
- [`count_uniq`](#count_uniq-stats) returns the number of unique non-empty values for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`count_uniq_hash`](#count_uniq_hash-stats) returns the number of unique hashes for non-empty values at the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`max`](#max-stats) returns the maximum value over the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`median`](#median-stats) returns the [median](https://en.wikipedia.org/wiki/Median) value over the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`median`](#median-stats) returns the [median](https://en.wikipedia.org/wiki/Median) value over the given numeric [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`min`](#min-stats) returns the minimum value over the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`quantile`](#quantile-stats) returns the given quantile for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`quantile`](#quantile-stats) returns the given quantile for the given numeric [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`rate`](#rate-stats) returns the average per-second rate of matching logs on the selected time range.
- [`rate_sum`](#rate_sum-stats) returns the average per-second rate of sum for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
- [`row_any`](#row_any-stats) returns a sample [log entry](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) per each selected [stats group](#stats-by-fields).
@@ -3180,7 +3167,7 @@ See also:
- [`uniq_values`](#uniq_values-stats)
- [`count`](#count-stats)
### count_uniq_hash stats
### count_uniq_hash
`count_uniq_hash(field1, ..., fieldN)` [stats pipe function](#stats-pipe-functions) calculates the number of unique hashes for non-empty `(field1, ..., fieldN)` tuples.
This is a good estimation for the number of unique values in general case, while it works faster and uses less memory than [`count_uniq`](#count_uniq-stats)
@@ -3229,8 +3216,8 @@ See also:
### median stats
`median(field1, ..., fieldN)` [stats pipe function](#stats-pipe-functions) calculates the estimated [median](https://en.wikipedia.org/wiki/Median) value across
the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
`median(field1, ..., fieldN)` [stats pipe function](#stats-pipe-functions) calculates the [median](https://en.wikipedia.org/wiki/Median) value across
the give numeric [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model).
For example, the following query return median for the `duration` [field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
over logs for the last 5 minutes:
@@ -3267,7 +3254,7 @@ See also:
### quantile stats
`quantile(phi, field1, ..., fieldN)` [stats pipe function](#stats-pipe-functions) calculates an estimated `phi` [percentile](https://en.wikipedia.org/wiki/Percentile) over values
`quantile(phi, field1, ..., fieldN)` [stats pipe function](#stats-pipe-functions) calculates `phi` [percentile](https://en.wikipedia.org/wiki/Percentile) over numeric values
for the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model). The `phi` must be in the range `0 ... 1`, where `0` means `0th` percentile,
while `1` means `100th` percentile.

View File

@@ -33,8 +33,8 @@ Just download archive for the needed Operating system and architecture, unpack i
For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
```sh
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.4.0-victorialogs/victoria-logs-linux-amd64-v1.4.0-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v1.4.0-victorialogs.tar.gz
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.3.2-victorialogs/victoria-logs-linux-amd64-v1.3.2-victorialogs.tar.gz
tar xzf victoria-logs-linux-amd64-v1.3.2-victorialogs.tar.gz
./victoria-logs-prod
```
@@ -58,7 +58,7 @@ Here is the command to run VictoriaLogs in a Docker container:
```sh
docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
docker.io/victoriametrics/victoria-logs:v1.4.0-victorialogs
docker.io/victoriametrics/victoria-logs:v1.3.2-victorialogs
```
See also:

View File

@@ -20,17 +20,7 @@ unauthorized_user:
url_map:
- src_paths:
- "/api/v2/logs"
- "/api/v1/validate"
url_prefix: `<victoria-logs-base-url>`/insert/datadog/
- src_paths:
- "/api/v1/series"
- "/api/v2/series"
- "/api/beta/sketches"
- "/api/v1/validate"
- "/api/v1/check_run"
- "/intake"
- "/api/v1/metadata"
url_prefix: `<victoria-metrics-base-url>`/datadog/
url_prefix: "`<victoria-logs-base-url>`/insert/datadog/"
```
To start ingesting logs from DataDog agent please specify a custom URL instead of default one for sending collected logs to [VictoriaLogs](https://docs.victoriametrics.com/VictoriaLogs/):
@@ -50,7 +40,7 @@ custom:
apiKey: fakekey # Set any key, otherwise plugin fails
provider:
environment:
DD_DD_URL: `<vmauth-base-url>`/ # VMAuth endpoint for DataDog
LOGS_DD_URL: `<vmauth-base-url>`/ # VictoriaLogs endpoint for DataDog
```
Substitute the `<vmauth-base-url>` address with the real address of VMAuth proxy.
@@ -60,4 +50,3 @@ See also:
- [Data ingestion troubleshooting](https://docs.victoriametrics.com/victorialogs/data-ingestion/#troubleshooting).
- [How to query VictoriaLogs](https://docs.victoriametrics.com/victorialogs/querying/).
- [Docker-compose demo for Datadog integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/datadog-agent).
- [Docker-compose demo for Datadog Serverless integration with VictoriaLogs](https://github.com/VictoriaMetrics/VictoriaMetrics/tree/master/deployment/docker/victorialogs/datadog-serverless).

View File

@@ -11,9 +11,6 @@ aliases:
- /victorialogs/data-ingestion/Filebeat.html
- /victorialogs/data-ingestion/filebeat.html
---
_Tested with filebeat [8.15.1+](https://www.elastic.co/guide/en/beats/libbeat/8.17/release-notes-8.15.1.html.)._
Specify [`output.elasticsearch`](https://www.elastic.co/guide/en/beats/filebeat/current/elasticsearch-output.html) section in the `filebeat.yml`
for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):

View File

@@ -226,7 +226,7 @@ All the [HTTP-based data ingestion protocols](#http-apis) support the following
- `ignore_fields` - an optional comma-separated list of [log field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model) names,
which must be ignored during data ingestion.
- `extra_fields` - an optional comma-separated list of [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
- `extra_fields` - an optional comma-separated list [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model),
which must be added to all the ingested logs. The format of every `extra_fields` entry is `field_name=field_value`.
If the log entry contains fields from the `extra_fields`, then they are overwritten by the values specified in `extra_fields`.

View File

@@ -11,11 +11,10 @@ aliases:
- /victorialogs/data-ingestion/Vector.html
- /victorialogs/data-ingestion/vector.html
---
VictoriaLogs can accept logs from [Vector](https://vector.dev/) via the following protocols:
- Elasticsearch - see [these docs](#elasticsearch)
- HTTP JSON - see [these docs](#http)
VictoriaLogs supports given below Vector sinks:
- [Elasticsearch](#elasticsearch)
- [Loki](#loki)
- [HTTP JSON](#http)
## Elasticsearch
@@ -30,8 +29,8 @@ sinks:
type: elasticsearch
endpoints:
- http://localhost:9428/insert/elasticsearch/
mode: bulk
api_version: v8
compression: gzip
healthcheck:
enabled: false
query:
@@ -40,10 +39,30 @@ sinks:
_stream_fields: host,container_name
```
Replace `your_input` with the name of the `inputs` section, which collects logs. See [these docs](https://vector.dev/docs/reference/configuration/sources/) for details.
## Loki
Specify [Loki sink type](https://vector.dev/docs/reference/configuration/sinks/loki/) in the `vector.yaml`
for sending the collected logs to [VictoriaLogs](https://docs.victoriametrics.com/victorialogs/):
```yaml
sinks:
vlogs:
type: "loki"
endpoint: "http://localhost:9428/insert/loki/"
inputs:
- your_input
compression: gzip
path: /api/v1/push?_msg_field=message.message&_time_field=timestamp&_stream_fields=source
encoding:
codec: json
labels:
source: vector
```
Substitute the `localhost:9428` address inside `endpoints` section with the real TCP address of VictoriaLogs.
Replace `your_input` with the name of the `inputs` section, which collects logs. See [these docs](https://vector.dev/docs/reference/configuration/sources/) for details.
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on parameters specified
in the `sinks.vlogs.query` section.
@@ -60,8 +79,8 @@ sinks:
type: elasticsearch
endpoints:
- http://localhost:9428/insert/elasticsearch/
mode: bulk
api_version: v8
compression: gzip
healthcheck:
enabled: false
query:
@@ -83,15 +102,60 @@ sinks:
type: elasticsearch
endpoints:
- http://localhost:9428/insert/elasticsearch/
mode: bulk
api_version: v8
compression: gzip
healthcheck:
enabled: false
query:
_msg_field: message
_time_field: timestamp
_stream_fields: host,container_name
ignore_fields: log.offset,event.original
_ignore_fields: log.offset,event.original
```
When Vector ingests logs into VictoriaLogs at a high rate, then it may be needed to tune `batch.max_events` option.
For example, the following config is optimized for higher than usual ingestion rate:
```yaml
sinks:
vlogs:
inputs:
- your_input
type: elasticsearch
endpoints:
- http://localhost:9428/insert/elasticsearch/
mode: bulk
api_version: v8
healthcheck:
enabled: false
query:
_msg_field: message
_time_field: timestamp
_stream_fields: host,container_name
batch:
max_events: 1000
```
If the Vector sends logs to VictoriaLogs in another datacenter, then it may be useful enabling data compression via `compression = "gzip"` option.
This usually allows saving network bandwidth and costs by up to 5 times:
```yaml
sinks:
vlogs:
inputs:
- your_input
type: elasticsearch
endpoints:
- http://localhost:9428/insert/elasticsearch/
mode: bulk
api_version: v8
healthcheck:
enabled: false
compression: gzip
query:
_msg_field: message
_time_field: timestamp
_stream_fields: host,container_name
```
By default, the ingested logs are stored in the `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/keyconcepts/#multitenancy).
@@ -122,8 +186,8 @@ sinks:
## HTTP
Vector can be configured with [HTTP sink type](https://vector.dev/docs/reference/configuration/sinks/http/)
for sending data to VictoriaLogs via [JSON stream API](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api) format:
Vector can be configured with [HTTP](https://vector.dev/docs/reference/configuration/sinks/http/) sink type
for sending data to [JSON stream API](https://docs.victoriametrics.com/victorialogs/data-ingestion/#json-stream-api):
```yaml
sinks:
@@ -132,40 +196,16 @@ sinks:
- your_input
type: http
uri: http://localhost:9428/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp
compression: gzip
encoding:
codec: json
framing:
method: newline_delimited
healthcheck:
enabled: false
```
Replace `your_input` with the name of the `inputs` section, which collects logs. See [these docs](https://vector.dev/docs/reference/configuration/sources/) for details.
Substitute the `localhost:9428` address inside `endpoints` section with the real TCP address of VictoriaLogs.
See [these docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) for details on parameters specified
in the query args of the uri (`_stream_fields`, `_msg_field` and `_time_field`).
It is recommended verifying whether the initial setup generates the needed [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
and uses the correct [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
This can be done by specifying `debug` [query arg](https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters) in the `uri`:
```yaml
sinks:
vlogs:
inputs:
- your_input
type: http
uri: http://localhost:9428/insert/jsonline?_stream_fields=host,container_name&_msg_field=message&_time_field=timestamp&debug=1
compression: gzip
encoding:
codec: json
framing:
method: newline_delimited
healthcheck:
enabled: false
request:
headers:
AccountID: "12"
ProjectID: "34"
```
See also:

View File

@@ -393,13 +393,6 @@ returns the most frequent values across log fields containing values no longer t
curl http://localhost:9428/select/logsql/facets -d 'query=_time:1h' -d 'max_value_len=100'
```
By default the `/select/logsql/facets` endpoint doesn't return log fields, which contan the same constant value across all the logs matching the given `query`.
Add `keep_const_fields=1` query arg if you need such log fields:
```sh
curl http://localhost:9428/select/logsql/facets -d 'query=_time:1h' -d 'keep_const_fields=1'
```
See also:
- [Extra filters](#extra-filters)
@@ -917,26 +910,22 @@ See also:
All the [HTTP querying APIs](#http-api) provided by VictoriaLogs support the following optional query args:
- `extra_filters` - this arg may contain extra [filters](https://docs.victoriametrics.com/victorialogs/logsql/#filters), which must be applied
- `extra_filters` - this arg may contain extra [`field:=value`](https://docs.victoriametrics.com/victorialogs/logsql/#exact-filter) filters, which must be applied
to the `query` before returning the results.
- `extra_stream_filters` - this arg may contain extra [stream filters](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter),
- `extra_stream_filters` - this arg may contain extra [`{field="value"}`](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter) filters,
which must be applied to the `query` before returning results.
The `extra_filters` and `extra_stream_filters` values can have the following format:
The filters must be passed as JSON object with `"field":"value"` entries. For example, the following JSON object applies `namespace:=my-app and env:prod` filter to the `query`
passed to [HTTP querying APIs](#http-api):
- JSON object with `"field":"value"` entries. For example, the following JSON applies `namespace:=my-app and env:=prod` filter to the `query`
passed to [HTTP querying APIs](#http-api): `extra_filters={"namespace":"my-app","env":"prod"}` .
```json
{
"namespace":"my-app",
"env":"prod"
}
```
The following JSON applies `{namespace="my-app",env="prod"}` [stream filter](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter)
to the `query`: `extra_stream_filters={"namespace":"my-app","env":"prod"}` .
Every JSON entry may contain either a single string value or an array of values. An array of `{"field:["v1","v2",..."vN"]}` values is converted
into `field:in(v1, v2, ... vN)` [filter](https://docs.victoriametrics.com/victorialogs/logsql/#multi-exact-filter) when passed to `extra_filters`.
The same array is converted into `{field=~"v1|v2|...|vN"}` [stream filter](https://docs.victoriametrics.com/victorialogs/logsql/#stream-filter).
- Arbitrary [LogsQL filter](https://docs.victoriametrics.com/victorialogs/logsql/#filters). For example, `extra_filters=foo:~bar -baz:x`.
The arg passed to `extra_filters` and `extra_stream_filters` must be properly encoded with [percent encoding](https://en.wikipedia.org/wiki/Percent-encoding).
The JSON object must be properly encoded with [percent encoding](https://en.wikipedia.org/wiki/Percent-encoding) before being passed to VictoriaLogs.
## Web UI

View File

@@ -23,15 +23,15 @@ or from [docker images](https://hub.docker.com/r/victoriametrics/vlogscli/tags):
### Running `vlogscli` from release binary
```sh
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.4.0-victorialogs/vlogscli-linux-amd64-v1.4.0-victorialogs.tar.gz
tar xzf vlogscli-linux-amd64-v1.4.0-victorialogs.tar.gz
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.3.2-victorialogs/vlogscli-linux-amd64-v1.3.2-victorialogs.tar.gz
tar xzf vlogscli-linux-amd64-v1.3.2-victorialogs.tar.gz
./vlogscli-prod
```
### Running `vlogscli` from Docker image
```sh
docker run --rm -it docker.io/victoriametrics/vlogscli:v1.4.0-victorialogs
docker run --rm -it docker.io/victoriametrics/vlogscli:v1.3.2-victorialogs
```
## Configuration

View File

@@ -31,7 +31,7 @@ the following changes to Grafana's `grafana.ini` config:
``` ini
[plugins]
allow_loading_unsigned_plugins = victoriametrics-logs-datasource
allow_loading_unsigned_plugins = victorialogs-datasource
```
If using `grafana-operator`, adjust `config` section in your `kind=Grafana` resource as below:
@@ -39,7 +39,7 @@ If using `grafana-operator`, adjust `config` section in your `kind=Grafana` reso
```
config:
plugins:
allow_loading_unsigned_plugins: "victoriametrics-logs-datasource"
allow_loading_unsigned_plugins: "victorialogs-datasource"
```
For detailed instructions on how to install the plugin in Grafana Cloud or locally,
@@ -56,7 +56,7 @@ datasources:
# displayed in Grafana panels and queries.
- name: VictoriaLogs
# <string, required> Sets the data source type.
type: victoriametrics-logs-datasource
type: victorialogs-datasource
# <string, required> Sets the access mode, either
# proxy or direct (Server or Browser in the UI).
access: proxy
@@ -80,8 +80,8 @@ Please find the example of provisioning Grafana instance with VictoriaLogs datas
grafana:
image: grafana/grafana:11.0.0
environment:
- GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.12.0/victoriametrics-logs-datasource-v0.12.0.zip;victoriametrics-logs-datasource
- GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victoriametrics-logs-datasource
- GF_INSTALL_PLUGINS=https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.12.0/victorialogs-datasource-v0.12.0.zip;victorialogs-datasource
- GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=victorialogs-datasource
ports:
- 3000:3000/tcp
volumes:
@@ -108,15 +108,15 @@ Option 1. Using Grafana provisioning:
``` yaml
env:
GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.12.0/victoriametrics-logs-datasource-v0.12.0.zip;victoriametrics-logs-datasource"
GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "victoriametrics-logs-datasource"
GF_INSTALL_PLUGINS: "https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.12.0/victorialogs-datasource-v0.12.0.zip;victorialogs-datasource"
GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "victorialogs-datasource"
```
Option 2. Using Grafana plugins section in `values.yaml`:
``` yaml
plugins:
- https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.12.0/victoriametrics-logs-datasource-v0.12.0.zip;victoriametrics-logs-datasource
- https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.12.0/victorialogs-datasource-v0.12.0.zip;victorialogs-datasource
```
Option 3. Using init container:
@@ -137,7 +137,7 @@ extraInitContainers:
set -ex
mkdir -p /var/lib/grafana/plugins/
ver=$(curl -s -L https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victoriametrics-logs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
rm /var/lib/grafana/plugins/vl-plugin.tar.gz
volumeMounts:
@@ -197,7 +197,7 @@ spec:
set -ex
mkdir -p /var/lib/grafana/plugins/
ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victoriametrics-logs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
rm /var/lib/grafana/plugins/vl-plugin.tar.gz
volumeMounts:
@@ -205,7 +205,7 @@ spec:
mountPath: /var/lib/grafana
config:
plugins:
allow_loading_unsigned_plugins: victoriametrics-logs-datasource
allow_loading_unsigned_plugins: victorialogs-datasource
```
See [Grafana operator reference](https://grafana-operator.github.io/grafana-operator/docs/grafana/) to find more about Grafana operator.
@@ -217,7 +217,7 @@ This example uses init container to download and install plugin.
``` bash
ver=$(curl -s https://api.github.com/repos/VictoriaMetrics/victorialogs-datasource/releases/latest | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+' | head -1)
curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victoriametrics-logs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
curl -L https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/$ver/victorialogs-datasource-$ver.tar.gz -o /var/lib/grafana/plugins/vl-plugin.tar.gz
tar -xf /var/lib/grafana/plugins/vl-plugin.tar.gz -C /var/lib/grafana/plugins/
rm /var/lib/grafana/plugins/vl-plugin.tar.gz
```
@@ -237,7 +237,7 @@ plugins = {{path to directory with plugin}}
``` ini
[plugins]
allow_loading_unsigned_plugins = victoriametrics-logs-datasource
allow_loading_unsigned_plugins = victorialogs-datasource
```
### 2. Run the plugin
@@ -251,7 +251,7 @@ yarn install
# run the app in the development mode
yarn dev
# build the plugin for production to the `victoriametrics-logs-datasource` folder and zip build
# build the plugin for production to the `victorialogs-datasource` folder and zip build
yarn build:zip
```
@@ -260,10 +260,10 @@ yarn build:zip
From the root folder of the project run the following command:
```
make vl-backend-plugin-build
make victorialogs-backend-plugin-build
```
This command will build executable multi-platform files to the `victoriametrics-logs-datasource` folder for the following platforms:
This command will build executable multi-platform files to the `victorialogs-datasource` folder for the following platforms:
* linux/amd64
* linux/arm64
@@ -278,20 +278,20 @@ This command will build executable multi-platform files to the `victoriametrics-
From the root folder of the project run the following command:
```
make vl-frontend-plugin-build
make victorialogs-frontend-plugin-build
```
This command will build all frontend app into `victoriametrics-logs-datasource` folder.
This command will build all frontend app into `victorialogs-datasource` folder.
### 5. How to build frontend and backend parts of the plugin:
When frontend and backend parts of the plugin is required, run the following command from the root folder of the project:
```
make vl-plugin-build
make victorialogs-datasource-plugin-build
```
This command will build frontend part and backend part or the plugin and locate both parts into `victoriametrics-logs-datasource` folder.
This command will build frontend part and backend part or the plugin and locate both parts into `victorialogs-datasource` folder.
## How to make new release

View File

@@ -98,9 +98,9 @@ groups:
interval: 5m
rules:
- alert: HasErrorLog
expr: 'env: "prod" AND status:~"error|warn" | stats by (service, kubernetes.pod) count() as errorLog | filter errorLog:>0'
expr: 'env: "prod" AND status:~"error|warn" | stats by (service) count() as errorLog | filter errorLog:>0'
annotations:
description: 'Service {{$labels.service}} (pod {{ index $labels "kubernetes.pod" }}) generated {{$labels.errorLog}} error logs in the last 5 minutes'
description: "Service {{$labels.service}} generated {{$labels.errorLog}} error logs in the last 5 minutes"
- name: ServiceRequest
type: vlogs
@@ -216,91 +216,32 @@ For additional tips on writing LogsQL, refer to this [doc](https://docs.victoria
## Frequently Asked Questions
### How to use [multitenancy](https://docs.victoriametrics.com/victorialogs/#multitenancy) in rules?
vmalert doesn't support multi-tenancy for VictoriaLogs in the same way as it [supports it for VictoriaMetrics in ENT version](https://docs.victoriametrics.com/vmalert/#multitenancy).
However, it is possible to specify the queried tenant from VictoriaLogs datasource via `headers` param in [Group config](https://docs.victoriametrics.com/vmalert/#groups).
For example, the following config will execute all the rules within the group against tenant with `AccountID=1` and `ProjectID=2`:
```yaml
groups:
- name: MyGroup
headers:
- "AccountID: 1"
- "ProjectID: 2"
rules: ...
```
By default, vmalert persists all results to the specific tenant in VictoriaMetrics that specified by `-remotewrite.url`. For example, if the `-remotewrite.url=http://vminsert:8480/insert/0/prometheus/`, all data goes to tenant `0`.
To persist different rule results to different tenants in VictoriaMetrics, there are following approaches:
1. To use the [multitenant endpoint of vminsert](https://docs.victoriametrics.com/cluster-victoriametrics/#multitenancy-via-labels) as the `-remoteWrite.url`, and add tenant labels under the group configuration.
For example, run vmalert with:
* How to use [multitenancy](https://docs.victoriametrics.com/victorialogs/#multitenancy) in rules?
* vmalert doesn't support multi-tenancy for VictoriaLogs in the same way as it [supports it for VictoriaMetrics in ENT version](https://docs.victoriametrics.com/vmalert/#multitenancy).
However, it is possible to specify the queried tenant from VictoriaLogs datasource via `headers` param in [Group config](https://docs.victoriametrics.com/vmalert/#groups).
For example, the following config will execute all the rules within the group against tenant with `AccountID=1` and `ProjectID=2`:
```yaml
groups:
- name: MyGroup
headers:
- "AccountID: 1"
- "ProjectID: 2"
rules: ...
```
./bin/vmalert -datasource.url=http://localhost:9428 -remoteWrite.url=http://vminsert:8480/insert/multitenant/prometheus ...
* How to use one vmalert for VictoriaLogs and VictoriaMetrics rules in the same time?
* We recommend running separate instances of vmalert for VictoriaMetrics and VictoriaLogs.
However, vmalert allows having many groups with different rule types (`vlogs`, `prometheus`, `graphite`).
But only one `-datasource.url` cmd-line flag can be specified, so it can't be configured with more than 1 datasource.
However, VictoriaMetrics and VictoriaLogs datasources have different query path prefixes, and it is possible to use [vmauth](https://docs.victoriametrics.com/vmauth/) to route requests of different types between datasources.
See example of vmauth config for such routing below:
```yaml
unauthorized_user:
url_map:
- src_paths:
- "/api/v1/query.*"
url_prefix: "http://victoriametrics:8428"
- src_paths:
- "/select/logsql/.*"
url_prefix: "http://victorialogs:9428"
```
With the rules below, `recordingTenant123` will be queried from VictoriaLogs tenant `123` and persisted to tenant `123` in VictoriaMetrics, while `recordingTenant123-456:789` will be queried from VictoriaLogs tenant `124` and persisted to tenant `456:789` in VictoriaMetrics.
```
groups:
- name: recordingTenant123
type: vlogs
headers:
- "AccountID: 123"
labels:
vm_account_id: 123
rules:
- record: recordingTenant123
expr: 'tags.path:/var/log/httpd OR tags.path:/var/log/nginx | stats by (tags.host) count() requests'
- name: recordingTenant124-456:789
type: vlogs
headers:
- "AccountID: 124"
labels:
vm_account_id: 456
vm_project_id: 789
rules:
- record: recordingTenant124-456:789
expr: 'tags.path:/var/log/httpd OR tags.path:/var/log/nginx | stats by (tags.host) count() requests'
```
2. To run [enterprise version of vmalert](https://docs.victoriametrics.com/enterprise/) with `-clusterMode` enabled, and specify tenant parameter per each group.
For example, run vmalert with:
```
./bin/vmalert -datasource.url=http://localhost:9428 -clusterMode=true -remoteWrite.url=http://vminsert:8480/ ...
```
With the rules below, `recordingTenant123` will be queried from VictoriaLogs tenant `123` and persisted to tenant `123` in VictoriaMetrics, while `recordingTenant123-456:789` will be queried from VictoriaLogs tenant `124` and persisted to tenant `456:789` in VictoriaMetrics.
```
groups:
- name: recordingTenant123
type: vlogs
headers:
- "AccountID: 123"
tenant: "123"
rules:
- record: recordingTenant123
expr: 'tags.path:/var/log/httpd OR tags.path:/var/log/nginx | stats by (tags.host) count() requests'
- name: recordingTenant124-456:789
type: vlogs
headers:
- "AccountID: 124"
tenant: "456:789"
rules:
- record: recordingTenant124-456:789
expr: 'tags.path:/var/log/httpd OR tags.path:/var/log/nginx | stats by (tags.host) count() requests'
```
### How to use one vmalert for VictoriaLogs and VictoriaMetrics rules in the same time?
We recommend running separate instances of vmalert for VictoriaMetrics and VictoriaLogs.
However, vmalert allows having many groups with different rule types (`vlogs`, `prometheus`, `graphite`).
But only one `-datasource.url` cmd-line flag can be specified, so it can't be configured with more than 1 datasource.
However, VictoriaMetrics and VictoriaLogs datasources have different query path prefixes, and it is possible to use [vmauth](https://docs.victoriametrics.com/vmauth/) to route requests of different types between datasources.
See example of vmauth config for such routing below:
```yaml
unauthorized_user:
url_map:
- src_paths:
- "/api/v1/query.*"
url_prefix: "http://victoriametrics:8428"
- src_paths:
- "/select/logsql/.*"
url_prefix: "http://victorialogs:9428"
```
Now, vmalert needs to be configured with `--datasource.url=http://vmauth:8427/` to send queries to vmauth, and vmauth will route them to the specified destinations as in configuration example above.
Now, vmalert needs to be configured with `--datasource.url=http://vmauth:8427/` to send queries to vmauth, and vmauth will route them to the specified destinations as in configuration example above.

Some files were not shown because too many files have changed in this diff Show More