mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2026-06-07 19:06:17 +03:00
Compare commits
1 Commits
docs/trace
...
vlogs-grac
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ed784e8c51 |
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlselect"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
|
||||
@@ -45,8 +44,6 @@ func main() {
|
||||
|
||||
vlstorage.Init()
|
||||
vlselect.Init()
|
||||
|
||||
insertutil.SetLogRowsStorage(&vlstorage.Storage{})
|
||||
vlinsert.Init()
|
||||
|
||||
go httpserver.Serve(listenAddrs, requestHandler, httpserver.ServeOptions{
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
@@ -32,10 +33,10 @@ var parserPool fastjson.ParserPool
|
||||
// RequestHandler processes Datadog insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
case "/insert/datadog/api/v1/validate":
|
||||
case "/api/v1/validate":
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
case "/insert/datadog/api/v2/logs":
|
||||
case "/api/v2/logs":
|
||||
return datadogLogsIngestion(w, r)
|
||||
default:
|
||||
return false
|
||||
@@ -73,7 +74,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
||||
cp.IgnoreFields = *datadogIgnoreFields
|
||||
}
|
||||
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
@@ -101,7 +102,7 @@ func datadogLogsIngestion(w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
var (
|
||||
v2LogsRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/datadog/api/v2/logs"}`)
|
||||
v2LogsRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
|
||||
v2LogsRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/datadog/api/v2/logs"}`)
|
||||
)
|
||||
|
||||
// datadog message field has two formats:
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bufferedwriter"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
@@ -30,38 +31,36 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
// This header is needed for Logstash
|
||||
w.Header().Set("X-Elastic-Product", "Elasticsearch")
|
||||
|
||||
if strings.HasPrefix(path, "/insert/elasticsearch/_ilm/policy") {
|
||||
if strings.HasPrefix(path, "/_ilm/policy") {
|
||||
// Return fake response for Elasticsearch ilm request.
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(path, "/insert/elasticsearch/_index_template") {
|
||||
if strings.HasPrefix(path, "/_index_template") {
|
||||
// Return fake response for Elasticsearch index template request.
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(path, "/insert/elasticsearch/_ingest") {
|
||||
if strings.HasPrefix(path, "/_ingest") {
|
||||
// Return fake response for Elasticsearch ingest pipeline request.
|
||||
// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/put-pipeline-api.html
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(path, "/insert/elasticsearch/_nodes") {
|
||||
if strings.HasPrefix(path, "/_nodes") {
|
||||
// Return fake response for Elasticsearch nodes discovery request.
|
||||
// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/cluster.html
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(path, "/insert/elasticsearch/logstash") || strings.HasPrefix(path, "/insert/elasticsearch/_logstash") {
|
||||
if strings.HasPrefix(path, "/logstash") || strings.HasPrefix(path, "/_logstash") {
|
||||
// Return fake response for Logstash APIs requests.
|
||||
// See: https://www.elastic.co/guide/en/elasticsearch/reference/8.8/logstash-apis.html
|
||||
fmt.Fprintf(w, `{}`)
|
||||
return true
|
||||
}
|
||||
switch path {
|
||||
// some clients may omit trailing slash
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
|
||||
case "/insert/elasticsearch/", "/insert/elasticsearch":
|
||||
case "/", "":
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
// Return fake response for Elasticsearch ping request.
|
||||
@@ -76,7 +75,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
|
||||
return true
|
||||
case "/insert/elasticsearch/_license":
|
||||
case "/_license":
|
||||
// Return fake response for Elasticsearch license request.
|
||||
fmt.Fprintf(w, `{
|
||||
"license": {
|
||||
@@ -87,7 +86,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
}`)
|
||||
return true
|
||||
case "/insert/elasticsearch/_bulk":
|
||||
case "/_bulk":
|
||||
startTime := time.Now()
|
||||
bulkRequestsTotal.Inc()
|
||||
|
||||
@@ -96,7 +95,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return true
|
||||
}
|
||||
@@ -129,7 +128,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
|
||||
var (
|
||||
bulkRequestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/elasticsearch/_bulk"}`)
|
||||
bulkRequestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
|
||||
bulkRequestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/elasticsearch/_bulk"}`)
|
||||
)
|
||||
|
||||
func readBulkRequest(streamName string, r io.Reader, encoding string, timeFields, msgFields []string, lmp insertutil.LogMessageProcessor) (int, error) {
|
||||
|
||||
@@ -11,10 +11,12 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httputil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timeutil"
|
||||
)
|
||||
|
||||
@@ -35,7 +37,6 @@ type CommonParams struct {
|
||||
DecolorizeFields []string
|
||||
ExtraFields []logstorage.Field
|
||||
|
||||
IsTimeFieldSet bool
|
||||
Debug bool
|
||||
DebugRequestURI string
|
||||
DebugRemoteAddr string
|
||||
@@ -49,10 +50,8 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var isTimeFieldSet bool
|
||||
timeFields := []string{"_time"}
|
||||
if tfs := httputil.GetArray(r, "_time_field", "VL-Time-Field"); len(tfs) > 0 {
|
||||
isTimeFieldSet = true
|
||||
timeFields = tfs
|
||||
}
|
||||
|
||||
@@ -88,11 +87,9 @@ func GetCommonParams(r *http.Request) (*CommonParams, error) {
|
||||
IgnoreFields: ignoreFields,
|
||||
DecolorizeFields: decolorizeFields,
|
||||
ExtraFields: extraFields,
|
||||
|
||||
IsTimeFieldSet: isTimeFieldSet,
|
||||
Debug: debug,
|
||||
DebugRequestURI: debugRequestURI,
|
||||
DebugRemoteAddr: debugRemoteAddr,
|
||||
Debug: debug,
|
||||
DebugRequestURI: debugRequestURI,
|
||||
DebugRemoteAddr: debugRemoteAddr,
|
||||
}
|
||||
|
||||
return cp, nil
|
||||
@@ -145,29 +142,6 @@ func GetCommonParamsForSyslog(tenantID logstorage.TenantID, streamFields, ignore
|
||||
return cp
|
||||
}
|
||||
|
||||
// LogRowsStorage is an interface for ingesting logs into the storage.
|
||||
type LogRowsStorage interface {
|
||||
// MustAddRows must add lr to the underlying storage.
|
||||
MustAddRows(lr *logstorage.LogRows)
|
||||
|
||||
// CanWriteData must returns non-nil error if logs cannot be added to the underlying storage.
|
||||
CanWriteData() error
|
||||
}
|
||||
|
||||
var logRowsStorage LogRowsStorage
|
||||
|
||||
// SetLogRowsStorage sets the storage for writing data to via LogMessageProcessor.
|
||||
//
|
||||
// This function must be called before using LogMessageProcessor and CanWriteData from this package.
|
||||
func SetLogRowsStorage(storage LogRowsStorage) {
|
||||
logRowsStorage = storage
|
||||
}
|
||||
|
||||
// CanWriteData returns non-nil error if data cannot be written to the underlying storage.
|
||||
func CanWriteData() error {
|
||||
return logRowsStorage.CanWriteData()
|
||||
}
|
||||
|
||||
// LogMessageProcessor is an interface for log message processors.
|
||||
type LogMessageProcessor interface {
|
||||
// AddRow must add row to the LogMessageProcessor with the given timestamp and fields.
|
||||
@@ -182,6 +156,7 @@ type LogMessageProcessor interface {
|
||||
}
|
||||
|
||||
type logMessageProcessor struct {
|
||||
isStreamMode bool
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
stopCh chan struct{}
|
||||
@@ -210,6 +185,10 @@ func (lmp *logMessageProcessor) initPeriodicFlush() {
|
||||
case <-lmp.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
if vlstorage.CanWriteData() != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
lmp.mu.Lock()
|
||||
if time.Since(lmp.lastFlushTime) >= d {
|
||||
lmp.flushLocked()
|
||||
@@ -224,6 +203,10 @@ func (lmp *logMessageProcessor) initPeriodicFlush() {
|
||||
//
|
||||
// If streamFields is non-nil, then it is used as log stream fields instead of the pre-configured stream fields.
|
||||
func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []logstorage.Field) {
|
||||
if !lmp.canWriteData() {
|
||||
lmp.waitUntilStorageIsAvailable()
|
||||
}
|
||||
|
||||
lmp.rowsIngestedTotal.Inc()
|
||||
n := logstorage.EstimatedJSONRowLen(fields)
|
||||
lmp.bytesIngestedTotal.Add(n)
|
||||
@@ -252,6 +235,23 @@ func (lmp *logMessageProcessor) AddRow(timestamp int64, fields, streamFields []l
|
||||
}
|
||||
}
|
||||
|
||||
func (lmp *logMessageProcessor) waitUntilStorageIsAvailable() {
|
||||
t := timerpool.Get(time.Second)
|
||||
for lmp.canWriteData() {
|
||||
select {
|
||||
case <-lmp.stopCh:
|
||||
timerpool.Put(t)
|
||||
return
|
||||
case <-t.C:
|
||||
timerpool.Put(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (lmp *logMessageProcessor) canWriteData() bool {
|
||||
return !lmp.isStreamMode || vlstorage.CanWriteData() == nil
|
||||
}
|
||||
|
||||
// InsertRowProcessor is used by native data ingestion protocol parser.
|
||||
type InsertRowProcessor interface {
|
||||
// AddInsertRow must add r to the underlying storage.
|
||||
@@ -291,7 +291,7 @@ func (lmp *logMessageProcessor) AddInsertRow(r *logstorage.InsertRow) {
|
||||
// flushLocked must be called under locked lmp.mu.
|
||||
func (lmp *logMessageProcessor) flushLocked() {
|
||||
lmp.lastFlushTime = time.Now()
|
||||
logRowsStorage.MustAddRows(lmp.lr)
|
||||
vlstorage.MustAddRows(lmp.lr)
|
||||
lmp.lr.ResetKeepSettings()
|
||||
}
|
||||
|
||||
@@ -313,8 +313,9 @@ func (cp *CommonParams) NewLogMessageProcessor(protocolName string, isStreamMode
|
||||
rowsIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_rows_ingested_total{type=%q}", protocolName))
|
||||
bytesIngestedTotal := metrics.GetOrCreateCounter(fmt.Sprintf("vl_bytes_ingested_total{type=%q}", protocolName))
|
||||
lmp := &logMessageProcessor{
|
||||
cp: cp,
|
||||
lr: lr,
|
||||
isStreamMode: isStreamMode,
|
||||
cp: cp,
|
||||
lr: lr,
|
||||
|
||||
rowsIngestedTotal: rowsIngestedTotal,
|
||||
bytesIngestedTotal: bytesIngestedTotal,
|
||||
|
||||
@@ -56,7 +56,6 @@ func NewLineReader(name string, r io.Reader) *LineReader {
|
||||
// Check for Err in this case.
|
||||
func (lr *LineReader) NextLine() bool {
|
||||
for {
|
||||
lr.Line = nil
|
||||
if lr.bufOffset >= len(lr.buf) {
|
||||
if lr.err != nil || lr.eofReached {
|
||||
return false
|
||||
|
||||
@@ -24,9 +24,6 @@ func TestLineReader_Success(t *testing.T) {
|
||||
if lr.NextLine() {
|
||||
t.Fatalf("expecting error on the second call to NextLine()")
|
||||
}
|
||||
if len(lr.Line) > 0 {
|
||||
t.Fatalf("unexpected non-empty line after failed NextLine(): %q", lr.Line)
|
||||
}
|
||||
if !reflect.DeepEqual(lines, linesExpected) {
|
||||
t.Fatalf("unexpected lines\ngot\n%q\nwant\n%q", lines, linesExpected)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage/netinsert"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
@@ -39,7 +40,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
@@ -84,5 +85,5 @@ var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/internal/insert"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/internal/insert"}`)
|
||||
|
||||
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/internal/insert"}`)
|
||||
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/internal/insert"}`)
|
||||
)
|
||||
|
||||
@@ -3,30 +3,29 @@ package journald
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/writeconcurrencylimiter"
|
||||
"github.com/VictoriaMetrics/metrics"
|
||||
)
|
||||
|
||||
// See https://github.com/systemd/systemd/blob/main/src/libsystemd/sd-journal/journal-file.c#L1703
|
||||
const maxFieldNameLen = 64
|
||||
const journaldEntryMaxNameLen = 64
|
||||
|
||||
var allowedJournaldEntryNameChars = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*`)
|
||||
|
||||
var (
|
||||
journaldStreamFields = flagutil.NewArrayString("journald.streamFields", "Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. "+
|
||||
@@ -37,7 +36,9 @@ var (
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field")
|
||||
journaldTenantID = flag.String("journald.tenantID", "0:0", "TenantID for logs ingested via the Journald endpoint. "+
|
||||
"See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#multitenancy")
|
||||
journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include Journald fields with double underscore prefixes")
|
||||
journaldIncludeEntryMetadata = flag.Bool("journald.includeEntryMetadata", false, "Include journal entry fields, which with double underscores.")
|
||||
|
||||
maxRequestSize = flagutil.NewBytes("journald.maxRequestSize", 64*1024*1024, "The maximum size in bytes of a single journald request")
|
||||
)
|
||||
|
||||
func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
|
||||
@@ -52,12 +53,11 @@ func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
|
||||
}
|
||||
cp.TenantID = tenantID
|
||||
}
|
||||
|
||||
if !cp.IsTimeFieldSet {
|
||||
if len(cp.TimeFields) == 0 {
|
||||
cp.TimeFields = []string{*journaldTimeField}
|
||||
}
|
||||
if len(cp.StreamFields) == 0 {
|
||||
cp.StreamFields = getStreamFields()
|
||||
cp.StreamFields = *journaldStreamFields
|
||||
}
|
||||
if len(cp.IgnoreFields) == 0 {
|
||||
cp.IgnoreFields = *journaldIgnoreFields
|
||||
@@ -66,23 +66,10 @@ func getCommonParams(r *http.Request) (*insertutil.CommonParams, error) {
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
func getStreamFields() []string {
|
||||
if len(*journaldStreamFields) > 0 {
|
||||
return *journaldStreamFields
|
||||
}
|
||||
return defaultStreamFields
|
||||
}
|
||||
|
||||
var defaultStreamFields = []string{
|
||||
"_MACHINE_ID",
|
||||
"_HOSTNAME",
|
||||
"_SYSTEMD_UNIT",
|
||||
}
|
||||
|
||||
// RequestHandler processes Journald Export insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
case "/insert/journald/upload":
|
||||
case "/upload":
|
||||
if r.Header.Get("Content-Type") != "application/vnd.fdo.journal" {
|
||||
httpserver.Errorf(w, r, "only application/vnd.fdo.journal encoding is supported for Journald")
|
||||
return true
|
||||
@@ -97,7 +84,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
// handleJournald parses Journal binary entries
|
||||
func handleJournald(r *http.Request, w http.ResponseWriter) {
|
||||
startTime := time.Now()
|
||||
requestsTotal.Inc()
|
||||
requestsJournaldTotal.Inc()
|
||||
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
@@ -106,25 +93,19 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
|
||||
encoding := r.Header.Get("Content-Encoding")
|
||||
reader, err := protoparserutil.GetUncompressedReader(r.Body, encoding)
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
logger.Errorf("cannot decode journald request: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
lmp := cp.NewLogMessageProcessor("journald", true)
|
||||
streamName := fmt.Sprintf("remoteAddr=%s, requestURI=%q", httpserver.GetQuotedRemoteAddr(r), r.RequestURI)
|
||||
err = processStreamInternal(streamName, reader, lmp, cp)
|
||||
protoparserutil.PutUncompressedReader(reader)
|
||||
lmp.MustClose()
|
||||
err = protoparserutil.ReadUncompressedData(r.Body, encoding, maxRequestSize, func(data []byte) error {
|
||||
lmp := cp.NewLogMessageProcessor("journald", false)
|
||||
err := parseJournaldRequest(data, lmp, cp)
|
||||
lmp.MustClose()
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
errorsTotal.Inc()
|
||||
httpserver.Errorf(w, r, "cannot read journald protocol data: %s", err)
|
||||
@@ -136,185 +117,102 @@ func handleJournald(r *http.Request, w http.ResponseWriter) {
|
||||
// See https://github.com/systemd/systemd/pull/34822
|
||||
w.Header().Set("Accept-Encoding", "zstd")
|
||||
|
||||
// update requestDuration only for successfully parsed requests
|
||||
// There is no need in updating requestDuration for request errors,
|
||||
// update requestJournaldDuration only for successfully parsed requests
|
||||
// There is no need in updating requestJournaldDuration for request errors,
|
||||
// since their timings are usually much smaller than the timing for successful request parsing.
|
||||
requestDuration.UpdateDuration(startTime)
|
||||
requestJournaldDuration.UpdateDuration(startTime)
|
||||
}
|
||||
|
||||
var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
|
||||
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
|
||||
requestsJournaldTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/journald/upload"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/journald/upload"}`)
|
||||
|
||||
requestJournaldDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/journald/upload"}`)
|
||||
)
|
||||
|
||||
func processStreamInternal(streamName string, r io.Reader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
|
||||
wcr := writeconcurrencylimiter.GetReader(r)
|
||||
defer writeconcurrencylimiter.PutReader(wcr)
|
||||
|
||||
lr := insertutil.NewLineReader("journald", wcr)
|
||||
|
||||
for {
|
||||
err := readJournaldLogEntry(streamName, lr, lmp, cp)
|
||||
wcr.DecConcurrency()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s: %w", streamName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fieldsBuf struct {
|
||||
fields []logstorage.Field
|
||||
|
||||
buf []byte
|
||||
name []byte
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (fb *fieldsBuf) reset() {
|
||||
fb.fields = fb.fields[:0]
|
||||
fb.buf = fb.buf[:0]
|
||||
fb.name = fb.name[:0]
|
||||
fb.value = fb.value[:0]
|
||||
}
|
||||
|
||||
func (fb *fieldsBuf) addField(name, value string) {
|
||||
bufLen := len(fb.buf)
|
||||
fb.buf = append(fb.buf, name...)
|
||||
nameCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
|
||||
|
||||
bufLen = len(fb.buf)
|
||||
fb.buf = append(fb.buf, value...)
|
||||
valueCopy := bytesutil.ToUnsafeString(fb.buf[bufLen:])
|
||||
|
||||
fb.fields = append(fb.fields, logstorage.Field{
|
||||
Name: nameCopy,
|
||||
Value: valueCopy,
|
||||
})
|
||||
}
|
||||
|
||||
func (fb *fieldsBuf) appendNextLineToValue(lr *insertutil.LineReader) error {
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("unexpected end of stream")
|
||||
}
|
||||
fb.value = append(fb.value, lr.Line...)
|
||||
fb.value = append(fb.value, '\n')
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFieldsBuf() *fieldsBuf {
|
||||
fb := fieldsBufPool.Get()
|
||||
if fb == nil {
|
||||
return &fieldsBuf{}
|
||||
}
|
||||
return fb.(*fieldsBuf)
|
||||
}
|
||||
|
||||
func putFieldsBuf(fb *fieldsBuf) {
|
||||
fb.reset()
|
||||
fieldsBufPool.Put(fb)
|
||||
}
|
||||
|
||||
var fieldsBufPool sync.Pool
|
||||
|
||||
// readJournaldLogEntry reads a single log entry in Journald format.
|
||||
//
|
||||
// See https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
|
||||
func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
|
||||
func parseJournaldRequest(data []byte, lmp insertutil.LogMessageProcessor, cp *insertutil.CommonParams) error {
|
||||
var fields []logstorage.Field
|
||||
var ts int64
|
||||
var size uint64
|
||||
var name, value string
|
||||
var line []byte
|
||||
|
||||
fb := getFieldsBuf()
|
||||
defer putFieldsBuf(fb)
|
||||
currentTimestamp := time.Now().UnixNano()
|
||||
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
return fmt.Errorf("cannot read the first field: %w", err)
|
||||
}
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
for {
|
||||
line := lr.Line
|
||||
if len(line) == 0 {
|
||||
// The end of a single log entry. Write it to the storage
|
||||
if len(fb.fields) > 0 {
|
||||
for len(data) > 0 {
|
||||
idx := bytes.IndexByte(data, '\n')
|
||||
switch {
|
||||
case idx > 0:
|
||||
// process fields
|
||||
line = data[:idx]
|
||||
data = data[idx+1:]
|
||||
case idx == 0:
|
||||
// next message or end of file
|
||||
// double new line is a separator for the next message
|
||||
if len(fields) > 0 {
|
||||
if ts == 0 {
|
||||
ts = time.Now().UnixNano()
|
||||
ts = currentTimestamp
|
||||
}
|
||||
lmp.AddRow(ts, fb.fields, nil)
|
||||
lmp.AddRow(ts, fields, nil)
|
||||
fields = fields[:0]
|
||||
}
|
||||
return nil
|
||||
// skip newline separator
|
||||
data = data[1:]
|
||||
continue
|
||||
case idx < 0:
|
||||
return fmt.Errorf("missing new line separator, unread data left=%d", len(data))
|
||||
}
|
||||
|
||||
// line could be either "key=value" or "key"
|
||||
// according to https://systemd.io/JOURNAL_EXPORT_FORMATS/#journal-export-format
|
||||
if n := bytes.IndexByte(line, '='); n >= 0 {
|
||||
// line = "key=value"
|
||||
fb.name = append(fb.name[:0], line[:n]...)
|
||||
name = bytesutil.ToUnsafeString(fb.name)
|
||||
|
||||
fb.value = append(fb.value[:0], line[n+1:]...)
|
||||
value = bytesutil.ToUnsafeString(fb.value)
|
||||
idx = bytes.IndexByte(line, '=')
|
||||
// could b either e key=value\n pair
|
||||
// or just key\n
|
||||
// with binary data at the buffer
|
||||
if idx > 0 {
|
||||
name = bytesutil.ToUnsafeString(line[:idx])
|
||||
value = bytesutil.ToUnsafeString(line[idx+1:])
|
||||
} else {
|
||||
// line = "key"
|
||||
// Parse the binary-encoded value from the next line according to "key\n<little_endian_size_64>value\n" format
|
||||
fb.name = append(fb.name[:0], line...)
|
||||
name = bytesutil.ToUnsafeString(fb.name)
|
||||
|
||||
fb.value = fb.value[:0]
|
||||
for len(fb.value) < 8 {
|
||||
if err := fb.appendNextLineToValue(lr); err != nil {
|
||||
return fmt.Errorf("cannot read value size: %w", err)
|
||||
}
|
||||
name = bytesutil.ToUnsafeString(line)
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("unexpected zero data for binary field value of key=%s", name)
|
||||
}
|
||||
size := binary.LittleEndian.Uint64(fb.value[:8])
|
||||
|
||||
// Read the value until its lenth exceeds the given size - the last char in the read value will always be '\n'
|
||||
// because it is appended by appendNextLineToValue().
|
||||
for uint64(len(fb.value[8:])) <= size {
|
||||
if err := fb.appendNextLineToValue(lr); err != nil {
|
||||
return fmt.Errorf("cannot read %q value with size %d bytes; read only %d bytes: %w", fb.name, size, len(fb.value[8:]), err)
|
||||
}
|
||||
}
|
||||
value = bytesutil.ToUnsafeString(fb.value[8 : len(fb.value)-1])
|
||||
if uint64(len(value)) != size {
|
||||
return fmt.Errorf("unexpected %q value size; got %d bytes; want %d bytes; value: %q", fb.name, len(value), size, value)
|
||||
}
|
||||
}
|
||||
|
||||
if !lr.NextLine() {
|
||||
if err := lr.Err(); err != nil {
|
||||
return fmt.Errorf("cannot read the next log field: %w", err)
|
||||
}
|
||||
|
||||
// add the last log field below before the return
|
||||
}
|
||||
|
||||
if len(name) > maxFieldNameLen {
|
||||
logger.Errorf("%s: field name size should not exceed %d bytes; got %d bytes: %q; skipping this field", streamName, maxFieldNameLen, len(name), name)
|
||||
continue
|
||||
}
|
||||
if !isValidFieldName(name) {
|
||||
logger.Errorf("%s: invalid field name %q; it must consist of `A-Z0-9_` chars and must start from non-digit char; skipping this field", streamName, name)
|
||||
continue
|
||||
}
|
||||
|
||||
if slices.Contains(cp.TimeFields, name) {
|
||||
t, err := strconv.ParseInt(value, 10, 64)
|
||||
// size of binary data encoded as le i64 at the begging
|
||||
idx, err := binary.Decode(data, binary.LittleEndian, &size)
|
||||
if err != nil {
|
||||
logger.Errorf("%s: cannot parse timestamp from the field %q: %w; using the current timestamp", streamName, name, err)
|
||||
ts = 0
|
||||
} else {
|
||||
// Convert journald microsecond timestamp to nanoseconds
|
||||
ts = t * 1e3
|
||||
return fmt.Errorf("failed to extract binary field %q value size: %w", name, err)
|
||||
}
|
||||
// skip binary data size
|
||||
data = data[idx:]
|
||||
if size == 0 {
|
||||
return fmt.Errorf("unexpected zero binary data size decoded %d", size)
|
||||
}
|
||||
if int(size) > len(data) {
|
||||
return fmt.Errorf("binary data size=%d cannot exceed size of the data at buffer=%d", size, len(data))
|
||||
}
|
||||
value = bytesutil.ToUnsafeString(data[:size])
|
||||
data = data[int(size):]
|
||||
// binary data must has new line separator for the new line or next field
|
||||
if len(data) == 0 {
|
||||
return fmt.Errorf("unexpected empty buffer after binary field=%s read", name)
|
||||
}
|
||||
lastB := data[0]
|
||||
if lastB != '\n' {
|
||||
return fmt.Errorf("expected new line separator after binary field=%s, got=%s", name, string(lastB))
|
||||
}
|
||||
data = data[1:]
|
||||
}
|
||||
if len(name) > journaldEntryMaxNameLen {
|
||||
return fmt.Errorf("journald entry name should not exceed %d symbols, got: %q", journaldEntryMaxNameLen, name)
|
||||
}
|
||||
if !allowedJournaldEntryNameChars.MatchString(name) {
|
||||
return fmt.Errorf("journald entry name should consist of `A-Z0-9_` characters and must start from non-digit symbol")
|
||||
}
|
||||
if slices.Contains(cp.TimeFields, name) {
|
||||
n, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse Journald timestamp, %w", err)
|
||||
}
|
||||
ts = n * 1e3
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -322,56 +220,18 @@ func readJournaldLogEntry(streamName string, lr *insertutil.LineReader, lmp inse
|
||||
name = "_msg"
|
||||
}
|
||||
|
||||
if name == "PRIORITY" {
|
||||
priority := journaldPriorityToLevel(value)
|
||||
fb.addField("level", priority)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(name, "__") || *journaldIncludeEntryMetadata {
|
||||
fb.addField(name, value)
|
||||
if *journaldIncludeEntryMetadata || !strings.HasPrefix(name, "__") {
|
||||
fields = append(fields, logstorage.Field{
|
||||
Name: name,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func journaldPriorityToLevel(priority string) string {
|
||||
// See https://wiki.archlinux.org/title/Systemd/Journal#Priority_level
|
||||
// and https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level
|
||||
switch priority {
|
||||
case "0":
|
||||
return "emerg"
|
||||
case "1":
|
||||
return "alert"
|
||||
case "2":
|
||||
return "critical"
|
||||
case "3":
|
||||
return "error"
|
||||
case "4":
|
||||
return "warning"
|
||||
case "5":
|
||||
return "notice"
|
||||
case "6":
|
||||
return "info"
|
||||
case "7":
|
||||
return "debug"
|
||||
default:
|
||||
return priority
|
||||
}
|
||||
}
|
||||
|
||||
func isValidFieldName(s string) bool {
|
||||
if len(s) == 0 {
|
||||
return false
|
||||
}
|
||||
c := s[0]
|
||||
if !(c >= 'A' && c <= 'Z' || c == '_') {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := 1; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if !(c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_') {
|
||||
return false
|
||||
if len(fields) > 0 {
|
||||
if ts == 0 {
|
||||
ts = currentTimestamp
|
||||
}
|
||||
lmp.AddRow(ts, fields, nil)
|
||||
}
|
||||
return true
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,81 +1,20 @@
|
||||
package journald
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func TestIsValidFieldName(t *testing.T) {
|
||||
f := func(name string, resultExpected bool) {
|
||||
t.Helper()
|
||||
|
||||
result := isValidFieldName(name)
|
||||
if result != resultExpected {
|
||||
t.Fatalf("unexpected result for isValidJournaldFieldName(%q); got %v; want %v", name, result, resultExpected)
|
||||
}
|
||||
}
|
||||
|
||||
f("", false)
|
||||
f("a", false)
|
||||
f("1", false)
|
||||
f("_", true)
|
||||
f("X", true)
|
||||
f("Xa", false)
|
||||
f("X_343", true)
|
||||
f("X_0123456789_AZ", true)
|
||||
f("SDDFD sdf", false)
|
||||
}
|
||||
|
||||
func TestGetCommonParams_TimeField(t *testing.T) {
|
||||
f := func(timeFieldHeader, expectedTimeField string) {
|
||||
t.Helper()
|
||||
|
||||
req, err := http.NewRequest("POST", "/insert/journald/upload", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error creating request: %s", err)
|
||||
}
|
||||
|
||||
if timeFieldHeader != "" {
|
||||
req.Header.Set("VL-Time-Field", timeFieldHeader)
|
||||
}
|
||||
|
||||
cp, err := getCommonParams(req)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
if len(cp.TimeFields) != 1 || cp.TimeFields[0] != expectedTimeField {
|
||||
t.Fatalf("unexpected TimeFields; got %v; want [%s]", cp.TimeFields, expectedTimeField)
|
||||
}
|
||||
}
|
||||
|
||||
// Test default behavior - when no custom time field is specified, journald uses __REALTIME_TIMESTAMP
|
||||
f("", "__REALTIME_TIMESTAMP")
|
||||
|
||||
// Test custom time field - when a custom time field is specified via HTTP header, it's respected
|
||||
f("custom_time", "custom_time")
|
||||
}
|
||||
|
||||
func TestPushJournald_Success(t *testing.T) {
|
||||
func TestPushJournaldOk(t *testing.T) {
|
||||
f := func(src string, timestampsExpected []int64, resultExpected string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
|
||||
r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create request: %s", err)
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeFields: []string{"__REALTIME_TIMESTAMP"},
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create commonParams: %s", err)
|
||||
}
|
||||
|
||||
buf := bytes.NewBufferString(src)
|
||||
if err := processStreamInternal("test", buf, tlp, cp); err != nil {
|
||||
if err := parseJournaldRequest([]byte(src), tlp, cp); err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
|
||||
@@ -83,17 +22,16 @@ func TestPushJournald_Success(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Single event
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n",
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n",
|
||||
[]int64{91723819283000},
|
||||
"{\"_msg\":\"Test message\"}",
|
||||
)
|
||||
|
||||
// Multiple events
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nPRIORITY=3\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nMESSAGE=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n",
|
||||
[]int64{91723819283000, 91723819284000},
|
||||
"{\"level\":\"error\",\"PRIORITY\":\"3\",\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
|
||||
"{\"_msg\":\"Test message\"}\n{\"_msg\":\"Test message2\"}",
|
||||
)
|
||||
|
||||
// Parse binary data
|
||||
@@ -101,66 +39,30 @@ func TestPushJournald_Success(t *testing.T) {
|
||||
[]int64{1729698775704404000},
|
||||
"{\"E\":\"JobStateChanged\",\"_BOOT_ID\":\"f778b6e2f7584a77b991a2366612a7b5\",\"_UID\":\"0\",\"_GID\":\"0\",\"_MACHINE_ID\":\"a4a970370c30a925df02a13c67167847\",\"_HOSTNAME\":\"ecd5e4555787\",\"_RUNTIME_SCOPE\":\"system\",\"_TRANSPORT\":\"journal\",\"_CAP_EFFECTIVE\":\"1ffffffffff\",\"_SYSTEMD_CGROUP\":\"/init.scope\",\"_SYSTEMD_UNIT\":\"init.scope\",\"_SYSTEMD_SLICE\":\"-.slice\",\"CODE_FILE\":\"\\u003cstdin>\",\"CODE_LINE\":\"1\",\"CODE_FUNC\":\"\\u003cmodule>\",\"SYSLOG_IDENTIFIER\":\"python3\",\"_COMM\":\"python3\",\"_EXE\":\"/usr/bin/python3.12\",\"_CMDLINE\":\"python3\",\"_msg\":\"foo\\nbar\\n\\n\\nasda\\nasda\",\"_PID\":\"2763\",\"_SOURCE_REALTIME_TIMESTAMP\":\"1729698775704375\"}",
|
||||
)
|
||||
|
||||
// Parse binary data with trailing newline
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x14\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasda\nasda\n\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_msg":"foo\nbar\n\n\nasda\nasda\n","_PID":"2763"}`,
|
||||
)
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_PID":"2763"}`,
|
||||
)
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_msg":"123456789\n","_PID":"2763"}`,
|
||||
)
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890\n_PID=2763\n\n",
|
||||
[]int64{1729698775704404000},
|
||||
`{"_CMDLINE":"python3","_msg":"1234567890","_PID":"2763"}`,
|
||||
)
|
||||
|
||||
// Empty field name must be ignored
|
||||
f("__REALTIME_TIMESTAMP=91723819283\na=b\n=Test message", nil, "")
|
||||
f("__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n__REALTIME_TIMESTAMP=91723819283\n=Test message\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
|
||||
|
||||
// field name starting with number must be ignored
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n\n__REALTIME_TIMESTAMP=91723819284\nMESSAGE=Test message2\n\n", []int64{91723819284000}, `{"_msg":"Test message2"}`)
|
||||
|
||||
// field name exceeding 64 bytes limit must be ignored
|
||||
f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n", nil, "")
|
||||
|
||||
// field name with invalid chars must be ignored
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n", nil, "")
|
||||
}
|
||||
|
||||
func TestPushJournald_Failure(t *testing.T) {
|
||||
f := func(data string) {
|
||||
t.Helper()
|
||||
|
||||
tlp := &insertutil.TestLogMessageProcessor{}
|
||||
|
||||
r, err := http.NewRequest("GET", "https://foo.bar/baz", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create request: %s", err)
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeFields: []string{"__REALTIME_TIMESTAMP"},
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
cp, err := getCommonParams(r)
|
||||
if err != nil {
|
||||
t.Fatalf("cannot create commonParams: %s", err)
|
||||
}
|
||||
|
||||
buf := bytes.NewBufferString(data)
|
||||
if err := processStreamInternal("test", buf, tlp, cp); err == nil {
|
||||
t.Fatalf("expecting non-nil error")
|
||||
if err := parseJournaldRequest([]byte(data), tlp, cp); err == nil {
|
||||
t.Fatalf("expected non nil error")
|
||||
}
|
||||
}
|
||||
|
||||
// too short binary encoded message
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasd")
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x00\x00\x00\x00\x00\x00\x00\x00_PID=2763\n\n")
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x001234567890_PID=2763\n\n")
|
||||
f("__REALTIME_TIMESTAMP=1729698775704404\n_CMDLINE=python3\nMESSAGE\n\x0A\x00\x00\x00\x00\x00\x00\x00123456789\n_PID=2763\n\n")
|
||||
|
||||
// too long binary encoded message
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasdakljlsfd")
|
||||
// missing new line terminator for binary encoded message
|
||||
f("__CURSOR=s=e0afe8412a6a49d2bfcf66aa7927b588;i=1f06;b=f778b6e2f7584a77b991a2366612a7b5;m=300bdfd420;t=62526e1182354;x=930dc44b370963b7\n__REALTIME_TIMESTAMP=1729698775704404\nMESSAGE\n\x13\x00\x00\x00\x00\x00\x00\x00foo\nbar\n\n\nasdaasda2")
|
||||
// missing new line terminator
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n=Test message")
|
||||
// empty field name
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n=Test message\n")
|
||||
// field name starting with number
|
||||
f("__REALTIME_TIMESTAMP=91723819283\n1incorrect=Test message\n")
|
||||
// field name exceeds 64 limit
|
||||
f("__REALTIME_TIMESTAMP=91723819283\ntoolooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooongcorrecooooooooooooong=Test message\n")
|
||||
// Only allow A-Z0-9 and '_'
|
||||
f("__REALTIME_TIMESTAMP=91723819283\nbadC!@$!@$as=Test message\n")
|
||||
}
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
package journald
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
)
|
||||
|
||||
func BenchmarkIsValidFieldName(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(benchmarkFields)))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
for _, field := range benchmarkFields {
|
||||
if !isValidFieldName(field) {
|
||||
panic(fmt.Errorf("cannot validate field %q", field))
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var benchmarkFields = strings.Split(
|
||||
"E,_BOOT_ID,_UID,_GID,_MACHINE_ID,_HOSTNAME,_RUNTIME_SCOPE,_TRANSPORT,_CAP_EFFECTIVE,_SYSTEMD_CGROUP,_SYSTEMD_UNIT,"+
|
||||
"_SYSTEMD_SLICE,CODE_FILE,CODE_LINE,CODE_FUNC,SYSLOG_IDENTIFIER,_COMM,_EXE,_CMDLINE,MESSAGE,_PID,_SOURCE_REALTIME_TIMESTAMP,_REALTIME_TIMESTAMP",
|
||||
",")
|
||||
|
||||
func BenchmarkPushJournaldPerformance(b *testing.B) {
|
||||
cp := &insertutil.CommonParams{
|
||||
TimeFields: []string{"__REALTIME_TIMESTAMP"},
|
||||
MsgFields: []string{"MESSAGE"},
|
||||
}
|
||||
const dataChunkSize = 1024 * 1024
|
||||
|
||||
data := generateJournaldData(dataChunkSize)
|
||||
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(data)))
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
r := &bytes.Reader{}
|
||||
blp := &insertutil.BenchmarkLogMessageProcessor{}
|
||||
for pb.Next() {
|
||||
r.Reset(data)
|
||||
if err := processStreamInternal("performance_test", r, blp, cp); err != nil {
|
||||
panic(fmt.Errorf("unexpected error: %w", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func generateJournaldData(size int) []byte {
|
||||
var buf []byte
|
||||
timestamp := time.Now().UnixMicro()
|
||||
binaryMsg := []byte("binary message data for performance test")
|
||||
var sizeBuf [8]byte
|
||||
|
||||
for len(buf) < size {
|
||||
timestamp++
|
||||
|
||||
var entry string
|
||||
// Generate a mix of simple and binary messages
|
||||
if timestamp%10 == 0 {
|
||||
// Generate binary message
|
||||
binary.LittleEndian.PutUint64(sizeBuf[:], uint64(len(binaryMsg)))
|
||||
entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE\n%s%s\n\n",
|
||||
timestamp,
|
||||
sizeBuf[:],
|
||||
binaryMsg,
|
||||
)
|
||||
} else {
|
||||
// Generate simple message
|
||||
entry = fmt.Sprintf("__REALTIME_TIMESTAMP=%d\nMESSAGE=Performance test message %d\n\n", timestamp, timestamp)
|
||||
}
|
||||
buf = append(buf, entry...)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
@@ -32,7 +33,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
@@ -119,5 +120,5 @@ var (
|
||||
requestsTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/jsonline"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/jsonline"}`)
|
||||
|
||||
requestDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
|
||||
requestDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/jsonline"}`)
|
||||
)
|
||||
|
||||
@@ -16,10 +16,10 @@ var disableMessageParsing = flag.Bool("loki.disableMessageParsing", false, "Whet
|
||||
// RequestHandler processes Loki insert requests
|
||||
func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
case "/insert/loki/api/v1/push":
|
||||
case "/api/v1/push":
|
||||
handleInsert(r, w)
|
||||
return true
|
||||
case "/insert/loki/ready":
|
||||
case "/ready":
|
||||
// See https://grafana.com/docs/loki/latest/api/#identify-ready-loki-instance
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("ready"))
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/valyala/fastjson"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
@@ -29,7 +30,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
@@ -58,7 +59,7 @@ func handleJSON(r *http.Request, w http.ResponseWriter) {
|
||||
|
||||
var (
|
||||
requestsJSONTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
requestJSONDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
requestJSONDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="json"}`)
|
||||
)
|
||||
|
||||
func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/protoparserutil"
|
||||
@@ -28,7 +29,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
@@ -62,7 +63,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
|
||||
var (
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/loki/api/v1/push",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func parseProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields, parseMessage bool) error {
|
||||
|
||||
@@ -58,29 +58,35 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
|
||||
}
|
||||
|
||||
func insertHandler(w http.ResponseWriter, r *http.Request, path string) bool {
|
||||
path = strings.TrimPrefix(path, "/insert")
|
||||
|
||||
switch path {
|
||||
case "/insert/jsonline":
|
||||
case "/jsonline":
|
||||
jsonline.RequestHandler(w, r)
|
||||
return true
|
||||
case "/insert/ready":
|
||||
case "/ready":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(200)
|
||||
fmt.Fprintf(w, `{"status":"ok"}`)
|
||||
return true
|
||||
}
|
||||
switch {
|
||||
// some clients may omit trailing slash at elasticsearch protocol.
|
||||
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
|
||||
case strings.HasPrefix(path, "/insert/elasticsearch"):
|
||||
case strings.HasPrefix(path, "/elasticsearch"):
|
||||
// some clients may omit trailing slash
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8353
|
||||
path = strings.TrimPrefix(path, "/elasticsearch")
|
||||
return elasticsearch.RequestHandler(path, w, r)
|
||||
|
||||
case strings.HasPrefix(path, "/insert/loki/"):
|
||||
case strings.HasPrefix(path, "/loki/"):
|
||||
path = strings.TrimPrefix(path, "/loki")
|
||||
return loki.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/insert/opentelemetry/"):
|
||||
case strings.HasPrefix(path, "/opentelemetry/"):
|
||||
path = strings.TrimPrefix(path, "/opentelemetry")
|
||||
return opentelemetry.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/insert/journald/"):
|
||||
case strings.HasPrefix(path, "/journald/"):
|
||||
path = strings.TrimPrefix(path, "/journald")
|
||||
return journald.RequestHandler(path, w, r)
|
||||
case strings.HasPrefix(path, "/insert/datadog/"):
|
||||
case strings.HasPrefix(path, "/datadog/"):
|
||||
path = strings.TrimPrefix(path, "/datadog")
|
||||
return datadog.RequestHandler(path, w, r)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
||||
@@ -21,7 +22,7 @@ func RequestHandler(path string, w http.ResponseWriter, r *http.Request) bool {
|
||||
switch path {
|
||||
// use the same path as opentelemetry collector
|
||||
// https://opentelemetry.io/docs/specs/otlp/#otlphttp-request
|
||||
case "/insert/opentelemetry/v1/logs":
|
||||
case "/v1/logs":
|
||||
if r.Header.Get("Content-Type") == "application/json" {
|
||||
httpserver.Errorf(w, r, "json encoding isn't supported for opentelemetry format. Use protobuf encoding")
|
||||
return true
|
||||
@@ -42,7 +43,7 @@ func handleProtobuf(r *http.Request, w http.ResponseWriter) {
|
||||
httpserver.Errorf(w, r, "cannot parse common params from request: %s", err)
|
||||
return
|
||||
}
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
httpserver.Errorf(w, r, "%s", err)
|
||||
return
|
||||
}
|
||||
@@ -70,7 +71,7 @@ var (
|
||||
requestsProtobufTotal = metrics.NewCounter(`vl_http_requests_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
errorsTotal = metrics.NewCounter(`vl_http_errors_total{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
|
||||
requestProtobufDuration = metrics.NewSummary(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
requestProtobufDuration = metrics.NewHistogram(`vl_http_request_duration_seconds{path="/insert/opentelemetry/v1/logs",format="protobuf"}`)
|
||||
)
|
||||
|
||||
func pushProtobufRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields []string, useDefaultStreamFields bool) error {
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlinsert/insertutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/app/vlstorage"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/cgroup"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||
@@ -384,7 +385,7 @@ func serveTCP(ln net.Listener, tenantID logstorage.TenantID, encoding string, us
|
||||
|
||||
// processStream parses a stream of syslog messages from r and ingests them into vlstorage.
|
||||
func processStream(protocol string, r io.Reader, encoding string, useLocalTimestamp bool, cp *insertutil.CommonParams) error {
|
||||
if err := insertutil.CanWriteData(); err != nil {
|
||||
if err := vlstorage.CanWriteData(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -101,8 +101,8 @@ func TestProcessStreamInternal_Success(t *testing.T) {
|
||||
currentYear := 2023
|
||||
timestampsExpected := []int64{1685794113000000000, 1685880513000000000, 1685814132345000000}
|
||||
resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches..."}
|
||||
{"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
{"priority":"165","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg"}
|
||||
{"priority":"123","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data."}`
|
||||
f(data, currentYear, timestampsExpected, resultExpected)
|
||||
}
|
||||
|
||||
|
||||
@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
|
||||
|
||||
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
|
||||
|
||||
See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rate_over_sum
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
208
app/vlselect/vmui/assets/index-DhqzKCNf.js
Normal file
208
app/vlselect/vmui/assets/index-DhqzKCNf.js
Normal file
File diff suppressed because one or more lines are too long
67
app/vlselect/vmui/assets/vendor-D8IJGiEn.js
Normal file
67
app/vlselect/vmui/assets/vendor-D8IJGiEn.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -35,10 +35,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaLogs">
|
||||
<meta property="og:url" content="https://victoriametrics.com/products/victorialogs/">
|
||||
<meta property="og:description" content="Explore your log data with VictoriaLogs UI">
|
||||
<script type="module" crossorigin src="./assets/index-721xTF8u.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-V4vnRsM-.js">
|
||||
<script type="module" crossorigin src="./assets/index-DhqzKCNf.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-C36SC0pJ.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -253,30 +253,33 @@ func processForceFlush(w http.ResponseWriter, r *http.Request) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Storage implements insertutil.LogRowsStorage interface
|
||||
type Storage struct{}
|
||||
|
||||
// CanWriteData returns non-nil error if it cannot write data to vlstorage
|
||||
func (*Storage) CanWriteData() error {
|
||||
if localStorage == nil {
|
||||
// The data can be always written in non-local mode.
|
||||
func CanWriteData() error {
|
||||
if netstorageInsert != nil {
|
||||
if netstorageInsert.IsBroken() {
|
||||
return &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("cannot write data to vlstorage because all the storage nodes are broken"),
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if localStorage.IsReadOnly() {
|
||||
if localStorage != nil && localStorage.IsReadOnly() {
|
||||
return &httpserver.ErrorWithStatusCode{
|
||||
Err: fmt.Errorf("cannot add rows into storage in read-only mode; the storage can be in read-only mode "+
|
||||
"because of lack of free disk space at -storageDataPath=%s", *storageDataPath),
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustAddRows adds lr to vlstorage
|
||||
//
|
||||
// It is advised to call CanWriteData() before calling MustAddRows()
|
||||
func (*Storage) MustAddRows(lr *logstorage.LogRows) {
|
||||
func MustAddRows(lr *logstorage.LogRows) {
|
||||
if localStorage != nil {
|
||||
// Store lr in the local storage.
|
||||
localStorage.MustAddRows(lr)
|
||||
|
||||
@@ -2,6 +2,7 @@ package netinsert
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -23,6 +24,11 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/timerpool"
|
||||
)
|
||||
|
||||
var (
|
||||
maxInsertRetries = flag.Int("insert.maxRetries", 20, "The maximum number of retry attempts when sending data to storage nodes. "+
|
||||
"After exhausting retries, the data is queued in a retry buffer and new ingestion requests are rejected with HTTP 429 until storage nodes recover.")
|
||||
)
|
||||
|
||||
// the maximum size of a single data block sent to storage node.
|
||||
const maxInsertBlockSize = 2 * 1024 * 1024
|
||||
|
||||
@@ -41,6 +47,10 @@ type Storage struct {
|
||||
|
||||
pendingDataBuffers chan *bytesutil.ByteBuffer
|
||||
|
||||
retryDataBuffersMu sync.Mutex
|
||||
needDrainRetryData atomic.Bool
|
||||
retryDataBuffers []*bytesutil.ByteBuffer
|
||||
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
@@ -110,6 +120,10 @@ func (sn *storageNode) backgroundFlusher() {
|
||||
case <-sn.s.stopCh:
|
||||
return
|
||||
case <-t.C:
|
||||
if sn.s.needDrainRetryData.Load() {
|
||||
sn.flushRetryData()
|
||||
continue
|
||||
}
|
||||
sn.flushPendingData()
|
||||
}
|
||||
}
|
||||
@@ -126,7 +140,31 @@ func (sn *storageNode) flushPendingData() {
|
||||
pendingData := sn.grabPendingDataForFlushLocked()
|
||||
sn.pendingDataMu.Unlock()
|
||||
|
||||
sn.mustSendInsertRequest(pendingData)
|
||||
_ = sn.mustSendInsertRequest(pendingData)
|
||||
}
|
||||
|
||||
func (sn *storageNode) flushRetryData() {
|
||||
sn.s.retryDataBuffersMu.Lock()
|
||||
defer sn.s.retryDataBuffersMu.Unlock()
|
||||
|
||||
for len(sn.s.retryDataBuffers) > 0 {
|
||||
lastIdx := len(sn.s.retryDataBuffers) - 1
|
||||
pendingData := sn.s.retryDataBuffers[lastIdx]
|
||||
sn.s.retryDataBuffers = sn.s.retryDataBuffers[:lastIdx]
|
||||
if !sn.mustSendInsertRequest(pendingData) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
sn.s.needDrainRetryData.Store(len(sn.s.retryDataBuffers) > 0)
|
||||
}
|
||||
|
||||
func (sn *storageNode) addRetryData(pendingData *bytesutil.ByteBuffer) {
|
||||
sn.s.retryDataBuffersMu.Lock()
|
||||
defer sn.s.retryDataBuffersMu.Unlock()
|
||||
|
||||
sn.s.retryDataBuffers = append(sn.s.retryDataBuffers, pendingData)
|
||||
sn.s.needDrainRetryData.Store(true)
|
||||
}
|
||||
|
||||
func (sn *storageNode) addRow(r *logstorage.InsertRow) {
|
||||
@@ -152,7 +190,7 @@ func (sn *storageNode) addRow(r *logstorage.InsertRow) {
|
||||
bbPool.Put(bb)
|
||||
|
||||
if pendingData != nil {
|
||||
sn.mustSendInsertRequest(pendingData)
|
||||
_ = sn.mustSendInsertRequest(pendingData)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,21 +204,37 @@ func (sn *storageNode) grabPendingDataForFlushLocked() *bytesutil.ByteBuffer {
|
||||
return pendingData
|
||||
}
|
||||
|
||||
func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) {
|
||||
// mustSendInsertRequest guarantees that data will be sent to storage nodes or buffered for retry.
|
||||
// It attempts to send pendingData to storage nodes with retry logic and returns:
|
||||
// - true: data was handled (successfully sent to a storage node, or operation was cancelled during shutdown)
|
||||
// - false: all storage nodes are unavailable after maxInsertRetries attempts, data has been added to retry buffer
|
||||
//
|
||||
// When this method returns false, it indicates that the storages are temporarily unavailable
|
||||
// and the data has been queued in the retry buffer for later processing when nodes become available.
|
||||
// The retry buffer prevents data loss while protecting against infinite memory accumulation.
|
||||
func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer) (handled bool) {
|
||||
defer func() {
|
||||
pendingData.Reset()
|
||||
sn.s.pendingDataBuffers <- pendingData
|
||||
if handled {
|
||||
pendingData.Reset()
|
||||
sn.s.pendingDataBuffers <- pendingData
|
||||
}
|
||||
}()
|
||||
|
||||
err := sn.sendInsertRequest(pendingData)
|
||||
if err == nil {
|
||||
return
|
||||
return true
|
||||
}
|
||||
|
||||
if !errors.Is(err, errTemporarilyDisabled) {
|
||||
logger.Warnf("%s; re-routing the data block to the remaining nodes", err)
|
||||
}
|
||||
for !sn.s.sendInsertRequestToAnyNode(pendingData) {
|
||||
|
||||
for i := 0; !sn.s.sendInsertRequestToAnyNode(pendingData); i++ {
|
||||
if *maxInsertRetries > 0 && i >= *maxInsertRetries {
|
||||
sn.addRetryData(pendingData)
|
||||
return false
|
||||
}
|
||||
|
||||
logger.Errorf("cannot send pending data to all storage nodes, since all of them are unavailable; re-trying to send the data in a second")
|
||||
|
||||
t := timerpool.Get(time.Second)
|
||||
@@ -188,11 +242,13 @@ func (sn *storageNode) mustSendInsertRequest(pendingData *bytesutil.ByteBuffer)
|
||||
case <-sn.s.stopCh:
|
||||
timerpool.Put(t)
|
||||
logger.Errorf("dropping %d bytes of data, since there are no available storage nodes", pendingData.Len())
|
||||
return
|
||||
return true
|
||||
case <-t.C:
|
||||
timerpool.Put(t)
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (sn *storageNode) sendInsertRequest(pendingData *bytesutil.ByteBuffer) error {
|
||||
@@ -272,13 +328,16 @@ var zstdBufPool bytesutil.ByteBufferPool
|
||||
// Call MustStop on the returned storage when it is no longer needed.
|
||||
func NewStorage(addrs []string, authCfgs []*promauth.Config, isTLSs []bool, concurrency int, disableCompression bool) *Storage {
|
||||
pendingDataBuffers := make(chan *bytesutil.ByteBuffer, concurrency*len(addrs))
|
||||
for i := 0; i < cap(pendingDataBuffers); i++ {
|
||||
for range cap(pendingDataBuffers) {
|
||||
pendingDataBuffers <- &bytesutil.ByteBuffer{}
|
||||
}
|
||||
|
||||
retryDataBuffers := []*bytesutil.ByteBuffer{}
|
||||
|
||||
s := &Storage{
|
||||
disableCompression: disableCompression,
|
||||
pendingDataBuffers: pendingDataBuffers,
|
||||
retryDataBuffers: retryDataBuffers,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
@@ -307,6 +366,12 @@ func (s *Storage) AddRow(streamHash uint64, r *logstorage.InsertRow) {
|
||||
sn.addRow(r)
|
||||
}
|
||||
|
||||
// IsBroken returns true if the storage is in a broken state where retry data needs to be drained first.
|
||||
// When true, it indicates that all storage nodes are temporarily unavailable and data is being buffered for retry.
|
||||
func (s *Storage) IsBroken() bool {
|
||||
return s.needDrainRetryData.Load()
|
||||
}
|
||||
|
||||
func (s *Storage) sendInsertRequestToAnyNode(pendingData *bytesutil.ByteBuffer) bool {
|
||||
startIdx := int(fastrand.Uint32n(uint32(len(s.sns))))
|
||||
for i := range s.sns {
|
||||
|
||||
@@ -20,8 +20,9 @@ var (
|
||||
"The time filter in RFC3339 format to finish the replay by. E.g. '2020-01-01T20:07:00Z'. "+
|
||||
"By default, is set to the current time.")
|
||||
replayRulesDelay = flag.Duration("replay.rulesDelay", time.Second,
|
||||
"Delay before evaluating the next rule within the group. Is important for chained rules. "+
|
||||
"Keep it equal or bigger than -remoteWrite.flushInterval. When set to >0, replay ignores group's concurrency setting.")
|
||||
"Delay between rules evaluation within the group. Could be important if there are chained rules inside the group "+
|
||||
"and processing need to wait for previous rule results to be persisted by remote storage before evaluating the next rule."+
|
||||
"Keep it equal or bigger than -remoteWrite.flushInterval.")
|
||||
replayMaxDatapoints = flag.Int("replay.maxDatapointsPerQuery", 1e3,
|
||||
"Max number of data points expected in one request. It affects the max time range for every '/query_range' request during the replay. The higher the value, the less requests will be made during replay.")
|
||||
replayRuleRetryAttempts = flag.Int("replay.ruleRetryAttempts", 5,
|
||||
|
||||
@@ -39,7 +39,7 @@ func (fr *fakeReplayQuerier) QueryRange(_ context.Context, q string, from, to ti
|
||||
}
|
||||
|
||||
func TestReplay(t *testing.T) {
|
||||
f := func(from, to string, maxDP int, ruleDelay time.Duration, cfg []config.Group, qb *fakeReplayQuerier) {
|
||||
f := func(from, to string, maxDP int, cfg []config.Group, qb *fakeReplayQuerier) {
|
||||
t.Helper()
|
||||
|
||||
fromOrig, toOrig, maxDatapointsOrig := *replayFrom, *replayTo, *replayMaxDatapoints
|
||||
@@ -51,7 +51,7 @@ func TestReplay(t *testing.T) {
|
||||
}()
|
||||
|
||||
*replayRuleRetryAttempts = 1
|
||||
*replayRulesDelay = ruleDelay
|
||||
*replayRulesDelay = time.Millisecond
|
||||
rwb := &remotewrite.DebugClient{}
|
||||
*replayFrom = from
|
||||
*replayTo = to
|
||||
@@ -65,7 +65,7 @@ func TestReplay(t *testing.T) {
|
||||
}
|
||||
|
||||
// one rule + one response
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:00.000Z", 10, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string]struct{}{
|
||||
@@ -74,7 +74,7 @@ func TestReplay(t *testing.T) {
|
||||
})
|
||||
|
||||
// one rule + multiple responses
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string]struct{}{
|
||||
@@ -87,7 +87,7 @@ func TestReplay(t *testing.T) {
|
||||
})
|
||||
|
||||
// datapoints per step
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T15:02:30.000Z", 60, []config.Group{
|
||||
{Interval: promutil.NewDuration(time.Minute), Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string]struct{}{
|
||||
@@ -101,7 +101,7 @@ func TestReplay(t *testing.T) {
|
||||
})
|
||||
|
||||
// multiple recording rules + multiple responses
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
|
||||
{Rules: []config.Rule{{Record: "foo", Expr: "sum(up)"}}},
|
||||
{Rules: []config.Rule{{Record: "bar", Expr: "max(up)"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
@@ -120,7 +120,7 @@ func TestReplay(t *testing.T) {
|
||||
})
|
||||
|
||||
// multiple alerting rules + multiple responses
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, time.Millisecond, []config.Group{
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, []config.Group{
|
||||
{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}}},
|
||||
{Rules: []config.Rule{{Alert: "bar", Expr: "max(up) < 1"}}},
|
||||
}, &fakeReplayQuerier{
|
||||
@@ -137,21 +137,4 @@ func TestReplay(t *testing.T) {
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// multiple alerting rules in one group+ multiple responses + concurrency
|
||||
f("2021-01-01T12:00:00.000Z", "2021-01-01T12:02:30.000Z", 1, 0, []config.Group{
|
||||
{Rules: []config.Rule{{Alert: "foo", Expr: "sum(up) > 1"}, {Alert: "bar", Expr: "max(up) < 1"}}, Concurrency: 2}}, &fakeReplayQuerier{
|
||||
registry: map[string]map[string]struct{}{
|
||||
"sum(up) > 1": {
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
"max(up) < 1": {
|
||||
"12:00:00+12:01:00": {},
|
||||
"12:01:00+12:02:00": {},
|
||||
"12:02:00+12:02:30": {},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -336,9 +335,7 @@ func (ar *AlertingRule) execRange(ctx context.Context, start, end time.Time) ([]
|
||||
var result []prompbmarshal.TimeSeries
|
||||
holdAlertState := make(map[uint64]*notifier.Alert)
|
||||
qFn := func(_ string) ([]datasource.Metric, error) {
|
||||
logger.Warnf("`query` template isn't supported in replay mode, mocked data is used")
|
||||
// mock query results to allow common used template {{ query <$expr> | first | value }}
|
||||
return []datasource.Metric{{Timestamps: []int64{0}, Values: []float64{math.NaN()}}}, nil
|
||||
return nil, fmt.Errorf("`query` template isn't supported in replay mode")
|
||||
}
|
||||
for _, s := range res.Data {
|
||||
ls, as, err := ar.expandTemplates(s, qFn, time.Time{})
|
||||
@@ -416,7 +413,7 @@ func (ar *AlertingRule) exec(ctx context.Context, ts time.Time, limit int) ([]pr
|
||||
return nil, fmt.Errorf("failed to execute query %q: %w", ar.Expr, err)
|
||||
}
|
||||
|
||||
ar.logDebugf(ts, nil, "query returned %d series (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
|
||||
ar.logDebugf(ts, nil, "query returned %d samples (elapsed: %s, isPartial: %t)", curState.Samples, curState.Duration, isPartialResponse(res))
|
||||
qFn := func(query string) ([]datasource.Metric, error) {
|
||||
res, _, err := ar.q.Query(ctx, query, ts)
|
||||
return res.Data, err
|
||||
|
||||
@@ -445,17 +445,11 @@ func (g *Group) Start(ctx context.Context, nts func() []notifier.Notifier, rw re
|
||||
|
||||
g.infof("re-started")
|
||||
case <-t.C:
|
||||
// calculate the real wall clock offset by stripping the monotonic clock first,
|
||||
// then evalTS can be corrected when wall clock is adjusted.
|
||||
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8790#issuecomment-2986541829
|
||||
offset := time.Now().Round(0).Sub(evalTS.Round(0))
|
||||
missed := (offset / g.Interval) - 1
|
||||
missed := (time.Since(evalTS) / g.Interval) - 1
|
||||
if missed < 0 {
|
||||
// missed can become < 0 due to irregular delays during evaluation
|
||||
// which can result in time.Since(evalTS) < g.Interval;
|
||||
// or the system wall clock was changed backward
|
||||
// which can result in time.Since(evalTS) < g.Interval
|
||||
missed = 0
|
||||
evalTS = time.Now()
|
||||
}
|
||||
if missed > 0 {
|
||||
g.metrics.iterationMissed.Inc()
|
||||
@@ -520,84 +514,36 @@ func (g *Group) Replay(start, end time.Time, rw remotewrite.RWClient, maxDataPoi
|
||||
iterations := int(end.Sub(start)/step) + 1
|
||||
fmt.Printf("\nGroup %q"+
|
||||
"\ninterval: \t%v"+
|
||||
"\nconcurrency: \t %d"+
|
||||
"\nrequests to make per rule: \t%d"+
|
||||
"\nrequests to make: \t%d"+
|
||||
"\nmax range per request: \t%v\n",
|
||||
g.Name, g.Interval, g.Concurrency, iterations, step)
|
||||
g.Name, g.Interval, iterations, step)
|
||||
if g.Limit > 0 {
|
||||
fmt.Printf("\nWarning: `limit: %d` param has no effect during replay.\n",
|
||||
fmt.Printf("\nPlease note, `limit: %d` param has no effect during replay.\n",
|
||||
g.Limit)
|
||||
}
|
||||
concurrency := g.Concurrency
|
||||
if g.Concurrency > 1 && replayDelay > 0 {
|
||||
fmt.Printf("\nWarning: group concurrency %d will be ignored since `-replay.rulesDelay` is %.3f seconds."+
|
||||
" Set -replay.rulesDelay=0 to enable concurrency for replay.\n", g.Concurrency, replayDelay.Seconds())
|
||||
concurrency = 1
|
||||
}
|
||||
|
||||
if concurrency == 1 {
|
||||
for _, rule := range g.Rules {
|
||||
var bar *pb.ProgressBar
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations)
|
||||
}
|
||||
// pass ri as a copy, so it can be modified within the replayRuleRange
|
||||
total += replayRuleRange(rule, ri, bar, rw, replayRuleRetryAttempts)
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
// sleep to let remote storage to flush data on-disk
|
||||
// so chained rules could be calculated correctly
|
||||
time.Sleep(replayDelay)
|
||||
for _, rule := range g.Rules {
|
||||
fmt.Printf("> Rule %q (ID: %d)\n", rule, rule.ID())
|
||||
var bar *pb.ProgressBar
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
sem := make(chan struct{}, g.Concurrency)
|
||||
res := make(chan int, len(g.Rules)*iterations)
|
||||
wg := sync.WaitGroup{}
|
||||
var bar *pb.ProgressBar
|
||||
if !disableProgressBar {
|
||||
bar = pb.StartNew(iterations * len(g.Rules))
|
||||
}
|
||||
for _, r := range g.Rules {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(r Rule, ri rangeIterator) {
|
||||
// pass ri as a copy, so it can be modified within the replayRuleRange
|
||||
res <- replayRuleRange(r, ri, bar, rw, replayRuleRetryAttempts)
|
||||
<-sem
|
||||
wg.Done()
|
||||
}(r, ri)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(res)
|
||||
close(sem)
|
||||
|
||||
if bar != nil {
|
||||
bar.Finish()
|
||||
}
|
||||
|
||||
total = 0
|
||||
for n := range res {
|
||||
total += n
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func replayRuleRange(r Rule, ri rangeIterator, bar *pb.ProgressBar, rw remotewrite.RWClient, replayRuleRetryAttempts int) int {
|
||||
fmt.Printf("> Rule %q (ID: %d)\n", r, r.ID())
|
||||
total := 0
|
||||
for ri.next() {
|
||||
n, err := replayRule(r, ri.s, ri.e, rw, replayRuleRetryAttempts)
|
||||
if err != nil {
|
||||
logger.Fatalf("rule %q: %s", r, err)
|
||||
ri.reset()
|
||||
for ri.next() {
|
||||
n, err := replayRule(rule, ri.s, ri.e, rw, replayRuleRetryAttempts)
|
||||
if err != nil {
|
||||
logger.Fatalf("rule %q: %s", rule, err)
|
||||
}
|
||||
total += n
|
||||
if bar != nil {
|
||||
bar.Increment()
|
||||
}
|
||||
}
|
||||
if bar != nil {
|
||||
bar.Increment()
|
||||
bar.Finish()
|
||||
}
|
||||
total += n
|
||||
// sleep to let remote storage to flush data on-disk
|
||||
// so chained rules could be calculated correctly
|
||||
time.Sleep(replayDelay)
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -624,10 +570,11 @@ type rangeIterator struct {
|
||||
s, e time.Time
|
||||
}
|
||||
|
||||
// next iterates with given step between start and end
|
||||
// by modifying iter, s and e.
|
||||
// Returns true until it reaches end.
|
||||
// next modifies ri and isn't thread-safe.
|
||||
func (ri *rangeIterator) reset() {
|
||||
ri.iter = 0
|
||||
ri.s, ri.e = time.Time{}, time.Time{}
|
||||
}
|
||||
|
||||
func (ri *rangeIterator) next() bool {
|
||||
ri.s = ri.start.Add(ri.step * time.Duration(ri.iter))
|
||||
if !ri.end.After(ri.s) {
|
||||
|
||||
@@ -162,7 +162,7 @@
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" style="width: 60%">Rule</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -594,7 +594,7 @@
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" title="The time when event was created">Updated at</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many samples were returned">Samples</th>
|
||||
{% if seriesFetchedEnabled %}<th scope="col" style="width: 10%" class="text-center" title="How many series were scanned by datasource during the evaluation">Series fetched</th>{% endif %}
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many seconds request took">Duration</th>
|
||||
<th scope="col" class="text-center" title="Time used for rule execution">Executed at</th>
|
||||
|
||||
@@ -524,7 +524,7 @@ func StreamListGroups(qw422016 *qt422016.Writer, r *http.Request, groups []apiGr
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" style="width: 60%">Rule</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many series were produced by the rule">Series</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many samples were produced by the rule">Samples</th>
|
||||
<th scope="col" style="width: 20%" class="text-center" title="How many seconds ago rule was executed">Updated</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -1697,7 +1697,7 @@ func StreamRuleDetails(qw422016 *qt422016.Writer, r *http.Request, rule apiRule)
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col" title="The time when event was created">Updated at</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many series expression returns. Each series will represent an alert.">Series returned</th>
|
||||
<th scope="col" style="width: 10%" class="text-center" title="How many samples were returned">Samples</th>
|
||||
`)
|
||||
//line app/vmalert/web.qtpl:598
|
||||
if seriesFetchedEnabled {
|
||||
|
||||
@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
|
||||
|
||||
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
|
||||
|
||||
See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rate_over_sum
|
||||
|
||||
File diff suppressed because one or more lines are too long
209
app/vmselect/vmui/assets/index-D-ssBbZq.js
Normal file
209
app/vmselect/vmui/assets/index-D-ssBbZq.js
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -36,10 +36,10 @@
|
||||
<meta property="og:title" content="UI for VictoriaMetrics">
|
||||
<meta property="og:url" content="https://victoriametrics.com/">
|
||||
<meta property="og:description" content="Explore and troubleshoot your VictoriaMetrics data">
|
||||
<script type="module" crossorigin src="./assets/index-BiQY-19a.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-D-ssBbZq.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/vendor-D8IJGiEn.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/vendor-D1GxaB_c.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-ojCMu5lE.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-D5re9hC6.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
|
||||
@@ -63,8 +63,6 @@ var (
|
||||
|
||||
cacheSizeStorageTSID = flagutil.NewBytes("storage.cacheSizeStorageTSID", 0, "Overrides max size for storage/tsid cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
cacheSizeStorageMetricName = flagutil.NewBytes("storage.cacheSizeStorageMetricName", 0, "Overrides max size for storage/metricName cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
cacheSizeIndexDBIndexBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBIndexBlocks", 0, "Overrides max size for indexdb/indexBlocks cache. "+
|
||||
"See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#cache-tuning")
|
||||
cacheSizeIndexDBDataBlocks = flagutil.NewBytes("storage.cacheSizeIndexDBDataBlocks", 0, "Overrides max size for indexdb/dataBlocks cache. "+
|
||||
@@ -113,7 +111,6 @@ func Init(resetCacheIfNeeded func(mrs []storage.MetricRow)) {
|
||||
storage.SetTSIDCacheSize(cacheSizeStorageTSID.IntN())
|
||||
storage.SetTagFiltersCacheSize(cacheSizeIndexDBTagFilters.IntN())
|
||||
storage.SetMetricNamesStatsCacheSize(cacheSizeMetricNamesStats.IntN())
|
||||
storage.SetMetricNameCacheSize(cacheSizeStorageMetricName.IntN())
|
||||
mergeset.SetIndexBlocksCacheSize(cacheSizeIndexDBIndexBlocks.IntN())
|
||||
mergeset.SetDataBlocksCacheSize(cacheSizeIndexDBDataBlocks.IntN())
|
||||
mergeset.SetDataBlocksSparseCacheSize(cacheSizeIndexDBDataBlocksSparse.IntN())
|
||||
|
||||
34
app/vmui/packages/vmui/package-lock.json
generated
34
app/vmui/packages/vmui/package-lock.json
generated
@@ -10,8 +10,6 @@
|
||||
"dependencies": {
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/lodash.get": "^4.4.9",
|
||||
"@types/lodash.orderBy": "^4.6.9",
|
||||
"@types/lodash.throttle": "^4.1.9",
|
||||
"@types/qs": "^6.9.18",
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-input-mask": "^3.0.6",
|
||||
@@ -20,8 +18,6 @@
|
||||
"dayjs": "^1.11.13",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.get": "^4.4.2",
|
||||
"lodash.orderBy": "^4.6.0",
|
||||
"lodash.throttle": "^4.1.1",
|
||||
"marked": "^15.0.8",
|
||||
"marked-emoji": "^2.0.0",
|
||||
"preact": "^10.26.5",
|
||||
@@ -2195,24 +2191,6 @@
|
||||
"@types/lodash": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/lodash.orderBy": {
|
||||
"version": "4.6.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash.orderby/-/lodash.orderby-4.6.9.tgz",
|
||||
"integrity": "sha512-T9o2wkIJOmxXwVTPTmwJ59W6eTi2FseiLR369fxszG649Po/xe9vqFNhf/MtnvT5jrbDiyWKxPFPZbpSVK0SVQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/lodash": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/lodash.throttle": {
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash.throttle/-/lodash.throttle-4.1.9.tgz",
|
||||
"integrity": "sha512-PCPVfpfueguWZQB7pJQK890F2scYKoDUL3iM522AptHWn7d5NQmeS/LTEHIcLr5PaTzl3dK2Z0xSUHHTHwaL5g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/lodash": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.14.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz",
|
||||
@@ -5764,18 +5742,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.orderBy": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.orderby/-/lodash.orderby-4.6.0.tgz",
|
||||
"integrity": "sha512-T0rZxKmghOOf5YPnn8EY5iLYeWCpZq8G41FfqoVHH5QDTAFaghJRmAdLiadEDq+ztgM2q5PjA+Z1fOwGrLgmtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.throttle": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz",
|
||||
"integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/loose-envify": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
|
||||
|
||||
@@ -7,8 +7,6 @@
|
||||
"dependencies": {
|
||||
"@types/lodash.debounce": "^4.0.9",
|
||||
"@types/lodash.get": "^4.4.9",
|
||||
"@types/lodash.orderBy": "^4.6.9",
|
||||
"@types/lodash.throttle": "^4.1.9",
|
||||
"@types/qs": "^6.9.18",
|
||||
"@types/react": "^19.1.2",
|
||||
"@types/react-input-mask": "^3.0.6",
|
||||
@@ -17,8 +15,6 @@
|
||||
"dayjs": "^1.11.13",
|
||||
"lodash.debounce": "^4.0.8",
|
||||
"lodash.get": "^4.4.2",
|
||||
"lodash.orderBy": "^4.6.0",
|
||||
"lodash.throttle": "^4.1.1",
|
||||
"marked": "^15.0.8",
|
||||
"marked-emoji": "^2.0.0",
|
||||
"preact": "^10.26.5",
|
||||
|
||||
@@ -742,23 +742,7 @@ Metric names are stripped from the resulting rollups. Add [keep_metric_names](#k
|
||||
|
||||
This function is supported by PromQL.
|
||||
|
||||
See also [irate](#irate), [rollup_rate](#rollup_rate) and [rate_prometheus](#rate_prometheus).
|
||||
|
||||
#### rate_prometheus
|
||||
|
||||
`rate_prometheus(series_selector[d])` {{% available_from "#" %}} is a [rollup function](#rollup-functions), which calculates the average per-second
|
||||
increase rate over the given lookbehind window `d` per each time series returned from the given [series_selector](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#filtering).
|
||||
The resulting calculation is equivalent to `increase_prometheus(series_selector[d]) / d`.
|
||||
|
||||
It doesn't take into account the last sample before the given lookbehind window `d` when calculating the result in the same way as Prometheus does.
|
||||
See [this article](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e) for details.
|
||||
|
||||
Metric names are stripped from the resulting rollups. Add [keep_metric_names](#keep_metric_names) modifier in order to keep metric names.
|
||||
|
||||
This function is usually applied to [counters](https://docs.victoriametrics.com/victoriametrics/keyconcepts/#counter).
|
||||
|
||||
See also [increase_prometheus](#increase_prometheus) and [rate](#rate).
|
||||
|
||||
See also [irate](#irate) and [rollup_rate](#rollup_rate).
|
||||
|
||||
#### rate_over_sum
|
||||
|
||||
|
||||
@@ -33,12 +33,8 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
const part = logicalParts.find(p => caretPosition[0] >= p.position[0] && caretPosition[0] <= p.position[1]);
|
||||
if (!part) return;
|
||||
const cursorStartPosition = caretPosition[0] - part.position[0];
|
||||
const prevPart = logicalParts.find(p => p.id === part.id - 1);
|
||||
const queryBeforeIncompleteFilter = prevPart ? value.substring(0, prevPart.position[1] + 1) : undefined;
|
||||
return {
|
||||
...part,
|
||||
queryBeforeIncompleteFilter,
|
||||
query: value,
|
||||
...getContextData(part, cursorStartPosition)
|
||||
};
|
||||
}, [logicalParts, caretPosition]);
|
||||
@@ -54,8 +50,6 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
return fieldValues;
|
||||
case ContextType.PipeName:
|
||||
return pipeList;
|
||||
case ContextType.FilterOrPipeName:
|
||||
return [...fieldNames, ...pipeList];
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
@@ -64,7 +58,7 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
const getUpdatedValue = (insertValue: string, logicalParts: LogicalPart[], id?: number) => {
|
||||
return logicalParts.reduce((acc, part) => {
|
||||
const value = part.id === id ? insertValue : part.value;
|
||||
const separator = part.separator === "|" ? " | " : " ";
|
||||
const separator = part.type === LogicalPartType.Pipe ? " | " : " ";
|
||||
return `${acc}${separator}${value}`;
|
||||
}, "").trim();
|
||||
};
|
||||
@@ -76,7 +70,7 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
modifiedInsert += ":";
|
||||
} else if (contextType === ContextType.FilterValue) {
|
||||
const insertWithQuotes = value.startsWith("_stream:") ? modifiedInsert : `${JSON.stringify(modifiedInsert)}`;
|
||||
modifiedInsert = `${contextData?.filterName || ""}${contextData?.operator || ":"}${insertWithQuotes}`;
|
||||
modifiedInsert = `${contextData?.filterName || ""}:${insertWithQuotes}`;
|
||||
}
|
||||
|
||||
return modifiedInsert;
|
||||
@@ -92,13 +86,7 @@ const LogsQueryEditorAutocomplete: FC<QueryEditorAutocompleteProps> = ({
|
||||
|
||||
const insertValue = getModifyInsert(insert, contextType, value, item.type);
|
||||
const newValue = getUpdatedValue(insertValue, logicalParts, id);
|
||||
const logicalPart = logicalParts.find(p => p.id === id);
|
||||
const getPositionCorrection = () => {
|
||||
if (logicalPart?.type === LogicalPartType.FilterOrPipe) return 1;
|
||||
if (item.type === ContextType.PipeName) return 1;
|
||||
return 0;
|
||||
};
|
||||
const updatedPosition = (position[0] || 1) + insertValue.length + getPositionCorrection();
|
||||
const updatedPosition = (position[0] || 1) + insertValue.length + (item.type === ContextType.PipeName ? 1 : 0);
|
||||
|
||||
onSelect(newValue, updatedPosition);
|
||||
}, [contextData, logicalParts]);
|
||||
|
||||
@@ -9,7 +9,6 @@ export const splitLogicalParts = (expr: string) => {
|
||||
const input = expr; //.replace(/\s*:\s*/g, ":");
|
||||
const parts: LogicalPart[] = [];
|
||||
let currentPart = "";
|
||||
let separator: undefined | " " | "|" = undefined;
|
||||
let isPipePart = false;
|
||||
|
||||
const quotes = ["'", "\"", "`"];
|
||||
@@ -44,9 +43,8 @@ export const splitLogicalParts = (expr: string) => {
|
||||
isPipePart = true;
|
||||
const countStartSpaces = currentPart.match(/^ */)?.[0].length || 0;
|
||||
const countEndSpaces = currentPart.match(/ *$/)?.[0].length || 0;
|
||||
pushPart(currentPart, true, [startIndex + countStartSpaces, i - countEndSpaces - 1], parts, separator);
|
||||
pushPart(currentPart, true, [startIndex + countStartSpaces, i - countEndSpaces - 1], parts);
|
||||
currentPart = "";
|
||||
separator = "|";
|
||||
startIndex = i + 1;
|
||||
continue;
|
||||
}
|
||||
@@ -56,8 +54,7 @@ export const splitLogicalParts = (expr: string) => {
|
||||
const nextStr = input.slice(i).replace(/^\s*/, "");
|
||||
const prevStr = input.slice(0, i).replace(/\s*$/, "");
|
||||
if (!nextStr.startsWith(":") && !prevStr.endsWith(":")) {
|
||||
pushPart(currentPart, false, [startIndex, i - 1], parts, separator);
|
||||
separator = " ";
|
||||
pushPart(currentPart, false, [startIndex, i - 1], parts);
|
||||
currentPart = "";
|
||||
startIndex = i + 1;
|
||||
continue;
|
||||
@@ -68,35 +65,26 @@ export const splitLogicalParts = (expr: string) => {
|
||||
}
|
||||
|
||||
// push the last part
|
||||
pushPart(currentPart, isPipePart, [startIndex, input.length], parts, separator);
|
||||
pushPart(currentPart, isPipePart, [startIndex, input.length], parts);
|
||||
|
||||
return parts;
|
||||
};
|
||||
|
||||
const pushPart = (currentPart: string, isPipePart: boolean, position: LogicalPartPosition, parts: LogicalPart[], separator: LogicalPart["separator"]) => {
|
||||
const pushPart = (currentPart: string, isPipePart: boolean, position: LogicalPartPosition, parts: LogicalPart[]) => {
|
||||
const trimmedPart = currentPart.trim();
|
||||
if (!trimmedPart) return;
|
||||
const isOperator = BUILDER_OPERATORS.includes(trimmedPart.toUpperCase());
|
||||
const pipesTypes = [LogicalPartType.Pipe, LogicalPartType.FilterOrPipe];
|
||||
const isPreviousPartPipe = parts.length > 0 && pipesTypes.includes(parts[parts.length - 1].type);
|
||||
|
||||
const getType = () => {
|
||||
if (isPreviousPartPipe) return LogicalPartType.FilterOrPipe;
|
||||
if (isPipePart) return LogicalPartType.Pipe;
|
||||
if (isOperator) return LogicalPartType.Operator;
|
||||
return LogicalPartType.Filter;
|
||||
};
|
||||
|
||||
parts.push({
|
||||
id: parts.length,
|
||||
value: trimmedPart,
|
||||
position,
|
||||
type: getType(),
|
||||
separator,
|
||||
type: isPipePart
|
||||
? LogicalPartType.Pipe
|
||||
: isOperator ? LogicalPartType.Operator : LogicalPartType.Filter,
|
||||
});
|
||||
};
|
||||
|
||||
export const getContextData = (part: LogicalPart, cursorPos: number): ContextData => {
|
||||
export const getContextData = (part: LogicalPart, cursorPos: number) => {
|
||||
const valueBeforeCursor = part.value.substring(0, cursorPos);
|
||||
const valueAfterCursor = part.value.substring(cursorPos);
|
||||
|
||||
@@ -107,91 +95,23 @@ export const getContextData = (part: LogicalPart, cursorPos: number): ContextDat
|
||||
contextType: ContextType.Unknown,
|
||||
};
|
||||
|
||||
// Determine context type based on logical part type
|
||||
determineContextType(part, valueBeforeCursor, valueAfterCursor, metaData);
|
||||
if (part.type === LogicalPartType.Filter) {
|
||||
const noColon = !valueBeforeCursor.includes(":") && !valueAfterCursor.includes(":");
|
||||
if (noColon) {
|
||||
metaData.contextType = ContextType.FilterUnknown;
|
||||
} else if (valueBeforeCursor.includes(":")) {
|
||||
const [filterName, filterValue] = valueBeforeCursor.split(":");
|
||||
metaData.contextType = ContextType.FilterValue;
|
||||
metaData.filterName = filterName;
|
||||
metaData.valueContext = filterValue;
|
||||
} else {
|
||||
metaData.contextType = ContextType.FilterName;
|
||||
}
|
||||
} else if (part.type === LogicalPartType.Pipe) {
|
||||
const valueStartWithPipe = PIPE_NAMES.some(p => part.value.startsWith(p));
|
||||
metaData.contextType = valueStartWithPipe ? ContextType.PipeValue : ContextType.PipeName;
|
||||
}
|
||||
|
||||
// Clean up quotes in valueContext
|
||||
metaData.valueContext = metaData.valueContext.replace(/^["']|["']$/g, "");
|
||||
|
||||
return metaData;
|
||||
};
|
||||
|
||||
/** Helper function to determine if a string starts with any of the pipe names */
|
||||
const startsWithPipe = (value: string): boolean => {
|
||||
return PIPE_NAMES.some(p => value.startsWith(p));
|
||||
};
|
||||
|
||||
/** Helper function to check for colon presence */
|
||||
const hasNoColon = (before: string, after: string): boolean => {
|
||||
return !before.includes(":") && !after.includes(":");
|
||||
};
|
||||
|
||||
/** Helper function to extract filter name and update metadata for filter values */
|
||||
const handleFilterValue = (valueBeforeCursor: string, metaData: ContextData): void => {
|
||||
const [filterName, ...filterValue] = valueBeforeCursor.split(":");
|
||||
metaData.contextType = ContextType.FilterValue;
|
||||
metaData.filterName = filterName;
|
||||
const enhanceOperators = ["=", "-", "!", "~", "<", ">", "<=", ">="] as const;
|
||||
const enhanceOperator = enhanceOperators.find(op => op === filterValue[0]);
|
||||
if (enhanceOperator) {
|
||||
metaData.valueContext = filterValue.slice(1).join(":");
|
||||
metaData.operator = `:${enhanceOperator}`;
|
||||
} else {
|
||||
metaData.valueContext = filterValue.join(":");
|
||||
metaData.operator = ":";
|
||||
}
|
||||
};
|
||||
|
||||
/** Function to determine context type based on part type and value */
|
||||
const determineContextType = (
|
||||
part: LogicalPart,
|
||||
valueBeforeCursor: string,
|
||||
valueAfterCursor: string,
|
||||
metaData: ContextData
|
||||
): void => {
|
||||
switch (part.type) {
|
||||
case LogicalPartType.Filter:
|
||||
handleFilterType(valueBeforeCursor, valueAfterCursor, metaData);
|
||||
break;
|
||||
|
||||
case LogicalPartType.Pipe:
|
||||
metaData.contextType = startsWithPipe(part.value)
|
||||
? ContextType.PipeValue
|
||||
: ContextType.PipeName;
|
||||
break;
|
||||
|
||||
case LogicalPartType.FilterOrPipe:
|
||||
handleFilterOrPipeType(part.value, valueBeforeCursor, metaData);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
/** Handle filter type context determination */
|
||||
const handleFilterType = (
|
||||
valueBeforeCursor: string,
|
||||
valueAfterCursor: string,
|
||||
metaData: ContextData
|
||||
): void => {
|
||||
if (hasNoColon(valueBeforeCursor, valueAfterCursor)) {
|
||||
metaData.contextType = ContextType.FilterUnknown;
|
||||
} else if (valueBeforeCursor.includes(":")) {
|
||||
handleFilterValue(valueBeforeCursor, metaData);
|
||||
} else {
|
||||
metaData.contextType = ContextType.FilterName;
|
||||
}
|
||||
};
|
||||
|
||||
/** Handle FilterOrPipeType context determination */
|
||||
const handleFilterOrPipeType = (
|
||||
value: string,
|
||||
valueBeforeCursor: string,
|
||||
metaData: ContextData
|
||||
): void => {
|
||||
if (startsWithPipe(value)) {
|
||||
metaData.contextType = ContextType.PipeValue;
|
||||
} else if (valueBeforeCursor.includes(":")) {
|
||||
handleFilterValue(valueBeforeCursor, metaData);
|
||||
} else {
|
||||
metaData.contextType = ContextType.FilterOrPipeName;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2,19 +2,15 @@ export enum LogicalPartType {
|
||||
Filter = "Filter",
|
||||
Pipe = "Pipe",
|
||||
Operator = "Operator",
|
||||
FilterOrPipe = "FilterOrPipe",
|
||||
}
|
||||
|
||||
export type LogicalPartPosition = [start: number, end: number];
|
||||
|
||||
export type LogicalPartSeparator = " " | "|";
|
||||
|
||||
export interface LogicalPart {
|
||||
id: number;
|
||||
value: string;
|
||||
type: LogicalPartType;
|
||||
position: LogicalPartPosition;
|
||||
separator?: LogicalPartSeparator;
|
||||
}
|
||||
|
||||
export interface ContextData {
|
||||
@@ -23,10 +19,6 @@ export interface ContextData {
|
||||
contextType: ContextType;
|
||||
valueContext: string;
|
||||
filterName?: string;
|
||||
query?: string;
|
||||
queryBeforeIncompleteFilter?: string;
|
||||
separator?: LogicalPartSeparator;
|
||||
operator?: ":" | ":!" | ":-" | ":=" | ":~" | ":<" | ":>" | ":<=" | ":>=";
|
||||
}
|
||||
|
||||
export enum ContextType {
|
||||
@@ -36,5 +28,4 @@ export enum ContextType {
|
||||
PipeName = "Pipes",
|
||||
PipeValue = "PipeValue",
|
||||
Unknown = "Unknown",
|
||||
FilterOrPipeName = "FilterOrPipeName",
|
||||
}
|
||||
|
||||
@@ -10,11 +10,11 @@ import { AUTOCOMPLETE_LIMITS } from "../../../../constants/queryAutocomplete";
|
||||
import { LogsFiledValues } from "../../../../api/types";
|
||||
import { useLogsDispatch, useLogsState } from "../../../../state/logsPanel/LogsStateContext";
|
||||
import { useTenant } from "../../../../hooks/useTenant";
|
||||
import { generateQuery } from "./utils";
|
||||
|
||||
type FetchDataArgs = {
|
||||
urlSuffix: string;
|
||||
setter: (value: LogsFiledValues[]) => void;
|
||||
setter: Dispatch<SetStateAction<AutocompleteOptions[]>>
|
||||
type: ContextType;
|
||||
params?: URLSearchParams;
|
||||
}
|
||||
|
||||
@@ -24,8 +24,7 @@ const icons = {
|
||||
[ContextType.FilterValue]: <ValueIcon/>,
|
||||
[ContextType.PipeName]: <FunctionIcon/>,
|
||||
[ContextType.PipeValue]: <LabelIcon/>,
|
||||
[ContextType.Unknown]: <ValueIcon/>,
|
||||
[ContextType.FilterOrPipeName]: <FunctionIcon/>
|
||||
[ContextType.Unknown]: <ValueIcon/>
|
||||
};
|
||||
|
||||
export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
@@ -62,7 +61,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
}));
|
||||
};
|
||||
|
||||
const fetchData = async ({ urlSuffix, setter, params }: FetchDataArgs) => {
|
||||
const fetchData = async ({ urlSuffix, setter, type, params }: FetchDataArgs) => {
|
||||
abortControllerRef.current.abort();
|
||||
abortControllerRef.current = new AbortController();
|
||||
const { signal } = abortControllerRef.current;
|
||||
@@ -74,7 +73,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
try {
|
||||
const cachedData = autocompleteCache.get(key);
|
||||
if (cachedData) {
|
||||
setter(cachedData);
|
||||
setter(processData(cachedData, type));
|
||||
setLoading(false);
|
||||
return;
|
||||
}
|
||||
@@ -87,7 +86,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
const value = (data?.values || []) as LogsFiledValues[];
|
||||
setter(value || []);
|
||||
setter(value ? processData(value, type) : []);
|
||||
dispatch({ type: "SET_AUTOCOMPLETE_CACHE", payload: { key, value } });
|
||||
}
|
||||
setLoading(false);
|
||||
@@ -102,7 +101,7 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
|
||||
// fetch field names
|
||||
useEffect(() => {
|
||||
const validContexts = [ContextType.FilterName, ContextType.FilterUnknown, ContextType.FilterOrPipeName];
|
||||
const validContexts = [ContextType.FilterName, ContextType.FilterUnknown];
|
||||
const isInvalidContext = !validContexts.includes(contextData?.contextType || ContextType.Unknown);
|
||||
if (!serverUrl || isInvalidContext) {
|
||||
return;
|
||||
@@ -110,14 +109,11 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
|
||||
setFieldNames([]);
|
||||
|
||||
const setter = (filterNames: LogsFiledValues[]) => {
|
||||
setFieldNames(processData(filterNames, ContextType.FilterName));
|
||||
};
|
||||
|
||||
fetchData({
|
||||
urlSuffix: "field_names",
|
||||
setter: setter,
|
||||
params: getQueryParams({ query: contextData?.queryBeforeIncompleteFilter || "*" })
|
||||
setter: setFieldNames,
|
||||
type: ContextType.FilterName,
|
||||
params: getQueryParams({ query: "*" })
|
||||
});
|
||||
|
||||
return () => abortControllerRef.current?.abort();
|
||||
@@ -132,14 +128,11 @@ export const useFetchLogsQLOptions = (contextData?: ContextData) => {
|
||||
|
||||
setFieldValues([]);
|
||||
|
||||
const setter = (filterValues: LogsFiledValues[]) => {
|
||||
setFieldValues(processData(filterValues, ContextType.FilterValue));
|
||||
};
|
||||
|
||||
fetchData({
|
||||
urlSuffix: "field_values",
|
||||
setter: setter,
|
||||
params: getQueryParams({ query: generateQuery(contextData), field: contextData.filterName })
|
||||
setter: setFieldValues,
|
||||
type: ContextType.FilterValue,
|
||||
params: getQueryParams({ query: "*", field: contextData.filterName })
|
||||
});
|
||||
|
||||
return () => abortControllerRef.current?.abort();
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
import { expect } from "vitest";
|
||||
import { generateQuery } from "./utils";
|
||||
import { ContextType } from "./types";
|
||||
|
||||
describe("utils", () => {
|
||||
describe("_time", () => {
|
||||
it("should return the trimmed value by `-`", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_time",
|
||||
query: "_stream:{type=\"WatchEvent\"} _time:2025-04-1",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_time=2025-04-1",
|
||||
valueContext: "2025-04-1"
|
||||
})).toStrictEqual("_stream:{type=\"WatchEvent\"} _time:2025-04");
|
||||
});
|
||||
|
||||
it("should return the trimmed value by `:` if char `-` also exist in the query", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_time",
|
||||
query: "_stream:{type=\"WatchEvent\"} _time:2025-04-10T23:45:5",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_time=2025-04-10T23:45:5",
|
||||
valueContext: "2025-04-10T23:45:5"
|
||||
})).toStrictEqual("_stream:{type=\"WatchEvent\"} _time:2025-04-10T23:45");
|
||||
});
|
||||
|
||||
it("should return default `*` instead of -time filter", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_time",
|
||||
query: "_stream:{type=\"WatchEvent\"} _time:202",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_time=202",
|
||||
valueContext: "202"
|
||||
})).toStrictEqual("_stream:{type=\"WatchEvent\"} *");
|
||||
});
|
||||
});
|
||||
|
||||
describe("_stream", () => {
|
||||
it("should add regexp to filter value", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_stream",
|
||||
query: "_stream:{type=\"WatchEve",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_stream:{type=\"WatchEve",
|
||||
valueContext: "{type=\"WatchEve"
|
||||
})).toStrictEqual("_stream:{type=~\"WatchEve.*\"}");
|
||||
});
|
||||
|
||||
it("should add regexp to filter value if cursor in the middle of value", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_stream",
|
||||
query: "_stream:{type=\"WatchEve\"}",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_stream:{type=\"WatchEve",
|
||||
valueContext: "{type=\"WatchEve"
|
||||
})).toStrictEqual("_stream:{type=~\"WatchEve.*\"}");
|
||||
});
|
||||
|
||||
it("should return * if do not have value after =", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_stream",
|
||||
query: "_stream:{type=",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_stream:{type=",
|
||||
valueContext: "{type="
|
||||
})).toStrictEqual("*");
|
||||
});
|
||||
});
|
||||
|
||||
it("_msg", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_msg",
|
||||
query: "_stream:{type=\"WatchEvent\"} _msg:453",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_msg:453",
|
||||
valueContext: "453"
|
||||
})).toStrictEqual("_stream:{type=\"WatchEvent\"} *");
|
||||
});
|
||||
|
||||
it("_stream_id", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "_stream:{type=\"WatchEvent\"}",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "_stream_id",
|
||||
query: "_stream:{type=\"WatchEvent\"} _stream_id:453",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "_stream_id:453",
|
||||
valueContext: "453"
|
||||
})).toStrictEqual("_stream:{type=\"WatchEvent\"} *");
|
||||
});
|
||||
|
||||
describe("other fields", () => {
|
||||
it("should add prefix filter to other type of field names", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "repo.name",
|
||||
query: "repo.name:Victori",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "repo.name:Victori",
|
||||
valueContext: "Victori"
|
||||
})).toStrictEqual("repo.name:Victori*");
|
||||
});
|
||||
|
||||
it("should add prefix filter to other type of field names with escaped via double quote", () => {
|
||||
expect(generateQuery({
|
||||
queryBeforeIncompleteFilter: "",
|
||||
contextType: ContextType.FilterValue,
|
||||
filterName: "repo.name",
|
||||
query: "repo.name:\"Victori",
|
||||
valueAfterCursor: "",
|
||||
valueBeforeCursor: "repo.name:\"Victori",
|
||||
valueContext: "Victori"
|
||||
})).toStrictEqual("repo.name:Victori*");
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,61 +0,0 @@
|
||||
import { ContextData } from "./types";
|
||||
|
||||
const getStreamFieldQuery = (valueContext: string) => {
|
||||
if (valueContext.includes("=")) {
|
||||
const [fieldName, fieldValue] = valueContext.split("=");
|
||||
if (fieldValue) {
|
||||
return `_stream:${fieldName}=~${fieldValue}.*"}`;
|
||||
}
|
||||
}
|
||||
|
||||
return "*";
|
||||
};
|
||||
|
||||
const getLastPartUntilDelimiter = (value: string, delimiter: string) => {
|
||||
const lastIndexOfDelimiter = value.lastIndexOf(delimiter);
|
||||
return lastIndexOfDelimiter !== -1 ? value.slice(0, lastIndexOfDelimiter) : "";
|
||||
};
|
||||
|
||||
const getDateQuery = (contextData: ContextData) => {
|
||||
let fieldValue = "";
|
||||
if (contextData.valueContext.includes(":")) {
|
||||
fieldValue = getLastPartUntilDelimiter(contextData.valueContext, ":");
|
||||
} else if (contextData.valueContext.includes("-")) {
|
||||
fieldValue = getLastPartUntilDelimiter(contextData.valueContext, "-");
|
||||
}
|
||||
return fieldValue ? `${contextData.filterName}:${fieldValue}` : "*";
|
||||
};
|
||||
|
||||
/**
|
||||
* Generates a query string based on the provided context data.
|
||||
*
|
||||
* The function processes the input based on the `filterName` property:
|
||||
*
|
||||
* - If `filterName` is `_msg` or `_stream_id`, the query cannot be generated specifically,
|
||||
* so a wildcard query (`"*"`) is returned.
|
||||
*
|
||||
* - If `filterName` is `_stream`, the query is generated using regexp (`{type=~"value.*"}`).
|
||||
*
|
||||
* - If `filterName` is `_time`, a simplified query is created by trimming the value up
|
||||
* to the first occurrence of a delimiter such as `-` or `:`.
|
||||
*
|
||||
* - For all other values of `filterName`, a prefix query is returned using
|
||||
* the `query` value with a `*` appended (e.g., `"value*"`).
|
||||
*
|
||||
* @param {ContextData} contextData - The context object containing query parameters and metadata.
|
||||
* @returns {string} The generated query string.
|
||||
*/
|
||||
export const generateQuery = (contextData: ContextData): string => {
|
||||
let fieldQuery = "";
|
||||
if (!contextData.filterName || !contextData.query || ["_msg", "_stream_id"].includes(contextData.filterName)) {
|
||||
fieldQuery = "*";
|
||||
} else if ("_stream" === contextData.filterName) {
|
||||
fieldQuery = getStreamFieldQuery(contextData.valueContext);
|
||||
} else if ("_time" === contextData.filterName) {
|
||||
fieldQuery = getDateQuery(contextData);
|
||||
} else {
|
||||
fieldQuery = `${contextData.filterName}:${contextData.valueContext}*`;
|
||||
}
|
||||
|
||||
return contextData.queryBeforeIncompleteFilter ? `${contextData.queryBeforeIncompleteFilter}${contextData.separator ?? " "}${fieldQuery}` : fieldQuery;
|
||||
};
|
||||
@@ -19,8 +19,8 @@ import {
|
||||
LOGS_URL_PARAMS,
|
||||
WITHOUT_GROUPING
|
||||
} from "../../../constants/logs";
|
||||
import { getFromStorage, saveToStorage } from "../../../utils/storage";
|
||||
import LogParsingSwitches from "../../Configurators/LogsSettings/LogParsingSwitches";
|
||||
import { useLocalStorageBoolean } from "../../../hooks/useLocalStorageBoolean";
|
||||
|
||||
const {
|
||||
GROUP_BY,
|
||||
@@ -48,7 +48,7 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
|
||||
const [dateFormat, setDateFormat] = useState(searchParams.get(DATE_FORMAT) || LOGS_DATE_FORMAT);
|
||||
const [errorFormat, setErrorFormat] = useState("");
|
||||
|
||||
const [disabledHovers, handleSetDisabledHovers] = useLocalStorageBoolean("LOGS_DISABLED_HOVERS");
|
||||
const [disabledHovers, setDisabledHovers] = useState(!!getFromStorage("LOGS_DISABLED_HOVERS"));
|
||||
|
||||
const isGroupChanged = groupBy !== LOGS_GROUP_BY;
|
||||
const isDisplayFieldsChanged = displayFields.length !== 1 || displayFields[0] !== LOGS_DISPLAY_FIELDS;
|
||||
@@ -117,6 +117,11 @@ const GroupLogsConfigurators: FC<Props> = ({ logs }) => {
|
||||
handleClose();
|
||||
};
|
||||
|
||||
const handleSetDisabledHovers = (value: boolean) => {
|
||||
setDisabledHovers(value);
|
||||
saveToStorage("LOGS_DISABLED_HOVERS", value);
|
||||
};
|
||||
|
||||
const tooltipContent = () => {
|
||||
if (!hasChanges) return title;
|
||||
return (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { FC } from "preact/compat";
|
||||
import React, { FC } from "preact/compat";
|
||||
import classNames from "classnames";
|
||||
import { MouseEvent as ReactMouseEvent, ReactNode } from "react";
|
||||
import "./style.scss";
|
||||
@@ -14,7 +14,6 @@ interface ButtonProps {
|
||||
disabled?: boolean
|
||||
children?: ReactNode
|
||||
className?: string
|
||||
"data-id"?: string
|
||||
onClick?: (e: ReactMouseEvent<HTMLButtonElement>) => void
|
||||
onMouseDown?: (e: ReactMouseEvent<HTMLButtonElement>) => void
|
||||
}
|
||||
@@ -32,7 +31,6 @@ const Button: FC<ButtonProps> = ({
|
||||
disabled,
|
||||
onClick,
|
||||
onMouseDown,
|
||||
"data-id": dataId
|
||||
}) => {
|
||||
|
||||
const classesButton = classNames({
|
||||
@@ -52,7 +50,6 @@ const Button: FC<ButtonProps> = ({
|
||||
aria-label={ariaLabel}
|
||||
onClick={onClick}
|
||||
onMouseDown={onMouseDown}
|
||||
data-id={dataId}
|
||||
>
|
||||
{startIcon}{children}{endIcon}
|
||||
</button>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import React from "react";
|
||||
import { getCssVariable } from "../../../utils/theme";
|
||||
|
||||
export const LogoIcon = () => (
|
||||
@@ -642,47 +643,3 @@ export const PauseIcon = () => (
|
||||
<path d="M6 19h4V5H6v14zm8-14v14h4V5h-4z" />
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const ScrollToTopIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path
|
||||
d="M8 12l4-4 4 4m-4-4v12"
|
||||
strokeWidth="2"
|
||||
stroke="currentColor"
|
||||
fill="none"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const SortIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path d="M4 3 L4 15 L1.5 15 L5.5 21 L9.5 15 L7 15 L7 3 Z"/>
|
||||
<path d="M13 21 L13 9 L10.5 9 L14.5 3 L18.5 9 L16 9 L16 21 Z"/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const SortArrowDownIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path d="M10.5 3 L10.5 15 L8 15 L12 21 L16 15 L13.5 15 L13.5 3 Z"/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export const SortArrowUpIcon = () => (
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="currentColor"
|
||||
>
|
||||
<path d="M10.5 21 L10.5 9 L8 9 L12 3 L16 9 L13.5 9 L13.5 21 Z"/>
|
||||
</svg>
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
import { FC, useEffect, useState } from "preact/compat";
|
||||
import Button from "../Main/Button/Button";
|
||||
import Tooltip from "../Main/Tooltip/Tooltip";
|
||||
import { ScrollToTopIcon } from "../Main/Icons";
|
||||
import classNames from "classnames";
|
||||
import "./style.scss";
|
||||
import { useCallback } from "react";
|
||||
|
||||
interface ScrollToTopButtonProps {
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const ScrollToTopButton: FC<ScrollToTopButtonProps> = ({ className }) => {
|
||||
const [isVisible, setIsVisible] = useState(false);
|
||||
|
||||
const checkScrollPosition = () => {
|
||||
const scrollPosition = window.pageYOffset || document.documentElement.scrollTop;
|
||||
const visibleHeightThreshold = window.innerHeight;
|
||||
|
||||
setIsVisible(scrollPosition > visibleHeightThreshold);
|
||||
};
|
||||
|
||||
const scrollToTop = useCallback(() => {
|
||||
window.scrollTo({
|
||||
top: 0,
|
||||
behavior: "smooth"
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
window.addEventListener("scroll", checkScrollPosition);
|
||||
checkScrollPosition();
|
||||
|
||||
return () => {
|
||||
window.removeEventListener("scroll", checkScrollPosition);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={classNames({
|
||||
"vm-scroll-to-top-button": true,
|
||||
"vm-scroll-to-top-button_visible": isVisible
|
||||
}, className)}
|
||||
>
|
||||
<Tooltip title="Scroll to top">
|
||||
<Button
|
||||
variant="contained"
|
||||
color="primary"
|
||||
onClick={scrollToTop}
|
||||
ariaLabel="Scroll to top"
|
||||
startIcon={<ScrollToTopIcon />}
|
||||
/>
|
||||
</Tooltip>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ScrollToTopButton;
|
||||
@@ -1,26 +0,0 @@
|
||||
@use "src/styles/variables" as *;
|
||||
|
||||
.vm-scroll-to-top-button {
|
||||
position: fixed;
|
||||
bottom: 20px;
|
||||
right: 20px;
|
||||
z-index: 4;
|
||||
opacity: 0;
|
||||
visibility: hidden;
|
||||
transition: opacity 0.3s, visibility 0.3s;
|
||||
|
||||
&_visible {
|
||||
opacity: 1;
|
||||
visibility: visible;
|
||||
}
|
||||
|
||||
.vm-button {
|
||||
border-radius: 50%;
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
}
|
||||
@@ -18,8 +18,8 @@ const title = "Table settings";
|
||||
interface TableSettingsProps {
|
||||
columns: string[];
|
||||
selectedColumns?: string[];
|
||||
tableCompact?: boolean;
|
||||
toggleTableCompact?: () => void;
|
||||
tableCompact: boolean;
|
||||
toggleTableCompact: () => void;
|
||||
onChangeColumns: (arr: string[]) => void
|
||||
}
|
||||
|
||||
@@ -195,20 +195,18 @@ const TableSettings: FC<TableSettingsProps> = ({
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{toggleTableCompact && tableCompact !== undefined && (
|
||||
<div className="vm-table-settings-modal-section">
|
||||
<div className="vm-table-settings-modal-section__title">
|
||||
<div className="vm-table-settings-modal-section">
|
||||
<div className="vm-table-settings-modal-section__title">
|
||||
Table view
|
||||
</div>
|
||||
<div className="vm-table-settings-modal-columns-list__item">
|
||||
<Switch
|
||||
label={"Compact view"}
|
||||
value={tableCompact}
|
||||
onChange={toggleTableCompact}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="vm-table-settings-modal-columns-list__item">
|
||||
<Switch
|
||||
label={"Compact view"}
|
||||
value={tableCompact}
|
||||
onChange={toggleTableCompact}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>)}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import { act, renderHook } from "@testing-library/preact";
|
||||
import { useLocalStorageBoolean } from "./useLocalStorageBoolean";
|
||||
import * as storageUtils from "../utils/storage";
|
||||
import { Mock } from "vitest";
|
||||
import { StorageKeys } from "../utils/storage";
|
||||
|
||||
vi.mock("../utils/storage");
|
||||
|
||||
const testStorageKey = "TEST_STORAGE_KEY" as StorageKeys;
|
||||
|
||||
describe("useLocalStorageBoolean", () => {
|
||||
const { getFromStorage, saveToStorage } = storageUtils;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("initializes with the value from localStorage", () => {
|
||||
const mockGetFromStorage = getFromStorage as Mock;
|
||||
mockGetFromStorage.mockReturnValueOnce(true);
|
||||
|
||||
const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
|
||||
|
||||
expect(result.current[0]).toBe(true);
|
||||
expect(getFromStorage).toHaveBeenCalledWith(testStorageKey);
|
||||
});
|
||||
|
||||
it("updates localStorage and state when setter is called", () => {
|
||||
const mockGetFromStorage = getFromStorage as Mock;
|
||||
mockGetFromStorage.mockReturnValueOnce(false);
|
||||
|
||||
const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
|
||||
|
||||
act(() => {
|
||||
result.current[1](true);
|
||||
});
|
||||
|
||||
expect(saveToStorage).toHaveBeenCalledWith(testStorageKey, true);
|
||||
expect(result.current[0]).toBe(false);
|
||||
});
|
||||
|
||||
it("reacts to changes in localStorage by storage events", () => {
|
||||
const mockGetFromStorage = getFromStorage as Mock;
|
||||
mockGetFromStorage.mockReturnValueOnce(false);
|
||||
|
||||
const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
|
||||
|
||||
// Simulate a storage event
|
||||
act(() => {
|
||||
mockGetFromStorage.mockReturnValueOnce(true);
|
||||
window.dispatchEvent(new StorageEvent("storage", { key: testStorageKey, newValue: "true" }));
|
||||
});
|
||||
|
||||
expect(result.current[0]).toBe(true);
|
||||
});
|
||||
|
||||
it("does not update state if the localStorage value remains the same", () => {
|
||||
const mockGetFromStorage = getFromStorage as Mock;
|
||||
mockGetFromStorage.mockReturnValueOnce(false);
|
||||
|
||||
const { result } = renderHook(() => useLocalStorageBoolean(testStorageKey));
|
||||
|
||||
act(() => {
|
||||
mockGetFromStorage.mockReturnValueOnce(false);
|
||||
window.dispatchEvent(new StorageEvent("storage", { key: testStorageKey, newValue: "false" }));
|
||||
});
|
||||
|
||||
expect(result.current[0]).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,31 +0,0 @@
|
||||
import { useMemo, useState } from "preact/compat";
|
||||
import { getFromStorage, saveToStorage, StorageKeys } from "../utils/storage";
|
||||
import useEventListener from "./useEventListener";
|
||||
import { useCallback } from "react";
|
||||
|
||||
/**
|
||||
* A custom hook that synchronizes a boolean state with a value stored in localStorage.
|
||||
*
|
||||
* @param {StorageKeys} key - The key used to access the corresponding value in localStorage.
|
||||
* @returns {[boolean, function]} A tuple containing the current boolean value from localStorage and a setter function to update the value in localStorage.
|
||||
*
|
||||
* The hook listens to the "storage" event to automatically update the state when the localStorage value changes.
|
||||
*/
|
||||
export const useLocalStorageBoolean = (key: StorageKeys): [boolean, (value: boolean) => void] => {
|
||||
const [value, setValue] = useState(!!getFromStorage(key));
|
||||
|
||||
const handleUpdateStorage = useCallback(() => {
|
||||
const newValue = !!getFromStorage(key);
|
||||
if (newValue !== value) {
|
||||
setValue(newValue);
|
||||
}
|
||||
}, [key, value]);
|
||||
|
||||
const setNewValue = useCallback((newValue: boolean) => {
|
||||
saveToStorage(key, newValue);
|
||||
}, [key]);
|
||||
|
||||
useEventListener("storage", handleUpdateStorage);
|
||||
|
||||
return useMemo(() => [value, setNewValue], [value, setNewValue]);
|
||||
};
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
&-header {
|
||||
background-color: $color-background-block;
|
||||
z-index: 3;
|
||||
z-index: 1;
|
||||
margin: -$padding-medium 0-$padding-medium 0;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
|
||||
@@ -1,67 +1,21 @@
|
||||
import React, { FC, useMemo, useCallback, createPortal } from "preact/compat";
|
||||
import React, { FC } from "preact/compat";
|
||||
import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
|
||||
import { createPortal } from "preact/compat";
|
||||
import JsonViewComponent from "../../../../../components/Views/JsonView/JsonView";
|
||||
import { ViewProps } from "../../types";
|
||||
import EmptyLogs from "../components/EmptyLogs/EmptyLogs";
|
||||
import JsonViewSettings from "./JsonViewSettings/JsonViewSettings";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import orderBy from "lodash.orderBy";
|
||||
import "./style.scss";
|
||||
import { Logs } from "../../../../../api/types";
|
||||
import { SortDirection } from "./types";
|
||||
import { useCallback } from "react";
|
||||
|
||||
const MemoizedJsonView = React.memo(JsonViewComponent);
|
||||
|
||||
const jsonQuerySortParam = "json_sort";
|
||||
const fieldSortQueryParamName = "json_field_sort";
|
||||
|
||||
const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
const getLogs = useCallback(() => data, [data]);
|
||||
|
||||
const [searchParams] = useSearchParams();
|
||||
const sortParam = searchParams.get(jsonQuerySortParam);
|
||||
const fieldSortParam = searchParams.get(fieldSortQueryParamName) as SortDirection;
|
||||
|
||||
const [sortField, sortDirection] = useMemo(() => {
|
||||
const [sortField, sortDirection] = sortParam?.split(":").map(decodeURIComponent) || [];
|
||||
return [sortField, sortDirection as "asc" | "desc" | undefined];
|
||||
}, [sortParam]);
|
||||
|
||||
const fields = useMemo(() => {
|
||||
const keys = new Set(data.flatMap(Object.keys));
|
||||
return Array.from(keys);
|
||||
}, [data]);
|
||||
|
||||
const orderedFieldsData = useMemo(() => {
|
||||
if (!fieldSortParam) return data;
|
||||
const orderedFields = fields.toSorted((a, b) => fieldSortParam === "asc" ? a.localeCompare(b): b.localeCompare(a));
|
||||
return data.map((item) => {
|
||||
return orderedFields.reduce((acc, field) => {
|
||||
if (item[field]) acc[field] = item[field];
|
||||
return acc;
|
||||
}, {} as Logs);
|
||||
});
|
||||
}, [fields, fieldSortParam, data]);
|
||||
|
||||
const sortedData = useMemo(() => {
|
||||
if (!sortField || !sortDirection) return orderedFieldsData;
|
||||
return orderBy(orderedFieldsData, [sortField], [sortDirection]);
|
||||
}, [orderedFieldsData, sortField, sortDirection]);
|
||||
|
||||
const renderSettings = () => {
|
||||
if (!settingsRef.current) return null;
|
||||
|
||||
return createPortal(
|
||||
data.length > 0 && (
|
||||
<div className="vm-json-view__settings-container">
|
||||
<DownloadLogsButton getLogs={getLogs} />
|
||||
<JsonViewSettings
|
||||
fields={fields}
|
||||
sortQueryParamName={jsonQuerySortParam}
|
||||
fieldSortQueryParamName={fieldSortQueryParamName}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
data.length > 0 && <DownloadLogsButton getLogs={getLogs} />,
|
||||
settingsRef.current
|
||||
);
|
||||
};
|
||||
@@ -71,11 +25,9 @@ const JsonView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
return (
|
||||
<>
|
||||
{renderSettings()}
|
||||
<MemoizedJsonView
|
||||
data={sortedData}
|
||||
/>
|
||||
<MemoizedJsonView data={data} />
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default JsonView;
|
||||
export default JsonView;
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
import { FC, useMemo, useRef } from "preact/compat";
|
||||
import Button from "../../../../../../components/Main/Button/Button";
|
||||
import { SettingsIcon, SortArrowDownIcon, SortArrowUpIcon, SortIcon } from "../../../../../../components/Main/Icons";
|
||||
import Tooltip from "../../../../../../components/Main/Tooltip/Tooltip";
|
||||
import Select from "../../../../../../components/Main/Select/Select";
|
||||
import useBoolean from "../../../../../../hooks/useBoolean";
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import Modal from "../../../../../../components/Main/Modal/Modal";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import "./style.scss";
|
||||
import { SortDirection } from "../types";
|
||||
|
||||
const title = "JSON settings";
|
||||
const directionList = ["asc", "desc"];
|
||||
|
||||
interface JsonSettingsProps {
|
||||
fields: string[];
|
||||
sortQueryParamName: string;
|
||||
fieldSortQueryParamName: string;
|
||||
}
|
||||
|
||||
const JsonViewSettings: FC<JsonSettingsProps> = ({
|
||||
fields,
|
||||
sortQueryParamName,
|
||||
fieldSortQueryParamName
|
||||
}) => {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
const buttonRef = useRef<HTMLDivElement>(null);
|
||||
const [fieldSortDirection, setFieldSortDirection] = useState<SortDirection>(null);
|
||||
|
||||
const {
|
||||
value: openSettings,
|
||||
toggle: toggleOpenSettings,
|
||||
setFalse: handleClose,
|
||||
} = useBoolean(false);
|
||||
|
||||
const [sortField, setSortField] = useState<string | null>(null);
|
||||
const [sortDirection, setSortDirection] = useState<SortDirection>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const sortParam = searchParams.get(sortQueryParamName);
|
||||
const isSortDirection = (value: string) : value is Exclude<SortDirection, null> => directionList.includes(value);
|
||||
if (sortParam) {
|
||||
const [field, direction] = sortParam.split(":").map(decodeURIComponent);
|
||||
if (field && (isSortDirection(direction))) {
|
||||
setSortField(field);
|
||||
setSortDirection(direction);
|
||||
}
|
||||
}
|
||||
|
||||
const fieldSortParam = searchParams.get(fieldSortQueryParamName);
|
||||
if (fieldSortParam === "asc" || fieldSortParam === "desc") {
|
||||
setFieldSortDirection(fieldSortParam);
|
||||
}
|
||||
}, [searchParams, sortQueryParamName, fieldSortQueryParamName, setSortField, setSortDirection, setFieldSortDirection]);
|
||||
|
||||
const updateSortParams = useCallback((field: string | null, direction: SortDirection) => {
|
||||
const updatedParams = new URLSearchParams(searchParams.toString());
|
||||
|
||||
if (!field || !direction) {
|
||||
updatedParams.delete(sortQueryParamName);
|
||||
} else {
|
||||
updatedParams.set(sortQueryParamName, `${field}:${direction || ""}`);
|
||||
}
|
||||
|
||||
setSearchParams(updatedParams);
|
||||
}, [searchParams, sortQueryParamName]);
|
||||
|
||||
const handleSort = (field: string) => {
|
||||
const newDirection: SortDirection = sortDirection || "asc";
|
||||
setSortField(field);
|
||||
setSortDirection(newDirection);
|
||||
updateSortParams(field, newDirection);
|
||||
};
|
||||
|
||||
const resetSort = () => {
|
||||
setSortField(null);
|
||||
setSortDirection(null);
|
||||
updateSortParams(null, null);
|
||||
};
|
||||
|
||||
const changeFieldSortDirection = useCallback(() => {
|
||||
let newFieldSortDirection: SortDirection = null;
|
||||
if (fieldSortDirection === null) {
|
||||
newFieldSortDirection = "asc";
|
||||
}else if (fieldSortDirection === "asc") {
|
||||
newFieldSortDirection = "desc";
|
||||
}
|
||||
setFieldSortDirection(newFieldSortDirection);
|
||||
const updatedParams = new URLSearchParams(searchParams.toString());
|
||||
|
||||
if (!newFieldSortDirection) {
|
||||
updatedParams.delete(fieldSortQueryParamName);
|
||||
} else {
|
||||
updatedParams.set(fieldSortQueryParamName, encodeURIComponent(newFieldSortDirection));
|
||||
}
|
||||
|
||||
setSearchParams(updatedParams);
|
||||
},[fieldSortDirection, searchParams, fieldSortQueryParamName]);
|
||||
|
||||
const handleChangeSortDirection = (direction: string) => {
|
||||
const field = sortField || fields[0];
|
||||
setSortField(field);
|
||||
setSortDirection(direction as SortDirection);
|
||||
updateSortParams(field, direction as SortDirection);
|
||||
};
|
||||
|
||||
const fieldSortMeta = useMemo(() => ({
|
||||
default: {
|
||||
title: "Set field sort order. Click to sort in ascending order",
|
||||
icon: <SortIcon />
|
||||
},
|
||||
asc: {
|
||||
title: "Fields sorted ascending. Click to sort in descending order",
|
||||
icon: <SortArrowDownIcon />
|
||||
},
|
||||
desc: {
|
||||
title: "Fields sorted descending. Click to reset sort",
|
||||
icon: <SortArrowUpIcon />
|
||||
},
|
||||
}), []);
|
||||
|
||||
const fieldSortButton = useMemo(() => {
|
||||
const { title, icon } = fieldSortMeta[fieldSortDirection ?? "default"];
|
||||
return <Tooltip title={title}>
|
||||
<Button
|
||||
variant="text"
|
||||
startIcon={icon}
|
||||
onClick={changeFieldSortDirection}
|
||||
ariaLabel={title}
|
||||
/>
|
||||
</Tooltip>;
|
||||
}, [fieldSortDirection, toggleOpenSettings, changeFieldSortDirection, fieldSortMeta]);
|
||||
|
||||
|
||||
return (
|
||||
<div className="vm-json-settings">
|
||||
{fieldSortButton}
|
||||
<Tooltip title={title}>
|
||||
<div ref={buttonRef}>
|
||||
<Button
|
||||
variant="text"
|
||||
startIcon={<SettingsIcon/>}
|
||||
onClick={toggleOpenSettings}
|
||||
ariaLabel={title}
|
||||
/>
|
||||
</div>
|
||||
</Tooltip>
|
||||
{openSettings && (
|
||||
<Modal
|
||||
title={title}
|
||||
className="vm-json-settings-modal"
|
||||
onClose={handleClose}
|
||||
>
|
||||
<div className="vm-json-settings-modal-section">
|
||||
<div className="vm-json-settings-modal-section__sort-settings-container">
|
||||
<Select
|
||||
value={sortField || ""}
|
||||
onChange={handleSort}
|
||||
list={fields}
|
||||
label="Select field"
|
||||
/>
|
||||
<Select
|
||||
value={sortDirection || ""}
|
||||
onChange={handleChangeSortDirection}
|
||||
list={directionList}
|
||||
label="Sort direction"
|
||||
/>
|
||||
{(sortField || sortDirection) && (
|
||||
<Button
|
||||
variant="outlined"
|
||||
color="error"
|
||||
onClick={resetSort}
|
||||
>
|
||||
Reset sort
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default JsonViewSettings;
|
||||
@@ -1,34 +0,0 @@
|
||||
@use "src/styles/variables" as *;
|
||||
|
||||
.vm-json-settings {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
|
||||
&-modal {
|
||||
.vm-modal-content-body {
|
||||
min-width: clamp(300px, 600px, 90vw);
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
&-section {
|
||||
padding-block: $padding-global;
|
||||
border-top: $border-divider;
|
||||
|
||||
&:first-child {
|
||||
padding-top: 0;
|
||||
border-top: none;
|
||||
}
|
||||
|
||||
&__sort-settings-container {
|
||||
display: grid;
|
||||
padding: $padding-medium;
|
||||
grid-template-columns: 1fr 1fr 80px;
|
||||
gap: $padding-medium;
|
||||
|
||||
@media (max-width: 500px) {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
@use "src/styles/variables" as *;
|
||||
|
||||
.vm-json-view {
|
||||
&__settings-container {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
export type SortDirection = "asc" | "desc" | null;
|
||||
@@ -19,8 +19,8 @@ interface LiveTailingSettingsProps {
|
||||
handleResumeLiveTailing: () => void;
|
||||
pauseLiveTailing: () => void;
|
||||
clearLogs: () => void;
|
||||
isRawJsonView: boolean;
|
||||
onRawJsonViewChange: (value: boolean) => void;
|
||||
isCompactTailingNumber: boolean;
|
||||
handleSetCompactTailing: (value: boolean) => void;
|
||||
}
|
||||
|
||||
const LiveTailingSettings: FC<LiveTailingSettingsProps> = ({
|
||||
@@ -32,8 +32,8 @@ const LiveTailingSettings: FC<LiveTailingSettingsProps> = ({
|
||||
handleResumeLiveTailing,
|
||||
pauseLiveTailing,
|
||||
clearLogs,
|
||||
isRawJsonView,
|
||||
onRawJsonViewChange
|
||||
isCompactTailingNumber,
|
||||
handleSetCompactTailing
|
||||
}) => {
|
||||
const settingButtonRef = useRef<HTMLDivElement>(null);
|
||||
const { value: isSettingsOpen, setFalse: closeSettings, setTrue: openSettings } = useBoolean(false);
|
||||
@@ -106,12 +106,12 @@ const LiveTailingSettings: FC<LiveTailingSettingsProps> = ({
|
||||
<div className="vm-live-tailing-view__settings-modal">
|
||||
<div className={"vm-live-tailing-view__settings-modal-item"}>
|
||||
<Switch
|
||||
label={"Raw JSON View"}
|
||||
value={isRawJsonView}
|
||||
onChange={onRawJsonViewChange}
|
||||
label={"Expandable Properties View"}
|
||||
value={isCompactTailingNumber}
|
||||
onChange={handleSetCompactTailing}
|
||||
/>
|
||||
<span className="vm-group-logs-configurator-item__info">
|
||||
When this option is enabled, logs will be displayed in raw JSON format. This improves performance and uses less CPU and memory.
|
||||
Switches log display to expandable properties view with additional visualization settings. Please note: when processing large volumes of data, it may increase system response time.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -7,18 +7,16 @@ import { useLiveTailingLogs } from "./useLiveTailingLogs";
|
||||
import { LOGS_DISPLAY_FIELDS, LOGS_URL_PARAMS } from "../../../../../constants/logs";
|
||||
import { useMemo } from "react";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import throttle from "lodash.throttle";
|
||||
import throttle from "lodash/throttle";
|
||||
import GroupLogsItem from "../../../GroupLogs/GroupLogsItem";
|
||||
import LiveTailingSettings from "./LiveTailingSettings";
|
||||
import Alert from "../../../../../components/Main/Alert/Alert";
|
||||
import { isDecreasing } from "../../../../../utils/array";
|
||||
import { useLocalStorageBoolean } from "../../../../../hooks/useLocalStorageBoolean";
|
||||
import ScrollToTopButton from "../../../../../components/ScrollToTopButton/ScrollToTopButton";
|
||||
|
||||
const SCROLL_THRESHOLD = 100;
|
||||
const scrollToBottom = () => window.scrollTo({
|
||||
top: document.documentElement.scrollHeight,
|
||||
behavior: "smooth"
|
||||
behavior: "instant"
|
||||
});
|
||||
const throttledScrollToBottom = throttle(scrollToBottom, 200);
|
||||
|
||||
@@ -30,7 +28,8 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
|
||||
const { setSearchParamsFromKeys } = useSearchParamsFromObject();
|
||||
const [rowsPerPage, setRowsPerPage] = useStateSearchParams(100, "rows_per_page");
|
||||
const [query, _setQuery] = useStateSearchParams("*", "query");
|
||||
const [isRawJsonView, setIsRawJsonView] = useLocalStorageBoolean("RAW_JSON_LIVE_VIEW");
|
||||
const [isCompactTailingStr] = useStateSearchParams(0, "compact_tailing");
|
||||
const isCompactTailingNumber = Boolean(Number(isCompactTailingStr));
|
||||
const {
|
||||
logs,
|
||||
isPaused,
|
||||
@@ -55,6 +54,10 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
|
||||
setSearchParamsFromKeys({ rows_per_page: limit });
|
||||
}, [setRowsPerPage, setSearchParamsFromKeys]);
|
||||
|
||||
const handleSetCompactTailing = useCallback((value: boolean) => {
|
||||
setSearchParamsFromKeys({ compact_tailing: Number(value) });
|
||||
}, [setSearchParamsFromKeys]);
|
||||
|
||||
useEffect(() => {
|
||||
startLiveTailing();
|
||||
return () => stopLiveTailing();
|
||||
@@ -108,10 +111,9 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
|
||||
handleResumeLiveTailing={handleResumeLiveTailing}
|
||||
pauseLiveTailing={pauseLiveTailing}
|
||||
clearLogs={clearLogs}
|
||||
isRawJsonView={isRawJsonView}
|
||||
onRawJsonViewChange={setIsRawJsonView}
|
||||
isCompactTailingNumber={isCompactTailingNumber}
|
||||
handleSetCompactTailing={handleSetCompactTailing}
|
||||
/>
|
||||
<ScrollToTopButton />
|
||||
<div
|
||||
ref={containerRef}
|
||||
className="vm-live-tailing-view__container"
|
||||
@@ -120,31 +122,28 @@ const LiveTailingView: FC<ViewProps> = ({ settingsRef }) => {
|
||||
? (<div className="vm-live-tailing-view__empty">Waiting for logs...</div>)
|
||||
: (<div className="vm-live-tailing-view__logs">
|
||||
{logs.map(({ _log_id, ...log }, idx) =>
|
||||
isRawJsonView ? (
|
||||
<pre
|
||||
key={idx}
|
||||
className="vm-live-tailing-view__log-row"
|
||||
onMouseDown={pauseLiveTailing}
|
||||
>
|
||||
{JSON.stringify(log)}
|
||||
</pre>
|
||||
) : (
|
||||
<GroupLogsItem
|
||||
key={_log_id}
|
||||
log={log}
|
||||
onItemClick={pauseLiveTailing}
|
||||
hideGroupButton={true}
|
||||
displayFields={displayFields}
|
||||
/>
|
||||
)
|
||||
isCompactTailingNumber
|
||||
? (
|
||||
<GroupLogsItem
|
||||
key={_log_id}
|
||||
log={log}
|
||||
onItemClick={pauseLiveTailing}
|
||||
hideGroupButton={true}
|
||||
displayFields={displayFields}
|
||||
/>
|
||||
) : (
|
||||
<pre
|
||||
key={idx}
|
||||
className="vm-live-tailing-view__log-row"
|
||||
>
|
||||
{JSON.stringify(log)}
|
||||
</pre>
|
||||
)
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{isLimitedLogsPerUpdate && (
|
||||
<Alert variant="warning">Too many logs per second detected. Large volumes of log data are difficult to process
|
||||
and may impact performance. We recommend adding filters to your query for better analysis and system
|
||||
performance.</Alert>)}
|
||||
{isLimitedLogsPerUpdate && (<Alert variant="warning">Too many logs per second detected. Large volumes of log data are difficult to process and may impact performance. We recommend adding filters to your query for better analysis and system performance.</Alert>)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -34,10 +34,9 @@
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
overflow: auto;
|
||||
min-height: calc(100vh - 120px);
|
||||
min-height: 200px;
|
||||
font-family: $font-family-monospace;
|
||||
padding-bottom: $padding-medium;
|
||||
transition: min-height 0.3s ease;
|
||||
}
|
||||
|
||||
&__empty {
|
||||
|
||||
@@ -4,8 +4,19 @@ import { Logs } from "../../../../../api/types";
|
||||
import { useAppState } from "../../../../../state/common/StateContext";
|
||||
import useBoolean from "../../../../../hooks/useBoolean";
|
||||
import { useTenant } from "../../../../../hooks/useTenant";
|
||||
import { LogFlowAnalyzer } from "./utils";
|
||||
|
||||
/**
|
||||
* Defines the maximum number of consecutive times logs can be fetched above the threshold
|
||||
* before showing a warning notification, and vice versa:
|
||||
* - If logs are fetched above a threshold this many times in a row -> show warning
|
||||
* - If warning is shown, it won't disappear until logs are fetched below a threshold
|
||||
* this many times in a row
|
||||
*
|
||||
* This threshold helps optimize log display performance when dealing with large volumes of logs.
|
||||
* If the threshold is consistently exceeded, users will be prompted to add filters to their query
|
||||
* for better system performance and more focused log analysis.
|
||||
*/
|
||||
const MAX_ATTEMPTS_FETCH_LOGS_PER_SECOND = 5;
|
||||
/**
|
||||
* Defines the log's threshold, after which will be shown a warning notification
|
||||
*/
|
||||
@@ -45,7 +56,7 @@ const createStreamProcessor = (
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.name !== "AbortError") {
|
||||
console.error("Stream processing error:", e);
|
||||
restartTailing();
|
||||
setError(String(e));
|
||||
}
|
||||
} finally {
|
||||
clearInterval(connectionCheckInterval);
|
||||
@@ -53,6 +64,31 @@ const createStreamProcessor = (
|
||||
};
|
||||
};
|
||||
|
||||
const updateLimitModeTracking = (
|
||||
linesCount: number,
|
||||
attemptsFetchLimitRef: React.MutableRefObject<number>,
|
||||
attemptsFetchLowRef: React.MutableRefObject<number>,
|
||||
isLimitedLogsPerUpdate: boolean,
|
||||
) => {
|
||||
if (linesCount > LOGS_THRESHOLD) {
|
||||
attemptsFetchLimitRef.current++;
|
||||
attemptsFetchLowRef.current = 0;
|
||||
} else {
|
||||
attemptsFetchLowRef.current++;
|
||||
attemptsFetchLimitRef.current = 0;
|
||||
}
|
||||
|
||||
if (attemptsFetchLimitRef.current > MAX_ATTEMPTS_FETCH_LOGS_PER_SECOND) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (attemptsFetchLowRef.current > MAX_ATTEMPTS_FETCH_LOGS_PER_SECOND) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return isLimitedLogsPerUpdate;
|
||||
};
|
||||
|
||||
const parseLogLines = (lines: string[], counterRef: React.MutableRefObject<bigint>): Logs[] => {
|
||||
return lines
|
||||
.map(line => {
|
||||
@@ -72,22 +108,27 @@ interface ProcessBufferedLogsParams {
|
||||
lines: string[];
|
||||
limit: number;
|
||||
counterRef: React.MutableRefObject<bigint>;
|
||||
attemptsFetchLimitRef: React.MutableRefObject<number>;
|
||||
attemptsFetchLowRef: React.MutableRefObject<number>;
|
||||
setIsLimitedLogsPerUpdate: (isLimited: boolean) => void;
|
||||
setLogs: React.Dispatch<React.SetStateAction<Logs[]>>;
|
||||
bufferLinesRef: React.MutableRefObject<string[]>;
|
||||
logFlowAnalyzerRef?: React.MutableRefObject<LogFlowAnalyzer>;
|
||||
isLimitedLogsPerUpdate: boolean;
|
||||
}
|
||||
|
||||
const processBufferedLogs = ({
|
||||
lines,
|
||||
limit,
|
||||
counterRef,
|
||||
attemptsFetchLimitRef,
|
||||
attemptsFetchLowRef,
|
||||
setIsLimitedLogsPerUpdate,
|
||||
setLogs,
|
||||
bufferLinesRef,
|
||||
logFlowAnalyzerRef
|
||||
isLimitedLogsPerUpdate
|
||||
}: ProcessBufferedLogsParams) => {
|
||||
const isLimitLogsMode = logFlowAnalyzerRef?.current?.update(lines.length) === "high";
|
||||
|
||||
const isLimitLogsMode = updateLimitModeTracking(lines.length, attemptsFetchLimitRef, attemptsFetchLowRef, isLimitedLogsPerUpdate);
|
||||
const limitedLines = isLimitLogsMode && lines.length > LOGS_THRESHOLD ? lines.slice(-LOGS_THRESHOLD) : lines;
|
||||
const newLogs = parseLogLines(limitedLines, counterRef);
|
||||
|
||||
@@ -114,7 +155,8 @@ export const useLiveTailingLogs = (query: string, limit: number) => {
|
||||
const intervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const bufferRef = useRef<string>("");
|
||||
const bufferLinesRef = useRef<string[]>([]);
|
||||
const logFlowAnalyzerRef = useRef(new LogFlowAnalyzer());
|
||||
const attemptsFetchLimitLogsPerSecondCountRef = useRef<number>(0);
|
||||
const attemptsFetchLowLogsPerSecondCountRef = useRef<number>(0);
|
||||
|
||||
const stopLiveTailing = useCallback(() => {
|
||||
if (readerRef.current) {
|
||||
@@ -197,10 +239,12 @@ export const useLiveTailingLogs = (query: string, limit: number) => {
|
||||
lines,
|
||||
limit,
|
||||
counterRef,
|
||||
attemptsFetchLimitRef: attemptsFetchLimitLogsPerSecondCountRef,
|
||||
attemptsFetchLowRef: attemptsFetchLowLogsPerSecondCountRef,
|
||||
setIsLimitedLogsPerUpdate,
|
||||
isLimitedLogsPerUpdate,
|
||||
setLogs,
|
||||
bufferLinesRef,
|
||||
logFlowAnalyzerRef
|
||||
bufferLinesRef
|
||||
});
|
||||
}, PROCESSING_INTERVAL_MS);
|
||||
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
export class LogFlowAnalyzer {
|
||||
private threshold: number;
|
||||
private windowSize: number;
|
||||
private minHighCount: number;
|
||||
private minNormalCount: number;
|
||||
private window: number[];
|
||||
private state: "normal" | "high";
|
||||
|
||||
/**
|
||||
* @param {number} threshold - The threshold value used for state evaluation. Defaults to 200.
|
||||
* @param {number} windowSize - The size of the window used for tracking data. Defaults to 10.
|
||||
* @param {number} minHighCount - The minimum number of high occurrences needed for state transition. Defaults to 6.
|
||||
* @param {number} minNormalCount - The minimum number of normal occurrences needed for state reset. Defaults to 2.
|
||||
* @return {void}
|
||||
*/
|
||||
constructor(threshold: number = 200, windowSize: number = 10, minHighCount: number = 6, minNormalCount: number = 2) {
|
||||
this.threshold = threshold;
|
||||
this.windowSize = windowSize;
|
||||
this.minHighCount = minHighCount;
|
||||
this.minNormalCount = minNormalCount;
|
||||
this.window = [];
|
||||
this.state = "normal";
|
||||
}
|
||||
|
||||
update(logCount: number): "normal" | "high" {
|
||||
this.window.push(logCount);
|
||||
if (this.window.length > this.windowSize) {
|
||||
this.window.shift();
|
||||
}
|
||||
|
||||
const highCount = this.window.filter((x) => x > this.threshold).length;
|
||||
|
||||
if (this.state === "normal") {
|
||||
if (highCount >= this.minHighCount) {
|
||||
this.state = "high";
|
||||
}
|
||||
} else if (this.state === "high") {
|
||||
if (highCount < this.minNormalCount) {
|
||||
this.state = "normal";
|
||||
}
|
||||
}
|
||||
|
||||
return this.state;
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import DownloadLogsButton from "../../../DownloadLogsButton/DownloadLogsButton";
|
||||
import { createPortal } from "preact/compat";
|
||||
import "./style.scss";
|
||||
import { ViewProps } from "../../types";
|
||||
import useBoolean from "../../../../../hooks/useBoolean";
|
||||
import useStateSearchParams from "../../../../../hooks/useStateSearchParams";
|
||||
import TableLogs from "../../TableLogs";
|
||||
import SelectLimit from "../../../../../components/Main/Pagination/SelectLimit/SelectLimit";
|
||||
@@ -17,6 +18,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
const { setSearchParamsFromKeys } = useSearchParamsFromObject();
|
||||
const [displayColumns, setDisplayColumns] = useState<string[]>([]);
|
||||
const [rowsPerPage, setRowsPerPage] = useStateSearchParams(100, "rows_per_page");
|
||||
const { value: tableCompact, toggle: toggleTableCompact } = useBoolean(false);
|
||||
|
||||
const columns = useMemo(() => {
|
||||
const keys = new Set<string>();
|
||||
@@ -50,6 +52,8 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
columns={columns}
|
||||
selectedColumns={displayColumns}
|
||||
onChangeColumns={setDisplayColumns}
|
||||
tableCompact={tableCompact}
|
||||
toggleTableCompact={toggleTableCompact}
|
||||
/>
|
||||
</div>
|
||||
</div>,
|
||||
@@ -65,7 +69,7 @@ const TableView: FC<ViewProps> = ({ data, settingsRef }) => {
|
||||
<MemoizedTableView
|
||||
logs={data}
|
||||
displayColumns={displayColumns}
|
||||
tableCompact={false}
|
||||
tableCompact={tableCompact}
|
||||
columns={columns}
|
||||
rowsPerPage={Number(rowsPerPage)}
|
||||
/>
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { FC, useMemo } from "preact/compat";
|
||||
import React, { FC, useMemo, useState } from "preact/compat";
|
||||
import { Logs } from "../../../api/types";
|
||||
import "./style.scss";
|
||||
import classNames from "classnames";
|
||||
import GroupLogsFieldRow from "./GroupLogsFieldRow";
|
||||
import { useLocalStorageBoolean } from "../../../hooks/useLocalStorageBoolean";
|
||||
import useEventListener from "../../../hooks/useEventListener";
|
||||
import { getFromStorage } from "../../../utils/storage";
|
||||
|
||||
interface Props {
|
||||
log: Logs;
|
||||
@@ -16,7 +17,16 @@ const GroupLogsFields: FC<Props> = ({ log, hideGroupButton }) => {
|
||||
.sort(([aKey], [bKey]) => aKey.localeCompare(bKey));
|
||||
}, [log]);
|
||||
|
||||
const [disabledHovers] = useLocalStorageBoolean("LOGS_DISABLED_HOVERS");
|
||||
const [disabledHovers, setDisabledHovers] = useState(!!getFromStorage("LOGS_DISABLED_HOVERS"));
|
||||
|
||||
const handleUpdateStage = () => {
|
||||
const newValDisabledHovers = !!getFromStorage("LOGS_DISABLED_HOVERS");
|
||||
if (newValDisabledHovers !== disabledHovers) {
|
||||
setDisabledHovers(newValDisabledHovers);
|
||||
}
|
||||
};
|
||||
|
||||
useEventListener("storage", handleUpdateStage);
|
||||
|
||||
return (
|
||||
<div
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import React, { FC, memo, useMemo } from "preact/compat";
|
||||
import React, { FC, memo, useMemo, useState } from "preact/compat";
|
||||
import { Logs } from "../../../api/types";
|
||||
import "./style.scss";
|
||||
import useBoolean from "../../../hooks/useBoolean";
|
||||
import { ArrowDownIcon, CopyIcon } from "../../../components/Main/Icons";
|
||||
import { ArrowDownIcon } from "../../../components/Main/Icons";
|
||||
import classNames from "classnames";
|
||||
import { useLogsState } from "../../../state/logsPanel/LogsStateContext";
|
||||
import dayjs from "dayjs";
|
||||
@@ -10,13 +10,10 @@ import { useTimeState } from "../../../state/time/TimeStateContext";
|
||||
import { marked } from "marked";
|
||||
import { useSearchParams } from "react-router-dom";
|
||||
import { LOGS_DATE_FORMAT, LOGS_URL_PARAMS } from "../../../constants/logs";
|
||||
import useEventListener from "../../../hooks/useEventListener";
|
||||
import { getFromStorage } from "../../../utils/storage";
|
||||
import { parseAnsiToHtml } from "../../../utils/ansiParser";
|
||||
import GroupLogsFields from "./GroupLogsFields";
|
||||
import { useLocalStorageBoolean } from "../../../hooks/useLocalStorageBoolean";
|
||||
import Button from "../../../components/Main/Button/Button";
|
||||
import Tooltip from "../../../components/Main/Tooltip/Tooltip";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import useCopyToClipboard from "../../../hooks/useCopyToClipboard";
|
||||
|
||||
interface Props {
|
||||
log: Logs;
|
||||
@@ -30,8 +27,6 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"], onItemClick,
|
||||
value: isOpenFields,
|
||||
toggle: toggleOpenFields,
|
||||
} = useBoolean(false);
|
||||
const [copied, setCopied] = useState<boolean>(false);
|
||||
const copyToClipboard = useCopyToClipboard();
|
||||
|
||||
const [searchParams] = useSearchParams();
|
||||
const { markdownParsing, ansiParsing } = useLogsState();
|
||||
@@ -73,29 +68,21 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"], onItemClick,
|
||||
return values;
|
||||
}, [log, hasFields, displayFields, ansiParsing]);
|
||||
|
||||
const [disabledHovers] = useLocalStorageBoolean("LOGS_DISABLED_HOVERS");
|
||||
const [disabledHovers, setDisabledHovers] = useState(!!getFromStorage("LOGS_DISABLED_HOVERS"));
|
||||
|
||||
const handleUpdateStage = () => {
|
||||
const newValDisabledHovers = !!getFromStorage("LOGS_DISABLED_HOVERS");
|
||||
if (newValDisabledHovers !== disabledHovers) {
|
||||
setDisabledHovers(newValDisabledHovers);
|
||||
}
|
||||
};
|
||||
|
||||
const handleClick = () => {
|
||||
toggleOpenFields();
|
||||
onItemClick?.(log);
|
||||
};
|
||||
|
||||
const handleCopy = useCallback(async (e: Event) => {
|
||||
e.stopPropagation();
|
||||
if (copied) return;
|
||||
try {
|
||||
await copyToClipboard(JSON.stringify(log, null, 2));
|
||||
setCopied(true);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}, [copied, copyToClipboard]);
|
||||
|
||||
useEffect(() => {
|
||||
if (copied === null) return;
|
||||
const timeout = setTimeout(() => setCopied(false), 2000);
|
||||
return () => clearTimeout(timeout);
|
||||
}, [copied]);
|
||||
useEventListener("storage", handleUpdateStage);
|
||||
|
||||
return (
|
||||
<div className="vm-group-logs-row">
|
||||
@@ -106,17 +93,6 @@ const GroupLogsItem: FC<Props> = ({ log, displayFields = ["_msg"], onItemClick,
|
||||
})}
|
||||
onClick={handleClick}
|
||||
>
|
||||
<Tooltip title={copied ? "Copied" : "Copy to clipboard"}>
|
||||
<Button
|
||||
className="vm-group-logs-row-content__copy-row"
|
||||
variant="text"
|
||||
color="gray"
|
||||
size="small"
|
||||
startIcon={<CopyIcon/>}
|
||||
onClick={handleCopy}
|
||||
ariaLabel="copy to clipboard"
|
||||
/>
|
||||
</Tooltip>
|
||||
{hasFields && (
|
||||
<div
|
||||
className={classNames({
|
||||
|
||||
@@ -132,7 +132,7 @@ $font-size-logs: var(--font-size-logs, $font-size-small);
|
||||
|
||||
&-content {
|
||||
display: flex;
|
||||
padding: 2px 24px 2px 0;
|
||||
padding: 2px 0;
|
||||
cursor: pointer;
|
||||
|
||||
&_interactive {
|
||||
@@ -140,23 +140,8 @@ $font-size-logs: var(--font-size-logs, $font-size-small);
|
||||
will-change: background-color;
|
||||
}
|
||||
|
||||
&__copy-row {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: 0;
|
||||
z-index: 1;
|
||||
visibility: hidden;
|
||||
|
||||
&.vm-button {
|
||||
padding: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
&_interactive:hover {
|
||||
background-color: $color-hover-black;
|
||||
.vm-group-logs-row-content__copy-row {
|
||||
visibility: visible;
|
||||
}
|
||||
}
|
||||
|
||||
&__arrow {
|
||||
|
||||
@@ -74,7 +74,7 @@ export const useFetchLogHits = (server: string, query: string) => {
|
||||
}
|
||||
}
|
||||
setIsLoading(prev => ({ ...prev, [id]: false }));
|
||||
}, [url, query, tenant]);
|
||||
}, [url, query]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
|
||||
@@ -14,13 +14,11 @@ export type StorageKeys = "AUTOCOMPLETE"
|
||||
| "THEME"
|
||||
| "LOGS_LIMIT"
|
||||
| "LOGS_MARKDOWN"
|
||||
| "LOGS_ANSI"
|
||||
| "LOGS_DISABLED_HOVERS"
|
||||
| "EXPLORE_METRICS_TIPS"
|
||||
| "LOGS_QUERY_HISTORY"
|
||||
| "METRICS_QUERY_HISTORY"
|
||||
| "SERVER_URL"
|
||||
| "RAW_JSON_LIVE_VIEW"
|
||||
| DeprecatedStorageKeys;
|
||||
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ publish-via-docker:
|
||||
--label "org.opencontainers.image.version=$(PKG_TAG)" \
|
||||
--label "org.opencontainers.image.created=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
||||
$(foreach registry,$(DOCKER_REGISTRIES),\
|
||||
--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)$(EXTRA_TAG_SUFFIX) \
|
||||
--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE) \
|
||||
) \
|
||||
-o type=image \
|
||||
--provenance=false \
|
||||
@@ -115,7 +115,7 @@ publish-via-docker:
|
||||
--label "org.opencontainers.image.version=$(PKG_TAG)" \
|
||||
--label "org.opencontainers.image.created=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
||||
$(foreach registry,$(DOCKER_REGISTRIES),\
|
||||
--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)$(EXTRA_TAG_SUFFIX)-scratch \
|
||||
--tag $(registry)/$(DOCKER_NAMESPACE)/$(APP_NAME):$(PKG_TAG)$(RACE)-scratch \
|
||||
) \
|
||||
-o type=image \
|
||||
--provenance=false \
|
||||
|
||||
@@ -36,30 +36,30 @@ services:
|
||||
user: root
|
||||
|
||||
vlinsert:
|
||||
image: victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
command:
|
||||
- "--storageNode=vlstorage-1:9428"
|
||||
- "--storageNode=vlstorage-2:9428"
|
||||
|
||||
vlselect-1:
|
||||
image: victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
command:
|
||||
- "--storageNode=vlstorage-1:9428"
|
||||
- "--storageNode=vlstorage-2:9428"
|
||||
vlselect-2:
|
||||
image: victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
command:
|
||||
- "--storageNode=vlstorage-1:9428"
|
||||
- "--storageNode=vlstorage-2:9428"
|
||||
|
||||
vlstorage-1:
|
||||
image: victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
command:
|
||||
- "--storageDataPath=/vlogs"
|
||||
volumes:
|
||||
- vldata-1:/vlogs
|
||||
vlstorage-2:
|
||||
image: victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
command:
|
||||
- "--storageDataPath=/vlogs"
|
||||
volumes:
|
||||
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
# VictoriaLogs instance, a single process responsible for
|
||||
# storing logs and serving read queries.
|
||||
victorialogs:
|
||||
image: victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
ports:
|
||||
- "9428:9428"
|
||||
command:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
services:
|
||||
# meta service will be ignored by compose
|
||||
.victorialogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
command:
|
||||
- -storageDataPath=/vlogs
|
||||
- -loggerFormat=json
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
include:
|
||||
- ../compose-base.yml
|
||||
name: fluentbit-oltp
|
||||
name: fluentbit-loki
|
||||
|
||||
@@ -59,7 +59,7 @@ services:
|
||||
- '--external.alert.source=explore?orgId=1&left=["now-1h","now","VictoriaMetrics",{"expr": },{"mode":"Metrics"},{"ui":[true,true,true,"none"]}]'
|
||||
restart: always
|
||||
vmanomaly:
|
||||
image: victoriametrics/vmanomaly:v1.24.1
|
||||
image: victoriametrics/vmanomaly:v1.23.3
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -3,7 +3,7 @@ version: "3"
|
||||
services:
|
||||
# Run `make package-victoria-logs` to build victoria-logs image
|
||||
vlogs:
|
||||
image: docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs
|
||||
image: docker.io/victoriametrics/victoria-logs:v1.23.3-victorialogs
|
||||
volumes:
|
||||
- vlogs:/vlogs
|
||||
ports:
|
||||
|
||||
@@ -14,25 +14,6 @@ aliases:
|
||||
---
|
||||
Please find the changelog for VictoriaMetrics Anomaly Detection below.
|
||||
|
||||
## v1.24.1
|
||||
Released: 2025-06-20
|
||||
|
||||
- BUGFIX: Resolved the issue first seen in [v1.23.0](#v1230) where some fit and infer jobs were silently skipped at task submission time (due to a bug in the new background scheduler behind [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler)) followed by similar warnings in the logs later on, such as:
|
||||
```shellhelp
|
||||
2025-06-19 14:32:50,568 - apscheduler.executors.default - WARNING - Run time of job "{job_name}" (trigger: interval[1 day, 0:00:00], next run at: 2025-06-20 14:32:50 UTC)" was missed by 0:00:01.024753
|
||||
```
|
||||
|
||||
- BUGFIX: Resolved the issue where `vmanomaly` service on [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) where `start_from` argument was set and [state restoration](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) was enabled, didn't resume infer jobs after respective fitted models were restored from the previous run. This could lead to a situation where the service, *if restore happened in-between fit calls*, would not produce any anomaly scores and stay idle until the next `fit_every` happens, which is *expected in stateless mode*, but not in *stateful* mode with `restore_state` enabled.
|
||||
|
||||
## v1.24.0
|
||||
Released: 2025-06-18
|
||||
|
||||
- FEATURE: Introduced stateful `vmanomaly` service with job persistence and state restoration capabilities. Added a new [`restore_state`](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) setting that enables the service to persist and restore its state between runs, including anomaly detection model instances and training data. This prevents unnecessary model refitting when restarting the service, significantly reducing startup time and computational overhead.
|
||||
|
||||
- IMPROVEMENT: More informative log messages for fit and infer stages and for sub-optimal configurations used in the [sharded mode](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability).
|
||||
|
||||
- BUGFIX: Now system interrupt signals are properly handled and lead to expected graceful shutdown if for some reason new background scheduler, introduced in [v1.23.0](#v1230) was already stopped in the middle of the fit or infer call. Previously, this could lead to a service crash with an unhandled exception.
|
||||
|
||||
## v1.23.3
|
||||
Released: 2025-06-13
|
||||
|
||||
@@ -55,13 +36,6 @@ Released: 2025-06-08
|
||||
## v1.23.0
|
||||
Released: 2025-06-05
|
||||
|
||||
> There is a known bug that can cause some fit and infer jobs to be silently skipped at task submission time (due to a bug in the new background scheduler behind [`PeriodicScheduler`](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler)) followed by similar warnings in the logs later on, such as:
|
||||
> ```shellhelp
|
||||
> 2025-06-19 14:32:50,568 - apscheduler.executors.default - WARNING - Run time of job "{job_name}" (trigger: interval[1 day, 0:00:00], next run at: 2025-06-20 14:32:50 UTC)" was missed by 0:00:01.024753
|
||||
> ```
|
||||
> Releases affected: [v1.23.0](#v1230) - [v1.23.3](#v1233).
|
||||
> **The issue has been resolved in patch [v1.24.1](#v1241), upgrade is recommended.**
|
||||
|
||||
- FEATURE: Added `decay` [argument](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay) to [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models). This parameters allows for newer data to be weighted more heavily in online models. By default this is set to 1 which means all data points are weighted the same to maintain backward compatibility with existing configs. The closer this value is to 0 the more important new data is.
|
||||
|
||||
- IMPROVEMENT: **Restored back parallelization** in the read/fit/infer pipeline, previously disabled in [v1.22.0](#v1220-experimental) due to deadlock issues. The new implementation prevents deadlocks, allowing to control the parallelization level via `n_workers` in [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/). It's suggested to upgrade from [v1.22.0](#v1220) - [v1.22.1](#v1221) to this version to regain the performance benefits of parallel processing.
|
||||
|
||||
@@ -91,10 +91,6 @@ To visualize and interact with both [self-monitoring metrics](https://docs.victo
|
||||
- For guidance on using the `vmanomaly` Grafana dashboard and drilling down into anomaly score visualizations, refer to the [default preset section](https://docs.victoriametrics.com/anomaly-detection/presets/#default).
|
||||
- To monitor `vmanomaly` health, operational performance, and potential issues in real time, visit the [self-monitoring section](https://docs.victoriametrics.com/anomaly-detection/self-monitoring/).
|
||||
|
||||
## Is vmanomaly stateful?
|
||||
By default, `vmanomaly` is **stateless**, meaning it does not retain any state between service restarts. However, it can be configured {{% available_from "v1.24.0" anomaly %}} to be **stateful** by enabling the `restore_state` setting in the [settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings). This allows the service to restore its state from a previous run (training data, trained models), ensuring that models continue to produce [anomaly scores](#what-is-anomaly-score) right after restart and without requiring a full retraining process or re-querying training data from VictoriaMetrics. This is particularly useful for long-running services that need to maintain continuity in anomaly detection without losing previously learned patterns, especially when using [online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) that continuously adapt to new data and update their internal state.
|
||||
|
||||
Please refer to the [state restoration section](https://docs.victoriametrics.com/anomaly-detection/components/settings#state-restoration) for more details on how it works and how to configure it.
|
||||
|
||||
## Choosing the right model for vmanomaly
|
||||
Selecting the best model for `vmanomaly` depends on the data's nature and the [types of anomalies](https://victoriametrics.com/blog/victoriametrics-anomaly-detection-handbook-chapter-2/#categories-of-anomalies) to detect. For instance, [Z-score](https://docs.victoriametrics.com/anomaly-detection/components/models#online-z-score) is suitable for data without trends or seasonality, while more complex patterns might require models like [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models#prophet).
|
||||
@@ -177,10 +173,10 @@ Anomaly scores for historical (backtesting) period can be produced and written b
|
||||
schedulers:
|
||||
scheduler_alias:
|
||||
class: 'backtesting' # or "scheduler.backtesting.BacktestingScheduler" until v1.13.0
|
||||
# define historical (inference) period to backtest on
|
||||
# define historical period to backtest on
|
||||
# should be bigger than at least (fit_window + fit_every) time range
|
||||
from_iso: '2024-01-01T00:00:00Z'
|
||||
to_iso: '2024-01-15T00:00:00Z'
|
||||
inference_only: True # to treat from-to as inference period, with automated fit intervals construction
|
||||
# copy these from your PeriodicScheduler args
|
||||
fit_window: 'P14D'
|
||||
fit_every: 'PT1H'
|
||||
@@ -220,9 +216,7 @@ Configuration above will produce N intervals of full length (`fit_window`=14d +
|
||||
|
||||
### On-disk mode
|
||||
|
||||
> {{% available_from "v1.13.0" anomaly %}} There is an option to save anomaly detection models to the host filesystem after the `fit` stage (instead of keeping them in memory by default). This is particularly useful for **resource-intensive setups** (e.g., many models, many metrics, or larger [`fit_window` argument](https://docs.victoriametrics.com/anomaly-detection/components/scheduler#periodic-scheduler-config-example)) and for 3rd-party models that store fit data (such as [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models#prophet) or [HoltWinters](https://docs.victoriametrics.com/anomaly-detection/components/models#holt-winters)). This reduces RAM consumption significantly, though at the cost of slightly slower `infer` stages. To enable this, set the environment variable `VMANOMALY_MODEL_DUMPS_DIR` to the desired location. If using [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md), starting from chart version `1.3.0` `.persistentVolume.enabled` should be set to `true` in [values.yaml](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/values.yaml). Similar optimization is available for data read from VictoriaMetrics TSDB {{% available_from "v1.16.0" anomaly %}}. To use this, set the environment variable `VMANOMALY_DATA_DUMPS_DIR` to the desired location.
|
||||
|
||||
> {{% available_from "v1.24.0" anomaly %}} This feature is best used in conjunction with [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings#state-restoration) to ensure that the model state is preserved across service restarts.
|
||||
> {{% available_from "v1.13.0" anomaly %}} There is an option to save anomaly detection models to the host filesystem after the `fit` stage (instead of keeping them in memory by default). This is particularly useful for **resource-intensive setups** (e.g., many models, many metrics, or larger [`fit_window` argument](https://docs.victoriametrics.com/anomaly-detection/components/scheduler#periodic-scheduler-config-example)) and for 3rd-party models that store fit data (such as [ProphetModel](https://docs.victoriametrics.com/anomaly-detection/components/models#prophet) or [HoltWinters](https://docs.victoriametrics.com/anomaly-detection/components/models#holt-winters)). This reduces RAM consumption significantly, though at the cost of slightly slower `infer` stages. To enable this, set the environment variable `VMANOMALY_MODEL_DUMPS_DIR` to the desired location. If using [Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md), starting from chart version `1.3.0` `.persistentVolume.enabled` should be set to `true` in [values.yaml](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/values.yaml). Similar optimization is available for data read from VictoriaMetrics TSDB {{% available_from "v1.16.0" anomaly %}}. To use this, set the environment variable `VMANOMALY_DATA_DUMPS_DIR` to the desired location.
|
||||
|
||||
Here's an example of how to set it up in docker-compose using volumes:
|
||||
```yaml
|
||||
@@ -230,7 +224,7 @@ services:
|
||||
# ...
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.24.1
|
||||
image: victoriametrics/vmanomaly:v1.23.3
|
||||
# ...
|
||||
ports:
|
||||
- "8490:8490"
|
||||
@@ -267,8 +261,6 @@ With the introduction of [online models](https://docs.victoriametrics.com/anomal
|
||||
- **Optimized resource utilization**: By spreading the computational load over time and reducing peak demands, online models make more efficient use of resources and inducing less data transfer from VictoriaMetrics TSDB, improving overall system performance.
|
||||
- **Faster convergence**: Online models can adapt {{% available_from "v1.23.0" anomaly %}} to changes in data patterns more quickly, which is particularly beneficial in dynamic environments where data characteristics may shift frequently. See `decay` argument description [here](https://docs.victoriametrics.com/anomaly-detection/components/models/#decay).
|
||||
|
||||
> {{% available_from "v1.24.0" anomaly %}} Online models are best used in conjunction with [stateful mode](https://docs.victoriametrics.com/anomaly-detection/components/settings#state-restoration) to preserve the model state across service restarts. This allows the model to continue adapting to new data without losing previously learned patterns, thus avoiding the need for a full `fit` stage to start working again.
|
||||
|
||||
Here's an example of how we can switch from (offline) [Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#z-score) to [Online Z-score model](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-z-score):
|
||||
|
||||
```yaml
|
||||
@@ -290,9 +282,6 @@ models:
|
||||
to something like
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
restore_state: True # to restore model state from previous runs if restarted, available since v1.24.0
|
||||
|
||||
schedulers:
|
||||
periodic:
|
||||
class: 'periodic'
|
||||
@@ -312,11 +301,11 @@ models:
|
||||
|
||||
As a result, switching from the offline Z-score model to the Online Z-score model results in significant data volume reduction, i.e. over one week:
|
||||
|
||||
**Old configuration**:
|
||||
**Old Configuration**:
|
||||
- `fit_window`: 2 days
|
||||
- `fit_every`: 1 hour
|
||||
|
||||
**New configuration**:
|
||||
**New Configuration**:
|
||||
- `fit_window`: 4 hours
|
||||
- `fit_every`: 180 days ( >1 week)
|
||||
|
||||
@@ -326,7 +315,7 @@ The new configuration performs only 1 `fit` call in 180 days, using 4 hours of d
|
||||
|
||||
P.s. `infer` data volume will remain the same for both models, so it does not affect the overall calculations.
|
||||
|
||||
**Data volume reduction**:
|
||||
**Data Volume Reduction**:
|
||||
- Old: 8064 hours/week (fit) + 168 hours/week (infer)
|
||||
- New: 4 hours/week (fit) + 168 hours/week (infer)
|
||||
|
||||
@@ -443,7 +432,7 @@ options:
|
||||
Here’s an example of using the config splitter to divide configurations based on the `extra_filters` argument from the reader section:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.24.1 && docker image tag victoriametrics/vmanomaly:v1.24.1 vmanomaly
|
||||
docker pull victoriametrics/vmanomaly:v1.23.3 && docker image tag victoriametrics/vmanomaly:v1.23.3 vmanomaly
|
||||
```
|
||||
|
||||
```sh
|
||||
|
||||
@@ -25,15 +25,15 @@ The following options are available:
|
||||
- [To run Docker image](#docker)
|
||||
- [To run in Kubernetes with Helm charts](#kubernetes-with-helm-charts)
|
||||
|
||||
> Anomaly detection models can be kept {{% available_from "v1.13.0" anomaly %}} **on host filesystem after `fit` stage** (instead of default in-memory option); This will drastically reduce RAM for larger configurations. Similar optimization {{% available_from "v1.16.0" anomaly %}} can be applied to data read from VictoriaMetrics TSDB. See instructions of how to enable it [here](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
|
||||
> There is a mode {{% available_from "v1.13.0" anomaly %}} to keep anomaly detection models on host filesystem after `fit` stage (instead of keeping them in-memory by default); This may lead to **noticeable reduction of RAM used** on bigger setups. Similar optimization {{% available_from "v1.16.0" anomaly %}} can be set for data read from VictoriaMetrics TSDB. See instructions [here](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode).
|
||||
|
||||
### Command-line arguments
|
||||
|
||||
The `vmanomaly` service supports a set of command-line arguments to configure its behavior, including options for licensing, logging levels, and more.
|
||||
The `vmanomaly` service supports several command-line arguments to configure its behavior, including options for licensing, logging levels, and more. These arguments can be passed when starting the service via Docker or any other setup. Below is the list of available options:
|
||||
|
||||
> `vmanomaly` supports {{% available_from "v1.18.5" anomaly %}} running on config **directories**, see the `config` positional arg description in help message below.
|
||||
|
||||
> Single-dashed command-line argument {{% available_from "v1.23.3" anomaly %}} format can be used, e.g. `-license.forceOffline` in addition to `--license.forceOffline`. This aligns better with other VictoriaMetrics ecosystem components. Mixing the two styles is also supported, e.g. `-license.forceOffline --loggerLevel INFO`.
|
||||
> Single-dashed command-line arguments {{% available_from "v1.23.3" anomaly %}} can be used, e.g. `-license.forceOffline` in addition to `--license.forceOffline`. This now aligns with other VictoriaMetrics ecosystem components in terms of command-line arguments. Mixing the two styles is also supported, e.g. `-license.forceOffline --loggerLevel INFO`.
|
||||
|
||||
```shellhelp
|
||||
usage: vmanomaly.py [-h] [--license STRING | --licenseFile PATH] [--license.forceOffline] [--loggerLevel {DEBUG,WARNING,FATAL,ERROR,INFO}] [--watch] [--dryRun] [--outputSpec PATH] config [config ...]
|
||||
@@ -62,9 +62,9 @@ You can specify these options when running `vmanomaly` to fine-tune logging leve
|
||||
|
||||
### Licensing
|
||||
|
||||
The license key can be specified with the help of the following [command-line](#command-line-arguments) arguments: `--license`, `--licenseFile`, `--license.forceOffline`
|
||||
The license key can be passed via the following command-line flags: `--license`, `--licenseFile`, `--license.forceOffline`
|
||||
|
||||
In order to make it easier to monitor the license expiration date, the following metrics are exposed (see
|
||||
In order to make it easier to monitor the license expiration date, the following metrics are exposed(see
|
||||
[Monitoring](https://docs.victoriametrics.com/anomaly-detection/components/monitoring/) section for details on how to scrape them):
|
||||
|
||||
```promtextmetric
|
||||
@@ -120,13 +120,13 @@ Below are the steps to get `vmanomaly` up and running inside a Docker container:
|
||||
1. Pull Docker image:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.24.1
|
||||
docker pull victoriametrics/vmanomaly:v1.23.3
|
||||
```
|
||||
|
||||
2. (Optional step) tag the `vmanomaly` Docker image:
|
||||
|
||||
```sh
|
||||
docker image tag victoriametrics/vmanomaly:v1.24.1 vmanomaly
|
||||
docker image tag victoriametrics/vmanomaly:v1.23.2 vmanomaly
|
||||
```
|
||||
|
||||
3. Start the `vmanomaly` Docker container with a *license file*, use the command below.
|
||||
@@ -160,7 +160,7 @@ docker run -it --user 1000:1000 \
|
||||
services:
|
||||
# ...
|
||||
vmanomaly:
|
||||
image: victoriametrics/vmanomaly:v1.24.1
|
||||
image: victoriametrics/vmanomaly:v1.23.3
|
||||
volumes:
|
||||
$YOUR_LICENSE_FILE_PATH:/license
|
||||
$YOUR_CONFIG_FILE_PATH:/config.yml
|
||||
@@ -191,27 +191,13 @@ You can run `vmanomaly` in Kubernetes environment
|
||||
with [these Helm charts](https://github.com/VictoriaMetrics/helm-charts/blob/master/charts/victoria-metrics-anomaly/README.md).
|
||||
|
||||
## How to configure vmanomaly
|
||||
To run `vmanomaly` you need to set up configuration file in `yaml` format.
|
||||
|
||||
To run `vmanomaly`, use YAML files or directories containing YAML files. The configuration files support shallow merge, allowing splitting the configuration into multiple files for better organization.
|
||||
> Before deploying, to check the correctness of your configuration validate config file(s) with `--dryRun` [command-line](#command-line-arguments) flag for chosen deployment method (Docker, Kubernetes, etc.). This will parse and merge all YAML files, run schema checks, logs errors and warnings (if found) and then exit without starting the service or requiring a license.
|
||||
|
||||
> If you are using directories, all `.yaml` files inside will be shallow merged, without deeper recursion. If you want to merge multiple YAML files, you can specify them as separate arguments, e.g.
|
||||
> ```shellhelp
|
||||
> vmanomaly config1.yaml config2.yaml ./config_dir/
|
||||
> ```
|
||||
|
||||
Before deploying, check the correctness of your configuration validate config file(s) with `--dryRun` [command-line](#command-line-arguments) flag for chosen deployment method (Docker, Kubernetes, etc.). This will parse and merge all YAML files, run schema checks, log errors and warnings (if found) and then exit without starting the service and requiring a license.
|
||||
|
||||
### Example
|
||||
|
||||
Here is an example of config file that will run [Prophet](https://docs.victoriametrics.com/anomaly-detection/components/models/#prophet) model on `vm_cache_entries` metric, with periodic scheduler that runs inference every minute and fits the model every day. The model will be trained on the last 2 weeks of data each time it is (re)fitted. The model will produce `anomaly_score`, `yhat`, `yhat_lower`, and `yhat_upper` [series](https://docs.victoriametrics.com/anomaly-detection/components/models/#vmanomaly-output) for debugging purposes. The model will be timezone-aware and will use cyclical encoding for the hour of the day and day of the week seasonality.
|
||||
Here is an example of config file that will run [Facebook Prophet](https://facebook.github.io/prophet/) model, that will be retrained every 2 hours on 14 days of previous data. It will generate [inference metrics](https://docs.victoriametrics.com/anomaly-detection/components/models#vmanomaly-output) (including `anomaly_score`) every 1 minute.
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/settings/
|
||||
n_workers: 4 # number of workers to run workload in parallel, set to 0 or negative number to use all available CPU cores
|
||||
anomaly_score_outside_data_range: 5.0 # default anomaly score for anomalies outside expected data range
|
||||
restore_state: True # restore state from previous run, available since v1.24.0
|
||||
|
||||
schedulers:
|
||||
1d_1m:
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler
|
||||
@@ -239,7 +225,6 @@ models:
|
||||
interval_width: 0.98 # see https://facebook.github.io/prophet/docs/uncertainty_intervals
|
||||
|
||||
reader:
|
||||
class: 'vm' # use VictoriaMetrics as a data source
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/reader/#vm-reader
|
||||
datasource_url: "http://victoriametrics:8428/" # [YOUR_DATASOURCE_URL]
|
||||
sampling_period: "1m"
|
||||
@@ -248,21 +233,12 @@ reader:
|
||||
cache: "sum(rate(vm_cache_entries))"
|
||||
|
||||
writer:
|
||||
class: 'vm' # use VictoriaMetrics as a data destination
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/writer/#vm-writer
|
||||
datasource_url: "http://victoriametrics:8428/" # [YOUR_DATASOURCE_URL]
|
||||
```
|
||||
|
||||
### Recommended steps
|
||||
|
||||
For optimal service behavior, consider the following tweaks when configuring `vmanomaly`:
|
||||
|
||||
- Set `settings.n_workers` {{% available_from "v1.23.0" anomaly %}} [arg](https://docs.victoriametrics.com/anomaly-detection/components/settings/#parallelization) > 1 to utilize more of available CPU cores for parallel workload processing. This can significantly improve performance, especially on larger datasets with a lot of `reader.queries` and longer `scheduler.fit_window` intervals. Setting it to zero or negative number will enable using all available CPU cores.
|
||||
|
||||
- Use [on-disk mode](https://docs.victoriametrics.com/anomaly-detection/faq/#on-disk-mode) {{% available_from "v1.13.0" anomaly %}} to reduce RAM usage, especially for larger datasets. This mode allows `vmanomaly` to keep models and the data on the host filesystem after the `fit` stage, rather than in memory.
|
||||
|
||||
- Use **state restoration** {{% available_from "v1.24.0" anomaly %}} to resume from the last known state for long-term stability. This is controlled by the `settings.restore_state` boolean [arg](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration).
|
||||
|
||||
**Schedulers**:
|
||||
- Configure the **inference frequency** in the [scheduler](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/) section of the configuration file.
|
||||
- Ensure that `infer_every` aligns with your **minimum required alerting frequency**.
|
||||
@@ -270,8 +246,7 @@ For optimal service behavior, consider the following tweaks when configuring `vm
|
||||
|
||||
**Reader**:
|
||||
- Setup the datasource to read data from in the [reader](https://docs.victoriametrics.com/anomaly-detection/components/reader/) section. Include tenant ID if using a [cluster version of VictoriaMetrics](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/) (`multitenant` value {{% available_from "v1.16.2" anomaly %}} can be also used here).
|
||||
- Define queries for input data using [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) under `reader.queries` section. Note, it's possible to override reader-level arguments at query level for increased flexibility, e.g. specifying per-query [timezone](https://docs.victoriametrics.com/anomaly-detection/faq#handling-timezones) or [sampling period](https://docs.victoriametrics.com/anomaly-detection/components/reader/#sampling-period).
|
||||
- For longer `fit_window` intervals in scheduler, consider splitting queries into smaller time ranges to avoid excessive memory usage, timeouts and hitting server-side constraints, so they can be queried separately and reconstructed on `vmanomaly` side. Please refer to this [example](https://docs.victoriametrics.com/anomaly-detection/faq/#handling-large-queries-in-vmanomaly) for more details.
|
||||
- Define queries for input data using [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/) under `reader.queries` section. Note, it's possible to override reader-level arguments at query level for increased flexibility, e.g. specifying per-query timezone, data frequency, data range, etc.
|
||||
|
||||
**Writer**:
|
||||
- Specify where and how to store anomaly detection metrics in the [writer](https://docs.victoriametrics.com/anomaly-detection/components/writer/) section.
|
||||
@@ -279,21 +254,17 @@ For optimal service behavior, consider the following tweaks when configuring `vm
|
||||
- Adding `for` label to `metric_format` argument is recommended for smoother visual experience in the [anomaly score dashboard](https://docs.victoriametrics.com/anomaly-detection/presets/#default). Please refer to `metric_format` argument description [here](https://docs.victoriametrics.com/anomaly-detection/components/writer/?highlight=metric_format#config-parameters).
|
||||
|
||||
**Models**:
|
||||
- Configure built-in models hyperparameters according to your needs in the [models](https://docs.victoriametrics.com/anomaly-detection/components/models/) section. Where possible, incorporate [domain knowledge](https://docs.victoriametrics.com/anomaly-detection/faq/#incorporating-domain-knowledge) for optimal results.
|
||||
- Configure built-in models parameters according to your needs in the [models](https://docs.victoriametrics.com/anomaly-detection/components/models/) section. Where possible, incorporate [domain knowledge](https://docs.victoriametrics.com/anomaly-detection/faq/#incorporating-domain-knowledge) for optimal results.
|
||||
- (Optional) Develop or integrate your [custom models](https://docs.victoriametrics.com/anomaly-detection/components/models/#custom-model-guide) with `vmanomaly`.
|
||||
- Adding `y` to `provide_series` [argument](https://docs.victoriametrics.com/anomaly-detection/components/models/#provide-series) values is recommended for smoother visual experience in the [anomaly score dashboard](https://docs.victoriametrics.com/anomaly-detection/presets/#default). Also, other `vmanomaly` [output](https://docs.victoriametrics.com/anomaly-detection/components/models#vmanomaly-output) series can be specified in `provide_series`, such as `yhat`, `yhat_lower`, `yhat_upper`, etc. This will allow you to visualize the expected values and their confidence intervals in the dashboard.
|
||||
> Only [univariate models](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) support the generation of such output. Other models, such as [multivariate](https://docs.victoriametrics.com/anomaly-detection/components/models/#multivariate-models) or [custom](https://docs.victoriametrics.com/anomaly-detection/components/models/#custom-model-guide), may not support this feature.
|
||||
|
||||
**Visualization**:
|
||||
- Set up [anomaly score dashboard](https://docs.victoriametrics.com/anomaly-detection/presets/#grafana-dashboard) to visualize the results of anomaly detection.
|
||||
- Set up [self-monitoring dashboard](https://docs.victoriametrics.com/anomaly-detection/self-monitoring/) to monitor the health of `vmanomaly` service and its components.
|
||||
- Adding `y` to `provide_series` arg values is recommended for smoother visual experience in the [anomaly score dashboard](https://docs.victoriametrics.com/anomaly-detection/presets/#default). Also, other `vmanomaly` [output](https://docs.victoriametrics.com/anomaly-detection/components/models#vmanomaly-output) can be used in `provide_series`.
|
||||
> **Note:** Only [univariate models](https://docs.victoriametrics.com/anomaly-detection/components/models/#univariate-models) support the generation of such output.
|
||||
|
||||
## Check also
|
||||
|
||||
Please refer to the following links for a deeper understanding of Anomaly Detection and `vmanomaly`:
|
||||
Here are the links for further deep dive into Anomaly Detection in general and `vmanomaly` in particular:
|
||||
|
||||
- [High Availability](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#high-availability) and [Horizontal Scalability](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability)
|
||||
- [State Restoration](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration)
|
||||
- [High Availability](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#high-availability)
|
||||
- [Horizontal Scalability](https://docs.victoriametrics.com/anomaly-detection/scaling-vmanomaly/#horizontal-scalability)
|
||||
- [Guide: Anomaly Detection and Alerting Setup](https://docs.victoriametrics.com/anomaly-detection/guides/guide-vmanomaly-vmalert/)
|
||||
- [FAQ](https://docs.victoriametrics.com/anomaly-detection/faq/)
|
||||
- [CHANGELOG](https://docs.victoriametrics.com/anomaly-detection/changelog/)
|
||||
|
||||
@@ -25,7 +25,6 @@ Here's a minimalistic full config example, demonstrating many-to-many configurat
|
||||
settings:
|
||||
n_workers: 4 # number of workers to run models in parallel
|
||||
anomaly_score_outside_data_range: 5.0 # default anomaly score for anomalies outside expected data range
|
||||
restore_state: True # restore state from previous run, if available
|
||||
|
||||
# how and when to run the models is defined by schedulers
|
||||
# https://docs.victoriametrics.com/anomaly-detection/components/scheduler/
|
||||
|
||||
@@ -513,19 +513,17 @@ Produced model instances are **stored in-memory** between consecutive re-fit cal
|
||||
|
||||
### Online Models
|
||||
|
||||
> Online models are best used **in combination with [stateful service](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) {{% available_from "v1.24.0" anomaly %}} to ensure that the model state is preserved if the service restarts and any aggregated model updates are not lost**. E.g. if the model was already trained on many weeks of data and is being updated on new datapoints every minute, there is no need to re-train it from scratch on the same data after each restart, as it can continue to update restored state on new datapoints.
|
||||
|
||||
Online (incremental) models {{% available_from "v1.15.0" anomaly %}} allow defining a smaller frame `fit_window` and less frequent `fit` calls to reduce the data burden from VictoriaMetrics. They make incremental updates to model parameters during each `infer_every` call, even on a single datapoint.
|
||||
Online (incremental) models{{% available_from "v1.15.0" anomaly %}} allow defining a smaller frame `fit_window` and less frequent `fit` calls to reduce the data burden from VictoriaMetrics. They make incremental updates to model parameters during each `infer_every` call, even on a single datapoint.
|
||||
If the model doesn't support online mode, it's called **offline** (its parameters are only updated during `fit` calls).
|
||||
|
||||
Main differences between offline and online:
|
||||
|
||||
**Fit stage**
|
||||
Fit stage
|
||||
- Both types have a `fit` stage, run on the `fit_window` data frame.
|
||||
- For offline models, `fit_window` should contain enough data to train the model (e.g., 2 seasonal periods).
|
||||
- For online models, training can start gradually from smaller chunks (e.g., 1 hour).
|
||||
|
||||
**Infer stage**
|
||||
Infer stage
|
||||
- Both types have an `infer` stage, run on new datapoints (timestamps > last seen timestamp of the previous `infer` call).
|
||||
- Offline models use a pre-trained (during `fit` call) *static* model to make every `infer` call until the next `fit` call, when the model is completely re-trained.
|
||||
- Online models use a pre-trained (during `fit` call) *dynamic* model, which is gradually updated during each `infer` call with new datapoints. However, to prevent the model from accumulating outdated behavior, each `fit` call resets the model from scratch.
|
||||
@@ -1278,7 +1276,7 @@ monitoring:
|
||||
Let's pull the docker image for `vmanomaly`:
|
||||
|
||||
```sh
|
||||
docker pull victoriametrics/vmanomaly:v1.24.1
|
||||
docker pull victoriametrics/vmanomaly:v1.23.3
|
||||
```
|
||||
|
||||
Now we can run the docker container putting as volumes both config and model file:
|
||||
@@ -1292,7 +1290,7 @@ docker run -it \
|
||||
-v $(PWD)/license:/license \
|
||||
-v $(PWD)/custom_model.py:/vmanomaly/model/custom.py \
|
||||
-v $(PWD)/custom.yaml:/config.yaml \
|
||||
victoriametrics/vmanomaly:v1.24.1 /config.yaml \
|
||||
victoriametrics/vmanomaly:v1.23.2 /config.yaml \
|
||||
--licenseFile=/license
|
||||
```
|
||||
|
||||
|
||||
@@ -70,8 +70,6 @@ options={`"scheduler.periodic.PeriodicScheduler"`, `"scheduler.oneoff.OneoffSche
|
||||
|
||||
## Periodic scheduler
|
||||
|
||||
> If `start_from` [parameter](#parameters-1) is used, it's suggested to also set `restore_state: true` in the [Settings section](https://docs.victoriametrics.com/anomaly-detection/components/settings/#state-restoration) of a config, so that the scheduler can restore its state from the previous run **if terminated or restarted in between scheduled runs** and continue producing anomaly scores without interruptions, otherwise the service will be idle until future `start_from` time is reached. E.g. if `start_from` is set to `20:00` and the service is started and then terminated and restarted at `20:30`, it will not produce any anomaly scores until the next day's `20:00` is reached (+23:30 of being idle), which introduces inconvenience for the users.
|
||||
|
||||
### Parameters
|
||||
|
||||
For periodic scheduler parameters are defined as differences in times, expressed in difference units, e.g. days, hours, minutes, seconds.
|
||||
|
||||
@@ -15,10 +15,6 @@ aliases:
|
||||
|
||||
Through the **Settings** section of a config, you can configure the following parameters of the anomaly detection service:
|
||||
|
||||
- [Anomaly score outside data range](#anomaly-score-outside-data-range) - specific anomaly score fo values outside the expected data range of particular query
|
||||
- [Parallelization](#parallelization) - number of workers to run workloads in parallel
|
||||
- [State restoration](#state-restoration) - whether to restore models' state in between runs if the service is restarted or stopped
|
||||
|
||||
## Anomaly Score Outside Data Range
|
||||
|
||||
This argument allows you to override the anomaly score for anomalies that are caused by values outside the expected **data range** of particular [query](https://docs.victoriametrics.com/anomaly-detection/components/models#queries). The reasons for such anomalies can be various, such as improperly constructed metricsQL queries, sensor malfunctions, or other issues that lead to unexpected values in the data and require investigation.
|
||||
@@ -30,7 +26,6 @@ Here's an example configuration that sets default anomaly score outside expected
|
||||
```yaml
|
||||
settings:
|
||||
n_workers: 4
|
||||
restore_state: True # restore state from previous run, if available
|
||||
anomaly_score_outside_data_range: 5.0
|
||||
|
||||
schedulers:
|
||||
@@ -97,7 +92,6 @@ Here's an example configuration that uses 4 workers for service's internal paral
|
||||
```yaml
|
||||
settings:
|
||||
n_workers: 4
|
||||
restore_state: False # do not restore state from previous run
|
||||
|
||||
schedulers:
|
||||
periodic:
|
||||
@@ -143,163 +137,3 @@ monitoring:
|
||||
push_frequency: 1m
|
||||
# other monitoring settings
|
||||
```
|
||||
|
||||
|
||||
## State Restoration
|
||||
|
||||
The `restore_state` argument {{% available_from "v1.24.0" anomaly %}} makes `vmanomaly` service **stateful** by persisting and restoring state between runs. If enabled, the service will save the state of anomaly detection models and their training data to local filesystem, allowing for seamless continuation of operations after service restarts.
|
||||
|
||||
By default, `restore_state` is set to `false`, meaning the service will start fresh on each restart, to maintain backward compatibility.
|
||||
|
||||
> This feature requires enabling [on-disk mode](https://docs.victoriametrics.com/anomaly-detection/faq#on-disk-mode) for the models and data. If not enabled, the service will exit with an error when `restore_state` is set to `true`.
|
||||
|
||||
### Benefits
|
||||
|
||||
This feature improves the experience of using the anomaly detection service in several ways:
|
||||
- **Operational continuity**: Production of anomaly scores is resumed from the last known state, minimizing downtime, especially useful in conbination with [periodic schedulers](https://docs.victoriametrics.com/anomaly-detection/components/scheduler/#periodic-scheduler) with `start_from` argument explicitly defined.
|
||||
- **Resource efficiency**: Avoids unnecessary resource and time consumption by not retraining models that have already been trained and remain actual, or querying redundant data from VictoriaMetrics TSDB.
|
||||
- **Config hot-reloading**: Allows for on-the-fly configuration changes with the reuse of unchanged models/data/scheduler combinations, avoiding unnecessary retraining and manual service restarts. *Please note, that hot reloading is not yet supported yet, will be a part of recent releases.*
|
||||
|
||||
### How it works
|
||||
|
||||
**Storage**: The service dumps its state into a database file located at `$VMANOMALY_MODEL_DUMPS_DIR/vmanomaly.db`. This database contains metadata about model configurations, schedulers and references to the trained model instances and their respective data.
|
||||
|
||||
**State restoration**: When the service starts with `restore_state` set to `true`, it will:
|
||||
1. Check for the existence of the database file in the specified directory.
|
||||
2. If the file does not exist, it will create a new database file and initialize the state with the current configuration, training models as needed. If the file exists, then it compares the loaded state with the current configuration to ensure compatibility - what can be reused and what needs to be retrained (e.g., if the model class or hyperparameters have changed, it will not restore the state for that model, same for schedulers or reader queries). For reusable components, previously saved state, including model configurations, trained model instances, and their training data, will be restored.
|
||||
3. Subsequently, it will check for model "staleness" and retrain models if necessary, based on the current configuration and the last training time stored in the database vs next scheduled training time. If the model is **actual**, it will continue to use the previously trained model instances or its training data. If the model is **stale** (e.g. `fit_every` time has passed since the last training), it will retrain the model using the latest data of `fit_window` length from VictoriaMetrics TSDB.
|
||||
|
||||
**State update**: The service periodically saves the updated state after each "atomic" operations, such as (model_alias, query_alias)-based training or inference. This ensures that the state is always up-to-date and can be restored in case of a service restart. [Online models](https://docs.victoriametrics.com/anomaly-detection/components/models/#online-models) are also updated after each inference, while [offline models](https://docs.victoriametrics.com/anomaly-detection/components/models/#offline-models) are only saved after each training operation as they do not change the state during consecutive fit calls.
|
||||
|
||||
**Cleanup behavior**: When `restore_state` is switched from `true` to `false`, the database file is automatically removed on the next service startup to prevent inconsistent behavior. All the artifacts (such as model dumps and data dumps) will be removed as well, so the service will start fresh without any previous state.
|
||||
|
||||
Here's an example configuration that enables state restoration:
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
restore_state: true
|
||||
n_workers: 4
|
||||
|
||||
schedulers:
|
||||
periodic:
|
||||
class: periodic
|
||||
fit_every: 5m
|
||||
fit_window: 3h
|
||||
infer_every: 30s
|
||||
# other schedulers
|
||||
|
||||
models:
|
||||
zscore_online:
|
||||
class: zscore_online
|
||||
z_threshold: 3.5
|
||||
clip_predictions: True
|
||||
# other models
|
||||
|
||||
reader:
|
||||
class: vm
|
||||
datasource_url: 'https://play.victoriametrics.com'
|
||||
tenant_id: "0"
|
||||
queries:
|
||||
example_query:
|
||||
expr: 'rand() + 1' # example query that generates random values between 1 and 2
|
||||
data_range: [1., 2.]
|
||||
# other queries
|
||||
sampling_period: 30s
|
||||
latency_offset: 10ms
|
||||
query_from_last_seen_timestamp: False
|
||||
verify_tls: False
|
||||
# other reader settings
|
||||
|
||||
writer:
|
||||
class: "vm"
|
||||
datasource_url: http://localhost:8428
|
||||
metric_format:
|
||||
__name__: "$VAR"
|
||||
for: "$QUERY_KEY"
|
||||
# other writer settings
|
||||
|
||||
monitoring:
|
||||
push:
|
||||
url: http://localhost:8428
|
||||
push_frequency: 1m
|
||||
# other monitoring settings
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
For a configuration with the following models, queries and schedulers:
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
n_workers: 4
|
||||
restore_state: True # enables state restoration
|
||||
schedulers:
|
||||
periodic_1d:
|
||||
class: periodic
|
||||
fit_every: 1h
|
||||
infer_every: 30s
|
||||
fit_window: 24h
|
||||
models:
|
||||
zscore_online:
|
||||
class: zscore_online
|
||||
z_threshold: 3.5
|
||||
schedulers: ['periodic_1d']
|
||||
prophet:
|
||||
class: prophet
|
||||
schedulers: ['periodic_1d']
|
||||
queries: ['q1', 'q2']
|
||||
args:
|
||||
interval_width: 0.98
|
||||
reader:
|
||||
class: vm
|
||||
datasource_url: 'https://play.victoriametrics.com'
|
||||
tenant_id: "0"
|
||||
queries:
|
||||
q1:
|
||||
expr: 'some_metricsql_query_1'
|
||||
q2:
|
||||
expr: 'some_metricsql_query_2'
|
||||
sampling_period: 30s
|
||||
# other components like writer, monitoring, etc.
|
||||
```
|
||||
|
||||
if the service is restarted in less than 1 hour after the last training (now < next scheduled fit time), it will restore the state of the `zscore_online` and `prophet` models if their signature (class, hyperparameters, schedulers, etc.) has not changed. It will load the trained model instances or their training data from disk and continue producing [anomaly scores](https://docs.victoriametrics.com/anomaly-detection/faq#what-is-anomaly-score) without retraining. If there are changes or new queries added to the configuration, the service will add these to scheduled jobs for fit and infer. That's what is changed and what is restored in a config below:
|
||||
|
||||
```yaml
|
||||
settings:
|
||||
n_workers: 2 # changed, but does not affect state restoration
|
||||
restore_state: True # enables state restoration, still enabled
|
||||
schedulers:
|
||||
periodic_1d: # can be fully reused, no changes
|
||||
class: periodic
|
||||
fit_every: 1h # unchanged, still fits every hour
|
||||
infer_every: 30s # unchanged, still infers every 30 seconds
|
||||
fit_window: 24h # unchanged, still fits on the last 24 hours of data
|
||||
models:
|
||||
zscore_online: # can't be reused, because its `z_threshold` has changed
|
||||
class: zscore_online # unchanged, still the same model class
|
||||
z_threshold: 3.0 # changed, needs retraining!
|
||||
schedulers: ['periodic_1d'] # unchanged, still attached to the same scheduler
|
||||
prophet: # can be partially reused, because its class and schedulers are unchanged but queries have changed
|
||||
class: prophet # unchanged, still the same model class
|
||||
schedulers: ['periodic_1d'] # unchanged, still attached to the same scheduler
|
||||
queries: ['q1', 'q3'] # changed, added new query 'q3', drops 'q2', so (prophet, q2) should be trained from scratch
|
||||
args:
|
||||
interval_width: 0.98 # unchanged, still the same argument
|
||||
reader: # can be partially reused, because its class and datasource URL are unchanged, but queries have changed
|
||||
class: vm # unchanged, still the same reader class
|
||||
datasource_url: 'https://play.victoriametrics.com' # unchanged, still the same datasource URL
|
||||
tenant_id: "0" # unchanged, still the same tenant ID
|
||||
queries:
|
||||
q1:
|
||||
expr: 'some_metricsql_query_1' # unchanged, still the same query
|
||||
q2:
|
||||
expr: 'some_metricsql_query_2' # will be removed, no longer used by any model
|
||||
q3:
|
||||
expr: 'some_metricsql_query_3' # new query, added to the reader, and used by the `prophet` model
|
||||
sampling_period: 30s # unchanged, still the same sampling period
|
||||
# other components like writer, monitoring, etc. remain unchanged
|
||||
```
|
||||
This means that the service upon restart:
|
||||
1. Won't restore the state of `zscore_online` model, because its `z_threshold` argument **has changed**, retraining from scratch is needed on the last `fit_window` = 24 hours of data for `q1`, `q2` and `q3` (as model's `queries` arg is not set so it defaults to all queries found in the reader).
|
||||
2. Will **partially** restore the state of `prophet` model, because its class and schedulers are unchanged, but **only instances trained on timeseries returned by `q1` query**. New fit/infer jobs will be set for new query `q3`. The old query `q2` artifacts will be dropped upon restart - all respective models and data for (`prophet`, `q2`) combination will be removed from the database file and from the disk.
|
||||
|
||||
@@ -387,7 +387,7 @@ services:
|
||||
restart: always
|
||||
vmanomaly:
|
||||
container_name: vmanomaly
|
||||
image: victoriametrics/vmanomaly:v1.24.1
|
||||
image: victoriametrics/vmanomaly:v1.23.3
|
||||
depends_on:
|
||||
- "victoriametrics"
|
||||
ports:
|
||||
|
||||
@@ -18,11 +18,8 @@ according to [these docs](https://docs.victoriametrics.com/victorialogs/quicksta
|
||||
|
||||
## tip
|
||||
|
||||
## [v1.24.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.24.0-victorialogs)
|
||||
|
||||
Released at 2025-06-20
|
||||
|
||||
* FEATURE: add `-http.disableKeepAlive` to disable HTTP keep-alives for incoming connections. The flag could improve load balancing among replicas behind HTTP load balancers. See [#9125](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9125) and [#2395](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2395) for details.
|
||||
* FEATURE: [vlinsert](https://docs.victoriametrics.com/victorialogs/data-ingestion/): add `-insert.maxRetries` command-line flag for limiting retry attempts when sending data to storage nodes. After exhausting retries, data is queued in a retry buffer and new ingestion requests are rejected with HTTP 429 until storage nodes recover. See [#9121](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9121).
|
||||
* BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): support `-` as a timestamp value, as described in [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3).
|
||||
* FEATURE: [`delete` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#delete-pipe): allow deleting all the fields with common prefix via `... | delete prefix*` syntax.
|
||||
* FEATURE: [`fields` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#fields-pipe): allow keeping all the fields with common prefix via `... | fields prefix*` syntax.
|
||||
* FEATURE: [`copy` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#copy-pipe): allow copying all the fields with common prefix to fields with another common prefix via `... | copy old_prefix* as new_prefix*` syntax.
|
||||
@@ -30,10 +27,10 @@ Released at 2025-06-20
|
||||
* FEATURE: [`unpack_json` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_json-pipe): allow unpacking JSON fields with common prefix via `... fields (prefix*)` syntax.
|
||||
* FEATURE: [`unpack_logfmt` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#unpack_logfmt-pipe): allow unpacking JSON fields with common prefix via `... fields (prefix*)` syntax.
|
||||
* FEATURE: [`avg` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the average value over all the fields with common prefix via `avg(prefix*)` syntax.
|
||||
* FEATURE: [`max` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#max-stats): allow calculating the maximum value over all the fields with common prefix via `max(prefix*)` syntax.
|
||||
* FEATURE: [`min` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#min-stats): allow calculating the minimum value over all the fields with common prefix via `min(prefix*)` syntax.
|
||||
* FEATURE: [`median` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#median-stats): allow calculating the median value over all the fields with common prefix via `median(prefix*)` syntax.
|
||||
* FEATURE: [`quantile` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#quantile-stats): allow calculating the maximum value over all the fields with common prefix via `quantile(prefix*)` syntax.
|
||||
* FEATURE: [`max` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the maximum value over all the fields with common prefix via `max(prefix*)` syntax.
|
||||
* FEATURE: [`min` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the minimum value over all the fields with common prefix via `min(prefix*)` syntax.
|
||||
* FEATURE: [`median` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the median value over all the fields with common prefix via `median(prefix*)` syntax.
|
||||
* FEATURE: [`quantile` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#avg-stats): allow calculating the maximum value over all the fields with common prefix via `quantile(prefix*)` syntax.
|
||||
* FEATURE: [`sum` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#sum-stats): allow calculating the sum for all the fields with common prefix via `sum(prefix*)` syntax.
|
||||
* FEATURE: [`sum_len` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#sum_len-stats): allow calculating the sum of byte lengths for all the fields with common prefix via `sum_len(prefix*)` syntax.
|
||||
* FEATURE: [`count` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#count-stats): allow calculating the number of logs with at least a single non-empty field across fields with common prefix via `count(prefix*)` syntax.
|
||||
@@ -47,23 +44,8 @@ Released at 2025-06-20
|
||||
* FEATURE: [`json_values` stats function](https://docs.victoriametrics.com/victorialogs/logsql/#json_values-stats): allow fetching values for all the fields with common prefix via `json_values(prefix*)` syntax.
|
||||
* FEATURE: [`-insert.maxLineSizeBytes`](https://docs.victoriametrics.com/victorialogs/faq/#what-length-a-log-record-is-expected-to-have): add logging of the number of bytes skipped for oversize lines.
|
||||
* FEATURE: add `-insert.disable` and `-select.disable` command-line flags for disabling both public and internal HTTP endpoints (`/insert/*` + `/internal/insert` and `/select/*` + `/internal/select/*` respectively). See [#9061](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9061).
|
||||
* FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): enhance autocomplete with parsed field suggestions from unpack pipe. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8806).
|
||||
* FEATURE: [Journald data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/): automatically add `level` log field according to [the `PRIORITY` field value](https://wiki.archlinux.org/title/Systemd/Journal#Priority_level). This enables [log level highlighting in Grafana](https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level). See [#8535](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8535).
|
||||
* FEATURE: [Syslog data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/): automatically add `level` log field according to [the `severity` field value](https://en.wikipedia.org/wiki/Syslog#Severity_level). This enables [log level highlighting in Grafana](https://grafana.com/docs/grafana/latest/explore/logs-integration/#log-level).
|
||||
* FEATURE: [Journald data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/): use `(_MACHINE_ID, _HOSTNAME, _SYSTEMD_UNIT)` fields as [log stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields) by default. See [#9143](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9143).
|
||||
* FEATURE: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): optimize `(any_filter or *)` filters to `*`. This avoids executing the `any_filter`. Such filters are frequently generated by Grafana.
|
||||
* FEATURE: [`/select/logsql/query` endpoint](https://docs.victoriametrics.com/victorialogs/querying/#querying-logs): optimize the input query after adding the `limit` to it. This improves performance and reduces memory usage for queries ending with [`sort` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#sort-pipe). See [#9200](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9200). Thanks to @vadimalekseev for [the fix](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9201).
|
||||
|
||||
* BUGFIX: [query API](https://docs.victoriametrics.com/victorialogs/querying/#querying-logs): properly set storage node authorization in cluster mode when [Basic Auth](https://docs.victoriametrics.com/victorialogs/cluster/#security) is enabled. See [#9080](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9080).
|
||||
* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): live tailing tab automatically reconnects when the connection is lost. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9129).
|
||||
* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix issue with hits chart ignoring selected AccountID and ProjectID. See [#9157](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9157).
|
||||
* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix missing field values in auto-complete. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8749)
|
||||
* BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): remove the compact mode of the table tab and add field sorting capabilities to the JSON tab. See [#7047](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7047).
|
||||
* BUGFIX: [Journald data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/): properly read log timestamp from `__REALTIME_TIMESTAMP` field according to [the docs](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#time-field). See [#9144](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9144). The bug has been introduced in [v1.22.0-victorialogs](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.22.0-victorialogs).
|
||||
* BUGFIX: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): support `-` as a timestamp value, as described in [RFC5424](https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3).
|
||||
* BUGFIX: [LogsQL](https://docs.victoriametrics.com/victorialogs/logsql/): properly handle quotes inside quoted strings such as `"\""`. Previously this could lead to panics. See [#9219](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9219).
|
||||
* BUGFIX: [LogsQL regexp filter](https://docs.victoriametrics.com/victorialogs/logsql/#regexp-filter): properly parse unquoted filter ending with `*`, such as `foo:~bar.*`. It must be parsed as `foo:~"bar.*"`, while previously it was incorrectly parsed as `foo:~"bar."`.
|
||||
* BUGFIX: [Journald data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/): properly read large Journald requests in streaming manner. See [#9070](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9070). Deprecate `-journald.maxRequestSize` command-line flag, since it is no longer used.
|
||||
|
||||
## [v1.23.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.23.3-victorialogs)
|
||||
|
||||
|
||||
@@ -3362,11 +3362,9 @@ It understands the following Syslog formats:
|
||||
|
||||
The following fields are unpacked:
|
||||
|
||||
- `level` - optained from `PRI`.
|
||||
- `priority` - obtained from `PRI`.
|
||||
- `facility` - calculated as `PRI / 8`.
|
||||
- `facility_keyword` - string representation of the `facility` field according to [these docs](https://en.wikipedia.org/wiki/Syslog#Facility).
|
||||
- `severity` - calculated as `PRI % 8`.
|
||||
- `priority` - it is obtained from `PRI`.
|
||||
- `facility` - it is calculated as `PRI / 8`.
|
||||
- `severity` - it is calculated as `PRI % 8`.
|
||||
- `format` - either `rfc3164` or `rfc5424` depending on which Syslog format is unpacked.
|
||||
- `timestamp` - timestamp in [ISO8601 format](https://en.wikipedia.org/wiki/ISO_8601). The `MMM DD hh:mm:ss` timestamp in [RFC3164](https://datatracker.ietf.org/doc/html/rfc3164)
|
||||
is automatically converted into [ISO8601 format](https://en.wikipedia.org/wiki/ISO_8601) by assuming that the timestamp belongs to the last 12 months.
|
||||
|
||||
@@ -36,8 +36,8 @@ Just download archive for the needed Operating system and architecture, unpack i
|
||||
For example, the following commands download VictoriaLogs archive for Linux/amd64, unpack and run it:
|
||||
|
||||
```sh
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.24.0-victorialogs/victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
|
||||
tar xzf victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.23.3-victorialogs/victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
|
||||
tar xzf victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
|
||||
./victoria-logs-prod -storageDataPath=victoria-logs-data
|
||||
```
|
||||
|
||||
@@ -61,7 +61,7 @@ Here is the command to run VictoriaLogs in a Docker container:
|
||||
|
||||
```sh
|
||||
docker run --rm -it -p 9428:9428 -v ./victoria-logs-data:/victoria-logs-data \
|
||||
docker.io/victoriametrics/victoria-logs:v1.24.0-victorialogs -storageDataPath=victoria-logs-data
|
||||
docker.io/victoriametrics/victoria-logs:v1.23.3-victorialogs -storageDataPath=victoria-logs-data
|
||||
```
|
||||
|
||||
See also:
|
||||
|
||||
@@ -400,8 +400,6 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
Disable CORS for all origins (*)
|
||||
-http.disableKeepAlive
|
||||
Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
|
||||
-http.disableResponseCompression
|
||||
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
|
||||
-http.header.csp string
|
||||
@@ -446,6 +444,8 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 262144)
|
||||
-insert.maxQueueDuration duration
|
||||
The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s)
|
||||
-insert.maxRetries int
|
||||
The maximum number of retry attempts when sending data to storage nodes. After exhausting retries, the data is queued in a retry buffer and new ingestion requests are rejected with HTTP 429 until storage nodes recover. (default 20)
|
||||
-internStringCacheExpireDuration duration
|
||||
The expiry duration for caches for interned strings. See https://en.wikipedia.org/wiki/String_interning . See also -internStringMaxLen and -internStringDisableCache (default 6m0s)
|
||||
-internStringDisableCache
|
||||
@@ -465,6 +465,9 @@ Pass `-help` to VictoriaLogs in order to see the list of supported command-line
|
||||
Value can contain comma inside single-quoted or double-quoted string, {}, [] and () braces.
|
||||
-journald.includeEntryMetadata
|
||||
Include journal entry fields, which with double underscores.
|
||||
-journald.maxRequestSize size
|
||||
The maximum size in bytes of a single journald request
|
||||
Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 67108864)
|
||||
-journald.streamFields array
|
||||
Comma-separated list of fields to use as log stream fields for logs ingested over journald protocol. See https://docs.victoriametrics.com/victorialogs/data-ingestion/journald/#stream-fields
|
||||
Supports an array of values separated by comma or specified via multiple flags.
|
||||
|
||||
@@ -155,8 +155,8 @@ The following guide covers the following topics for Linux host:
|
||||
Download and unpack the latest VictoriaLogs release:
|
||||
|
||||
```sh
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.24.0-victorialogs/victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
|
||||
tar xzf victoria-logs-linux-amd64-v1.24.0-victorialogs.tar.gz
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.23.3-victorialogs/victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
|
||||
tar xzf victoria-logs-linux-amd64-v1.23.3-victorialogs.tar.gz
|
||||
```
|
||||
|
||||
Start the first [`vlstorage` node](#architecture), which accepts incoming requests at the port `9491` and stores the ingested logs at `victoria-logs-data-1` directory:
|
||||
|
||||
@@ -22,29 +22,12 @@ Substitute the `localhost:9428` address inside `endpoints` section with the real
|
||||
|
||||
## Time field
|
||||
|
||||
VictoriaLogs uses the `__REALTIME_TIMESTAMP` field as [`_time` field](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field)
|
||||
for the logs ingested via journald protocol. Other field can be used instead of `__REALTIME_TIMESTAMP` by specifying it via `-journald.timeField` command-line flag.
|
||||
By default VictoriaLogs use the `__REALTIME_TIMESTAMP` field as [timestamp](https://docs.victoriametrics.com/victorialogs/keyconcepts/#time-field)
|
||||
for the logs ingested via journald protocol. This can be modified by setting the `-journald.timeField` command-line flag to the log field name,
|
||||
which contains the needed timestamp.
|
||||
|
||||
See [the list of supported Journald fields](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
|
||||
|
||||
## Level field
|
||||
|
||||
VictoriaLogs atomatically sets the `level` log field according to the [`PRIORITY` field falue](https://wiki.archlinux.org/title/Systemd/Journal).
|
||||
|
||||
## Stream fields
|
||||
|
||||
VictoriaLogs uses `(_MACHINE_ID, _HOSTNAME, _SYSTEMD_UNIT)` as [stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||
for logs ingested via jorunald protocol. The list of log stream fields can be changed via `-journald.streamFields` command-line flag if needed,
|
||||
by providing comma-separated list of journald fields form [this list](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
|
||||
|
||||
Please make sure that the log stream fields passed to `-journlad.streamFields` do not contain fields with high number or unbound number of unique values,
|
||||
since this may lead to [high cardinality issues](https://docs.victoriametrics.com/victorialogs/keyconcepts/#high-cardinality).
|
||||
|
||||
The following Journald fields are also good candidates for stream fields:
|
||||
|
||||
- `_TRANSPORT`
|
||||
- `_SYSTEMD_USER_UNIT`
|
||||
|
||||
|
||||
## Dropping fields
|
||||
|
||||
VictoriaLogs can be configured for skipping the given [log fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#data-model)
|
||||
@@ -53,6 +36,13 @@ This list can contain log field prefixes ending with `*` such as `some-prefix*`.
|
||||
|
||||
See [the list of supported Journald fields](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
|
||||
|
||||
## Stream fields
|
||||
|
||||
VictoriaLogs can be configured to use the particular fields as [log stream fields](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields)
|
||||
for logs ingested via jorunald protocol, via `-journald.streamFields` command-line flag, which accepts comma-separated list of fields to use as log stream fields.
|
||||
|
||||
See [the list of supported Journald fields](https://www.freedesktop.org/software/systemd/man/latest/systemd.journal-fields.html).
|
||||
|
||||
## Multitenancy
|
||||
|
||||
By default VictoriaLogs stores logs ingested via journald protocol into `(AccountID=0, ProjectID=0)` [tenant](https://docs.victoriametrics.com/victorialogs/#multitenancy).
|
||||
|
||||
@@ -47,9 +47,7 @@ from the received Syslog lines:
|
||||
- [`_msg`](https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field) - the `MESSAGE` field from the supported syslog formats above
|
||||
- `hostname`, `app_name` and `proc_id` - for unique identification of [log streams](https://docs.victoriametrics.com/victorialogs/keyconcepts/#stream-fields).
|
||||
It is possible to change the list of fields for log streams - see [these docs](#stream-fields).
|
||||
- `level` - string representation of the log level according to the `<PRI>` field value
|
||||
- `priority`, `facility` and `severity` - these fields are extracted from `<PRI>` field
|
||||
- `facility_keyword` - string representation of the `facility` field according to [these docs](https://en.wikipedia.org/wiki/Syslog#Facility)
|
||||
- `format` - this field is set to either `rfc3164` or `rfc5424` depending on the format of the parsed syslog line
|
||||
- `msg_id` - `MSGID` field from log line in `RFC5424` format.
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ or from docker images at [Docker Hub](https://hub.docker.com/r/victoriametrics/v
|
||||
### Running `vlogscli` from release binary
|
||||
|
||||
```sh
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.24.0-victorialogs/vlogscli-linux-amd64-v1.24.0-victorialogs.tar.gz
|
||||
tar xzf vlogscli-linux-amd64-v1.24.0-victorialogs.tar.gz
|
||||
curl -L -O https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.23.3-victorialogs/vlogscli-linux-amd64-v1.23.3-victorialogs.tar.gz
|
||||
tar xzf vlogscli-linux-amd64-v1.23.3-victorialogs.tar.gz
|
||||
./vlogscli-prod
|
||||
```
|
||||
|
||||
@@ -48,6 +48,7 @@ which queries `(AccountID=123, ProjectID=456)` [tenant](https://docs.victoriamet
|
||||
./vlogscli -header='AccountID: 123' -header='ProjectID: 456'
|
||||
```
|
||||
|
||||
|
||||
## Multitenancy
|
||||
|
||||
`AccountID` and `ProjectID` [values](https://docs.victoriametrics.com/victorialogs/#multitenancy)
|
||||
|
||||
@@ -1164,8 +1164,6 @@ Below is the output for `/path/to/vminsert -help`:
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
Disable CORS for all origins (*)
|
||||
-http.disableKeepAlive
|
||||
Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
|
||||
-http.disableResponseCompression
|
||||
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
|
||||
-http.header.csp string
|
||||
@@ -1472,8 +1470,6 @@ Below is the output for `/path/to/vmselect -help`:
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
Disable CORS for all origins (*)
|
||||
-http.disableKeepAlive
|
||||
Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
|
||||
-http.disableResponseCompression
|
||||
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
|
||||
-http.header.csp string
|
||||
@@ -1815,8 +1811,6 @@ Below is the output for `/path/to/vmstorage -help`:
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
Disable CORS for all origins (*)
|
||||
-http.disableKeepAlive
|
||||
Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
|
||||
-http.disableResponseCompression
|
||||
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
|
||||
-http.header.csp string
|
||||
|
||||
@@ -2459,8 +2459,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
Disable CORS for all origins (*)
|
||||
-http.disableKeepAlive
|
||||
Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
|
||||
-http.disableResponseCompression
|
||||
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
|
||||
-http.header.csp string
|
||||
|
||||
@@ -18,27 +18,16 @@ See also [LTS releases](https://docs.victoriametrics.com/victoriametrics/lts-rel
|
||||
|
||||
## tip
|
||||
|
||||
## [v1.120.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.120.0)
|
||||
|
||||
Released at 2025-06-20
|
||||
|
||||
* SECURITY: upgrade Go builder from Go1.24.3 to Go1.24.4. See [the list of issues addressed in Go1.24.4](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved).
|
||||
* SECURITY: upgrade base docker image (Alpine) from 3.21.3 to 3.22.0. See [Alpine 3.22.0 release notes](https://alpinelinux.org/posts/Alpine-3.22.0-released.html).
|
||||
|
||||
* FEATURE: all the VictoriaMetrics components: add `-http.disableKeepAlive` to disable HTTP keep-alives for incoming connections. The flag could improve load balancing among replicas behind HTTP load balancers. See [#9125](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9125) and [#2395](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/2395) for details.
|
||||
* FEATURE: [dashboards/cluster](https://grafana.com/grafana/dashboards/11176): add panel `Partitions scheduled for re-processing` to `Troubleshooting` row. It shows the amount of data scheduled for [downsampling](https://docs.victoriametrics.com/#downsampling) or [retention filters](https://docs.victoriametrics.com/#retention-filters). The new panel should help to correlate resource usage with background re-processing of partitions.
|
||||
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): support [rate_prometheus](https://docs.victoriametrics.com/victoriametrics/metricsql/#rate_prometheus) function, an equivalent to `increase_prometheus(series_selector[d]) / d`. See [#8901](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8901) and [#8891](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891) for details.
|
||||
* FEATURE: [MetricsQL](https://docs.victoriametrics.com/victoriametrics/metricsql/): respect staleness markers when calculating `rate` and `increase` functions. The new behavior will interrupt rate/increase calculation if last sample on the selected time window is a [staleness marker](https://docs.victoriametrics.com/victoriametrics/vmagent/#prometheus-staleness-markers), making the series to disappear immediately instead of slowly fading away. See more details in [#8891-comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8891#issuecomment-2875542721).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): do not break vmalert process under replay mode when rule uses `query` template, but only logging a warning.
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/victoriametrics/vmalert/): correct the rule evaluation timestamp if the system clock is changed during runtime. See [#8790](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/8790).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert/): respect [group](https://docs.victoriametrics.com/victoriametrics/vmalert/#groups) `concurrency` setting in [replay mode](https://docs.victoriametrics.com/victoriametrics/vmalert/#rules-backfilling) when `-replay.rulesDelay=0`. See this [#7387](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/7387) for details. Thanks to @BenNF for the [PR #9214](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9214).
|
||||
* FEATURE: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): allow overriding default limits for in-memory cache `storage/metricName` via flag `-storage.cacheSizeStorageMetricName`.
|
||||
|
||||
* BUGFIX: [vmsingle](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/): fix incorrect sorting of tag filters, which led to suboptimal tag filter evaluation order and potentially degraded query performance in rare cases. See [#9127](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/9127) for details.
|
||||
* BUGFIX: [vmbackup](https://docs.victoriametrics.com/vmbackup/), [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): fix server-side copying of objects for Azure Blob Storage when using managed identity for authentication. Previously, it wasn't possible to use [smart backups](https://docs.victoriametrics.com/victoriametrics/vmbackup/#smart-backups) strategy for `vmbackup` as server-side copy would fail. See [#9131](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9131).
|
||||
* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager/): increase startup healthcheck delay for storage reachability from 30 seconds to 3 minutes. This is required to avoid vmbackupmanager restarts when storage node startup take more than 30 seconds (e.g. when storage nodes stores more than 5Tb of data).
|
||||
* BUGFIX: [VictoriaMetrics Enterprise](https://docs.victoriametrics.com/enterprise.html) cluster: properly include FIPS binaries in release artifacts for platforms other than windows. Previously, FIPS binaries were only included in windows release artifacts. See [#9188](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9188).
|
||||
* BUGFIX: [stream aggregation](https://docs.victoriametrics.com/victoriametrics/stream-aggregation/): properly calculate `rate_sum` and `rate_avg` aggregations if aggregation `interval` is smaller than distance between samples timestamps. See [#9017](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/9017).
|
||||
|
||||
## [v1.119.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.119.0)
|
||||
|
||||
|
||||
@@ -1552,8 +1552,6 @@ See the docs at https://docs.victoriametrics.com/victoriametrics/vmagent/ .
|
||||
Incoming connections to -httpListenAddr are closed after the configured timeout. This may help evenly spreading load among a cluster of services behind TCP-level load balancer. Zero value disables closing of incoming connections (default 2m0s)
|
||||
-http.disableCORS
|
||||
Disable CORS for all origins (*)
|
||||
-http.disableKeepAlive
|
||||
Whether to disable HTTP keep-alive for incoming connections at -httpListenAddr
|
||||
-http.disableResponseCompression
|
||||
Disable compression of HTTP responses to save CPU resources. By default, compression is enabled to save network bandwidth
|
||||
-http.header.csp string
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user